From 74f483a6f595c4b33b8bcbf56c81e9a05aa558bb Mon Sep 17 00:00:00 2001 From: Tom Hvitved Date: Thu, 2 Nov 2023 11:07:57 +0100 Subject: [PATCH] C#: Update model conversion queries --- .../csharp/dataflow/internal/ExternalFlow.qll | 57 ++++++++++--------- .../utils/modelconverter/ConvertExtensions.py | 10 +++- .../utils/modelconverter/ExtractNeutrals.ql | 14 +++-- .../src/utils/modelconverter/ExtractSinks.ql | 11 ++-- .../utils/modelconverter/ExtractSources.ql | 11 ++-- .../utils/modelconverter/ExtractSummaries.ql | 11 ++-- .../utils/modelconverter/InterpretModel.qll | 21 +++++++ .../models-as-data/convert_extensions.py | 23 +++++--- 8 files changed, 101 insertions(+), 57 deletions(-) create mode 100644 csharp/ql/src/utils/modelconverter/InterpretModel.qll diff --git a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/ExternalFlow.qll b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/ExternalFlow.qll index 04f1f0517b4..778e5f99d3c 100644 --- a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/ExternalFlow.qll +++ b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/ExternalFlow.qll @@ -371,31 +371,21 @@ private string paramsString(InterpretedCallable c) { ) } +/** Gets the source/sink/summary/neutral base declaration corresponding to the supplied parameters. */ pragma[nomagic] -private Element interpretElement0( - string namespace, string type, boolean subtypes, string name, string signature -) { - exists(UnboundValueOrRefType t | elementSpec(namespace, type, subtypes, name, signature, _, t) | - exists(Declaration m | - ( - result = m - or - subtypes = true and result.(UnboundCallable).overridesOrImplementsUnbound(m) - ) and - m.getDeclaringType() = t and - hasName(m, name) - | - signature = "" - or - paramsString(m) = signature - ) +Declaration interpretBaseDeclaration(string namespace, string type, string name, string signature) { + exists(UnboundValueOrRefType t | elementSpec(namespace, type, _, name, signature, _, t) | + result = + any(Declaration d | + d.getDeclaringType() = t and + hasName(d, name) and + ( + signature = "" + or + paramsString(d) = signature + ) + ) or - ( - result = t - or - subtypes = true and - result = t.getASubTypeUnbound+() - ) and result = t and name = "" and signature = "" @@ -403,14 +393,27 @@ private Element interpretElement0( } /** Gets the source/sink/summary/neutral element corresponding to the supplied parameters. */ +pragma[nomagic] Element interpretElement( string namespace, string type, boolean subtypes, string name, string signature, string ext ) { elementSpec(namespace, type, subtypes, name, signature, ext) and - exists(Element e | e = interpretElement0(namespace, type, subtypes, name, signature) | - ext = "" and result = e + exists(Declaration base, Declaration d | + base = interpretBaseDeclaration(namespace, type, name, signature) and + ( + d = base + or + subtypes = true and + ( + d.(UnboundCallable).overridesOrImplementsUnbound(base) + or + d = base.(UnboundValueOrRefType).getASubTypeUnbound+() + ) + ) + | + ext = "" and result = d or - ext = "Attribute" and result.(Attributable).getAnAttribute().getType() = e + ext = "Attribute" and result.(Attributable).getAnAttribute().getType() = d ) } @@ -491,7 +494,7 @@ string parameterQualifiedTypeNamesToString(Callable c) { concat(int i, string s | s = parameterQualifiedType(c.getParameter(i)) | s, "," order by i) } -private predicate partialModel( +predicate partialModel( UnboundCallable c, string namespace, string type, string name, string parameters ) { QN::hasQualifiedName(c, namespace, type, name) and diff --git a/csharp/ql/src/utils/modelconverter/ConvertExtensions.py b/csharp/ql/src/utils/modelconverter/ConvertExtensions.py index 47c013f418e..e393dc2f9c9 100644 --- a/csharp/ql/src/utils/modelconverter/ConvertExtensions.py +++ b/csharp/ql/src/utils/modelconverter/ConvertExtensions.py @@ -27,7 +27,15 @@ projectDir = os.path.join(workDir, "project") dbDir = os.path.join(workDir, "db") # Make dummy project -helpers.run_cmd(['dotnet', 'new', 'console', '-o', projectDir], "Failed to create dummy project.") +helpers.run_cmd(['dotnet', 'new', 'webapp', '-o', projectDir], "Failed to create dummy project.") +# Add nuget packages for all packages where we have models +helpers.run_cmd(['dotnet', 'add', projectDir, 'package', 'Newtonsoft.Json']) +helpers.run_cmd(['dotnet', 'add', projectDir, 'package', 'Microsoft.EntityFrameworkCore']) +helpers.run_cmd(['dotnet', 'add', projectDir, 'package', 'Microsoft.EntityFrameworkCore.Relational']) +helpers.run_cmd(['dotnet', 'add', projectDir, 'package', 'Dapper']) +helpers.run_cmd(['dotnet', 'add', projectDir, 'package', 'ServiceStack']) +helpers.run_cmd(['dotnet', 'add', projectDir, 'package', 'ServiceStack.OrmLite']) +helpers.run_cmd(['dotnet', 'add', projectDir, 'package', 'System.Collections.Immutable']) helpers.run_cmd(['codeql', 'database', 'create', f'--language={language}', '-c', f'dotnet build {projectDir}', dbDir], "Failed to create dummy database.") print('Converting data extensions for C#.') diff --git a/csharp/ql/src/utils/modelconverter/ExtractNeutrals.ql b/csharp/ql/src/utils/modelconverter/ExtractNeutrals.ql index 7940f319c02..830365e161d 100644 --- a/csharp/ql/src/utils/modelconverter/ExtractNeutrals.ql +++ b/csharp/ql/src/utils/modelconverter/ExtractNeutrals.ql @@ -5,11 +5,13 @@ */ import csharp -import semmle.code.csharp.dataflow.internal.ExternalFlow +import InterpretModel -from string package, string type, string name, string signature, string kind, string provenance +from + string namespace0, string namespace, string type0, string type, string name0, string name, + string signature0, string signature, string kind, string provenance where - neutralModel(package, type, name, signature, kind, provenance) and - not provenance.matches("%generated") -select package, type, name, signature, kind, provenance order by - package, type, name, signature, kind + neutralModel(namespace0, type0, name0, signature0, kind, provenance) and + interpretCallable(namespace0, namespace, type0, type, name0, name, signature0, signature) +select namespace, type, name, signature, kind, provenance order by + namespace, type, name, signature, kind diff --git a/csharp/ql/src/utils/modelconverter/ExtractSinks.ql b/csharp/ql/src/utils/modelconverter/ExtractSinks.ql index 58302da2d04..1f3f3b99021 100644 --- a/csharp/ql/src/utils/modelconverter/ExtractSinks.ql +++ b/csharp/ql/src/utils/modelconverter/ExtractSinks.ql @@ -5,13 +5,14 @@ */ import csharp -import semmle.code.csharp.dataflow.internal.ExternalFlow +import InterpretModel from - string namespace, string type, boolean subtypes, string name, string signature, string ext, - string input, string kind, string provenance + string namespace0, string namespace, string type0, string type, boolean subtypes, string name0, + string name, string signature0, string signature, string ext, string input, string kind, + string provenance where - sinkModel(namespace, type, subtypes, name, signature, ext, input, kind, provenance) and - not provenance.matches("%generated") + sinkModel(namespace0, type0, subtypes, name0, signature0, ext, input, kind, provenance) and + interpretCallable(namespace0, namespace, type0, type, name0, name, signature0, signature) select namespace, type, subtypes, name, signature, ext, input, kind, provenance order by namespace, type, name, signature, input, kind diff --git a/csharp/ql/src/utils/modelconverter/ExtractSources.ql b/csharp/ql/src/utils/modelconverter/ExtractSources.ql index 0478f47937a..ad2ad46bfcc 100644 --- a/csharp/ql/src/utils/modelconverter/ExtractSources.ql +++ b/csharp/ql/src/utils/modelconverter/ExtractSources.ql @@ -5,13 +5,14 @@ */ import csharp -import semmle.code.csharp.dataflow.internal.ExternalFlow +import InterpretModel from - string namespace, string type, boolean subtypes, string name, string signature, string ext, - string output, string kind, string provenance + string namespace0, string namespace, string type0, string type, boolean subtypes, string name0, + string name, string signature0, string signature, string ext, string output, string kind, + string provenance where - sourceModel(namespace, type, subtypes, name, signature, ext, output, kind, provenance) and - not provenance.matches("%generated") + sourceModel(namespace0, type0, subtypes, name0, signature0, ext, output, kind, provenance) and + interpretCallable(namespace0, namespace, type0, type, name0, name, signature0, signature) select namespace, type, subtypes, name, signature, ext, output, kind, provenance order by namespace, type, name, signature, output, kind diff --git a/csharp/ql/src/utils/modelconverter/ExtractSummaries.ql b/csharp/ql/src/utils/modelconverter/ExtractSummaries.ql index 26239226a8c..93769c5b384 100644 --- a/csharp/ql/src/utils/modelconverter/ExtractSummaries.ql +++ b/csharp/ql/src/utils/modelconverter/ExtractSummaries.ql @@ -5,13 +5,14 @@ */ import csharp -import semmle.code.csharp.dataflow.internal.ExternalFlow +import InterpretModel from - string namespace, string type, boolean subtypes, string name, string signature, string ext, - string input, string output, string kind, string provenance + string namespace0, string namespace, string type0, string type, boolean subtypes, string name0, + string name, string signature0, string signature, string ext, string input, string output, + string kind, string provenance where - summaryModel(namespace, type, subtypes, name, signature, ext, input, output, kind, provenance) and - not provenance.matches("%generated") + summaryModel(namespace0, type0, subtypes, name0, signature0, ext, input, output, kind, provenance) and + interpretCallable(namespace0, namespace, type0, type, name0, name, signature0, signature) select namespace, type, subtypes, name, signature, ext, input, output, kind, provenance order by namespace, type, name, signature, input, output, kind diff --git a/csharp/ql/src/utils/modelconverter/InterpretModel.qll b/csharp/ql/src/utils/modelconverter/InterpretModel.qll new file mode 100644 index 00000000000..f8e646421ff --- /dev/null +++ b/csharp/ql/src/utils/modelconverter/InterpretModel.qll @@ -0,0 +1,21 @@ +import csharp +import semmle.code.csharp.dataflow.internal.ExternalFlow + +bindingset[namespace0, type0, name0, signature0] +predicate interpretCallable( + string namespace0, string namespace, string type0, string type, string name0, string name, + string signature0, string signature +) { + exists(Callable c, string signature1 | + c = interpretBaseDeclaration(namespace0, type0, name0, signature0) and + partialModel(c, namespace, type, name, signature1) and + if signature0 = "" then signature = "" else signature = signature1 + ) + or + // if the row cannot be parsed (e.g. if the element is not in the DB), return the existing row unchanged + not exists(interpretBaseDeclaration(namespace0, type0, name0, signature0)) and + namespace = namespace0 and + type = type0 and + name = name0 and + signature = signature0 +} diff --git a/misc/scripts/models-as-data/convert_extensions.py b/misc/scripts/models-as-data/convert_extensions.py index 51e1e30f389..5c45c7919c5 100644 --- a/misc/scripts/models-as-data/convert_extensions.py +++ b/misc/scripts/models-as-data/convert_extensions.py @@ -29,10 +29,12 @@ def merge(*dicts): return merged def parseData(data): - rows = { } + rows = [{ }, { }] for row in data: d = map(quote_if_needed, row) - insert_update(rows, row[0], " - [" + ', '.join(d) + ']\n') + provenance = row[-1] + targetRows = rows[1] if provenance.endswith("generated") else rows[0] + insert_update(targetRows, row[0], " - [" + ', '.join(d) + ']\n') return rows @@ -57,9 +59,10 @@ class Converter: def asAddsTo(self, rows, predicate): - extensions = { } - for key in rows: - extensions[key] = helpers.addsToTemplate.format(f"codeql/{self.language}-all", predicate, rows[key]) + extensions = [{ }, { }] + for i in range(2): + for key in rows[i]: + extensions[i][key] = helpers.addsToTemplate.format(f"codeql/{self.language}-all", predicate, rows[i][key]) return extensions @@ -75,7 +78,7 @@ class Converter: sources = self.getAddsTo("ExtractSources.ql", helpers.sourceModelPredicate) sinks = self.getAddsTo("ExtractSinks.ql", helpers.sinkModelPredicate) neutrals = self.getAddsTo("ExtractNeutrals.ql", helpers.neutralModelPredicate) - return merge(sources, sinks, summaries, neutrals) + return [merge(sources[0], sinks[0], summaries[0], neutrals[0]), merge(sources[1], sinks[1], summaries[1], neutrals[1])] def save(self, extensions): @@ -85,9 +88,13 @@ class Converter: # Create a file for each namespace and save models. extensionTemplate = """extensions: {0}""" - for entry in extensions: + for entry in extensions[0]: with open(self.extDir + "/" + entry + self.modelFileExtension, "w") as f: - f.write(extensionTemplate.format(extensions[entry])) + f.write(extensionTemplate.format(extensions[0][entry])) + + for entry in extensions[1]: + with open(self.extDir + "/generated/" + entry + self.modelFileExtension, "w") as f: + f.write(extensionTemplate.format(extensions[1][entry])) def run(self): extensions = self.makeContent()