diff --git a/cpp/ql/lib/qlpack.yml b/cpp/ql/lib/qlpack.yml index 8d15b6d142a..27529f55a29 100644 --- a/cpp/ql/lib/qlpack.yml +++ b/cpp/ql/lib/qlpack.yml @@ -9,6 +9,7 @@ dependencies: codeql/dataflow: ${workspace} codeql/rangeanalysis: ${workspace} codeql/ssa: ${workspace} + codeql/typeflow: ${workspace} codeql/tutorial: ${workspace} codeql/util: ${workspace} codeql/xml: ${workspace} diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternalsCommon.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternalsCommon.qll index 862070820f8..7e5b4de8122 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternalsCommon.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/SsaInternalsCommon.qll @@ -6,6 +6,7 @@ private import DataFlowImplCommon as DataFlowImplCommon private import DataFlowUtil private import semmle.code.cpp.models.interfaces.PointerWrapper private import DataFlowPrivate +private import TypeFlow private import semmle.code.cpp.ir.ValueNumbering /** @@ -955,11 +956,7 @@ private module Cached { * Holds if the address computed by `operand` is guaranteed to write * to a specific address. */ - private predicate isCertainAddress(Operand operand) { - valueNumberOfOperand(operand).getAnInstruction() instanceof VariableAddressInstruction - or - operand.getType() instanceof Cpp::ReferenceType - } + private predicate isCertainAddress(Operand operand) { isPointerToSingleObject(operand.getDef()) } /** * Holds if `address` is a use of an SSA variable rooted at `base`, and the diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll new file mode 100644 index 00000000000..69f94dad91b --- /dev/null +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TypeFlow.qll @@ -0,0 +1,278 @@ +private import cpp +private import semmle.code.cpp.ir.IR +private import codeql.typeflow.TypeFlow + +private module Input implements TypeFlowInput { + /** Holds if `alloc` dynamically allocates a single object. */ + private predicate isSingleObjectAllocation(AllocationExpr alloc) { + // i.e., `new int`; + alloc instanceof NewExpr + or + // i.e., `malloc(sizeof(int))` + exists(SizeofTypeOperator sizeOf | sizeOf = alloc.getSizeExpr() | + not sizeOf.getTypeOperand().getUnspecifiedType() instanceof ArrayType + ) + } + + /** + * Holds if `i` is the result of a dynamic allocation. + * + * `isObject` is `true` if the allocation allocated a single object, + * and `false` otherwise. + */ + private predicate isAllocation(Instruction i, boolean isObject) { + exists(AllocationExpr alloc | alloc = i.getUnconvertedResultExpression() | + if isSingleObjectAllocation(alloc) then isObject = true else isObject = false + ) + } + + private predicate hasExactSingleType(Instruction i) { + // The address of a variable is always a single object + i instanceof VariableAddressInstruction + or + // A reference always points to a single object + i.getResultLanguageType().hasUnspecifiedType(any(ReferenceType rt), false) + or + // `this` is never an array + i instanceof InitializeThisInstruction + or + // An allocation of a non-array object + isAllocation(i, true) + } + + private predicate hasExactBufferType(Instruction i) { + // Anything with an array type is a buffer + i.getResultLanguageType().hasUnspecifiedType(any(ArrayType at), false) + or + // An allocation expression that we couldn't conclude allocated a single + // expression is assigned a buffer type. + isAllocation(i, false) + } + + private newtype TTypeFlowNode = + TInstructionNode(Instruction i) or + TFunctionNode(IRFunction func) + + abstract class TypeFlowNode extends TTypeFlowNode { + /** Gets a textual representation of this node. */ + abstract string toString(); + + /** + * Gets the type of this node. This type may not be the most precise + * possible type, but will be used as a starting point of the analysis. + */ + abstract Type getType(); + + /** Gets the location of this node. */ + abstract Location getLocation(); + + /** Gets the underlying `Instruction` of this node, if any. */ + Instruction asInstruction() { none() } + + /** Gets the underlying `IRFunction` of this node, if any. */ + IRFunction asFunction() { none() } + + /** Holds if the value of this node is always null. */ + abstract predicate isNullValue(); + } + + private class InstructionNode extends TypeFlowNode, TInstructionNode { + Instruction instr; + + InstructionNode() { this = TInstructionNode(instr) } + + override string toString() { result = instr.toString() } + + override Type getType() { + if hasExactSingleType(instr) then result.isSingle() else result.isBuffer() + } + + override Location getLocation() { result = instr.getLocation() } + + override Instruction asInstruction() { result = instr } + + override predicate isNullValue() { + instr.(ConstantInstruction).getValue() = "0" and + instr.getResultIRType() instanceof IRAddressType + } + } + + /** Gets the `TypeFlowNode` corresponding to `i`. */ + additional InstructionNode instructionNode(Instruction i) { result.asInstruction() = i } + + private class FunctionNode extends TypeFlowNode, TFunctionNode { + IRFunction func; + + FunctionNode() { this = TFunctionNode(func) } + + override string toString() { result = func.toString() } + + Instruction getReturnValueInstruction() { + result = func.getReturnInstruction().(ReturnValueInstruction).getReturnValue() + } + + override Type getType() { result = instructionNode(this.getReturnValueInstruction()).getType() } + + override Location getLocation() { result = func.getLocation() } + + override IRFunction asFunction() { result = func } + + override predicate isNullValue() { + instructionNode(this.getReturnValueInstruction()).isNullValue() + } + } + + /** + * Gets an ultimiate definition of `phi`. That is, an input to `phi` that is + * not itself a `PhiInstruction`. + */ + private Instruction getAnUltimateLocalDefinition(PhiInstruction phi) { + result = phi.getAnInput*() and not result instanceof PhiInstruction + } + + /** + * Holds if this function is private (i.e., cannot be accessed outside its + * compilation unit). This means we can use a closed-world assumption about + * calls to this function. + */ + private predicate isPrivate(Function func) { + // static functions have internal linkage + func.isStatic() + or + // anonymous namespaces have internal linkage + func.getNamespace().getParentNamespace*().isAnonymous() + or + // private member functions are only called internally from inside the class + func.(MemberFunction).isPrivate() + } + + /** + * Holds if `arg` is an argument for the parameter `p` in a private callable. + */ + pragma[nomagic] + private predicate privateParamArg(InitializeParameterInstruction p, Instruction arg) { + exists(CallInstruction call, int i, Function func | + call.getArgument(pragma[only_bind_into](i)) = arg and + func = call.getStaticCallTarget() and + func.getParameter(pragma[only_bind_into](i)) = p.getParameter() and + isPrivate(func) + ) + } + + predicate joinStep(TypeFlowNode n1, TypeFlowNode n2) { + // instruction -> phi + getAnUltimateLocalDefinition(n2.asInstruction()) = n1.asInstruction() + or + // return value -> function + n2.(FunctionNode).getReturnValueInstruction() = n1.asInstruction() + or + // function -> call + exists(Function func | func = n1.asFunction().getFunction() | + not func.isVirtual() and + n2.asInstruction().(CallInstruction).getStaticCallTarget() = func + ) + or + // Argument -> parameter where the parameter's enclosing function + // is "private". + exists(Instruction arg, Instruction p | + privateParamArg(p, arg) and + n1.asInstruction() = arg and + n2.asInstruction() = p + ) + } + + /** + * Holds if knowing whether `i1` points to a single object or buffer implies + * knowing whether `i2` points to a single object or buffer. + */ + private predicate instructionStep(Instruction i1, Instruction i2) { + i2.(CopyInstruction).getSourceValue() = i1 + or + i2.(CopyValueInstruction).getSourceValue() = i1 + or + i2.(ConvertInstruction).getUnary() = i1 + or + i2.(CheckedConvertOrNullInstruction).getUnary() = i1 + or + i2.(InheritanceConversionInstruction).getUnary() = i1 + or + i2.(PointerArithmeticInstruction).getLeft() = i1 + } + + predicate step(TypeFlowNode n1, TypeFlowNode n2) { + instructionStep(n1.asInstruction(), n2.asInstruction()) + } + + predicate isNullValue(TypeFlowNode n) { n.isNullValue() } + + private newtype TType = + TSingle() or + TBuffer() + + class Type extends TType { + string toString() { + this.isSingle() and + result = "Single" + or + this.isBuffer() and + result = "Buffer" + } + + /** Holds if this type is the type that represents a single object. */ + predicate isSingle() { this = TSingle() } + + /** Holds if this type is the type that represents a buffer. */ + predicate isBuffer() { this = TBuffer() } + + /** + * Gets a super type of this type, if any. + * + * The type relation is `Single <: Buffer`. + */ + Type getASupertype() { + this.isSingle() and + result.isBuffer() + } + } + + predicate exactTypeBase(TypeFlowNode n, Type t) { + exists(Instruction instr | instr = n.asInstruction() | + hasExactSingleType(instr) and t.isSingle() + or + hasExactBufferType(instr) and t.isBuffer() + ) + } + + pragma[nomagic] + private predicate upcastCand(TypeFlowNode n, Type t1, Type t2) { + exists(TypeFlowNode next | + step(n, next) + or + joinStep(n, next) + | + n.getType() = t1 and + next.getType() = t2 and + t1 != t2 + ) + } + + private predicate upcast(TypeFlowNode n, Type t1) { + exists(Type t2 | upcastCand(n, t1, t2) | + // No need for transitive closure since the subtyping relation is just `Single <: Buffer` + t1.getASupertype() = t2 + ) + } + + predicate typeFlowBaseCand(TypeFlowNode n, Type t) { upcast(n, t) } +} + +private module TypeFlow = Make; + +/** + * Holds if `i` is an instruction that computes an address that points to a + * single object (as opposed to pointing into a buffer). + */ +pragma[nomagic] +predicate isPointerToSingleObject(Instruction i) { + TypeFlow::bestTypeFlow(Input::instructionNode(i), any(Input::Type t | t.isSingle()), _) +} diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/dataflow-consistency.expected b/cpp/ql/test/library-tests/dataflow/dataflow-tests/dataflow-consistency.expected index fa6958d92ea..109f5ffebd1 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/dataflow-consistency.expected +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/dataflow-consistency.expected @@ -33,6 +33,7 @@ argHasPostUpdate | test.cpp:67:29:67:35 | source1 | ArgumentNode is missing PostUpdateNode. | | test.cpp:813:19:813:35 | * ... | ArgumentNode is missing PostUpdateNode. | | test.cpp:848:23:848:25 | rpx | ArgumentNode is missing PostUpdateNode. | +| test.cpp:1057:19:1057:21 | * ... | ArgumentNode is missing PostUpdateNode. | postWithInFlow | BarrierGuard.cpp:49:6:49:6 | x [post update] | PostUpdateNode should not be the target of local flow. | | BarrierGuard.cpp:60:7:60:7 | x [post update] | PostUpdateNode should not be the target of local flow. | @@ -168,6 +169,13 @@ postWithInFlow | test.cpp:1045:9:1045:11 | ref arg buf | PostUpdateNode should not be the target of local flow. | | test.cpp:1051:5:1051:11 | content [post update] | PostUpdateNode should not be the target of local flow. | | test.cpp:1052:9:1052:9 | a [inner post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1056:5:1056:7 | * ... [post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1056:6:1056:7 | pp [inner post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1062:53:1062:53 | p [inner post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1072:3:1072:4 | * ... [post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1072:4:1072:4 | p [inner post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1073:3:1073:4 | * ... [post update] | PostUpdateNode should not be the target of local flow. | +| test.cpp:1073:4:1073:4 | p [inner post update] | PostUpdateNode should not be the target of local flow. | viableImplInCallContextTooLarge uniqueParameterNodeAtPosition uniqueParameterNodePosition diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp index b2bff6327c5..af9e18034ed 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp @@ -1050,4 +1050,26 @@ void flow_out_of_address_with_local_flow() { MyStruct a; a.content = nullptr; sink(&a); // $ SPURIOUS: ast -} \ No newline at end of file +} + +static void static_func_that_reassigns_pointer_before_sink(char** pp) { // $ ast-def=pp ir-def=*pp ir-def=**pp + *pp = ""; + indirect_sink(*pp); // clean +} + +void test_static_func_that_reassigns_pointer_before_sink() { + char* p = (char*)indirect_source(); + static_func_that_reassigns_pointer_before_sink(&p); +} + +void single_object_in_both_cases(bool b, int x, int y) { + int* p; + if(b) { + p = &x; + } else { + p = &y; + } + *p = source(); + *p = 0; + sink(*p); // clean +} diff --git a/cpp/ql/test/library-tests/dataflow/dataflow-tests/type-bugs.expected b/cpp/ql/test/library-tests/dataflow/dataflow-tests/type-bugs.expected index 6706d79e902..4d87c2da534 100644 --- a/cpp/ql/test/library-tests/dataflow/dataflow-tests/type-bugs.expected +++ b/cpp/ql/test/library-tests/dataflow/dataflow-tests/type-bugs.expected @@ -15,4 +15,5 @@ incorrectBaseType | test.cpp:848:23:848:25 | (reference dereference) | Expected 'Node.getType()' to be int, but it was int * | | test.cpp:854:10:854:36 | * ... | Expected 'Node.getType()' to be const int, but it was int | | test.cpp:867:10:867:30 | * ... | Expected 'Node.getType()' to be const int, but it was int | +| test.cpp:1062:52:1062:53 | *& ... | Expected 'Node.getType()' to be char, but it was char * | failures diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs index 3d6672560b6..62a053870e9 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs @@ -1,5 +1,4 @@ using System.Collections.Generic; -using System.IO; using System.Linq; using Semmle.Util.Logging; @@ -20,44 +19,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// assembly cache. /// /// Callback for progress. - public AssemblyCache(IEnumerable paths, IEnumerable frameworkPaths, ILogger logger) + public AssemblyCache(IEnumerable paths, IEnumerable frameworkPaths, ILogger logger) { this.logger = logger; foreach (var path in paths) { - if (File.Exists(path)) - { - dllsToIndex.Add(path); - continue; - } - - if (Directory.Exists(path)) - { - logger.LogInfo($"Finding reference DLLs in {path}..."); - AddReferenceDirectory(path); - } - else - { - logger.LogInfo("AssemblyCache: Path not found: " + path); - } + dllsToIndex.AddRange(path.GetDlls(logger)); } IndexReferences(frameworkPaths); } - /// - /// Finds all assemblies nested within a directory - /// and adds them to its index. - /// (Indexing is performed at a later stage by IndexReferences()). - /// - /// The directory to index. - private void AddReferenceDirectory(string dir) - { - foreach (var dll in new DirectoryInfo(dir).EnumerateFiles("*.dll", SearchOption.AllDirectories)) - { - dllsToIndex.Add(dll.FullName); - } - } - /// /// Indexes all DLLs we have located. /// Because this is a potentially time-consuming operation, it is put into a separate stage. diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs new file mode 100644 index 00000000000..98f43b9dcb7 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Semmle.Util.Logging; + +namespace Semmle.Extraction.CSharp.DependencyFetching +{ + /// + /// Used to represent a path to an assembly or a directory containing assemblies + /// and a selector function to determine which files to include, when indexing the assemblies. + /// + internal sealed class AssemblyLookupLocation(string path, Func includeFileName, bool indexSubdirectories = true) + { + public string Path => path; + + public AssemblyLookupLocation(string path) : this(path, _ => true) { } + + public static implicit operator AssemblyLookupLocation(string path) => new(path); + + /// + /// Finds all assemblies nested within the directory `path` + /// and adds them to the a list of assembly names to index. + /// Indexing is performed at a later stage. This only collects the names. + /// + /// List of dlls to index. + /// Logger. + private void AddReferenceDirectory(List dllsToIndex, ILogger logger) + { + try + { + var dlls = new DirectoryInfo(path).EnumerateFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = indexSubdirectories, MatchCasing = MatchCasing.CaseInsensitive, AttributesToSkip = FileAttributes.None }); + if (!dlls.Any()) + { + logger.LogWarning($"AssemblyLookupLocation: No DLLs found in the path '{path}'."); + return; + } + foreach (var dll in dlls) + { + if (includeFileName(dll.Name)) + { + dllsToIndex.Add(dll.FullName); + } + else + { + logger.LogInfo($"AssemblyLookupLocation: Skipping {dll.FullName}."); + } + } + } + catch (Exception e) + { + logger.LogError($"AssemblyLookupLocation: Error while searching for DLLs in '{path}': {e.Message}"); + } + } + + /// + /// Returns a list of paths to all assemblies in `path` that should be indexed. + /// + /// Logger + public List GetDlls(ILogger logger) + { + var dllsToIndex = new List(); + if (File.Exists(path)) + { + if (includeFileName(System.IO.Path.GetFileName(path))) + { + dllsToIndex.Add(path); + } + else + { + logger.LogInfo($"AssemblyLookupLocation: Skipping {path}."); + } + return dllsToIndex; + } + + if (Directory.Exists(path)) + { + logger.LogInfo($"AssemblyLookupLocation: Finding reference DLLs in {path}..."); + AddReferenceDirectory(dllsToIndex, logger); + } + else + { + logger.LogInfo("AssemblyLookupLocation: Path not found: " + path); + } + return dllsToIndex; + } + + public override bool Equals(object? obj) => + obj is AssemblyLookupLocation ap && path.Equals(ap.Path); + + public override int GetHashCode() => path.GetHashCode(); + + public override string ToString() => path; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs index 1f3f44425d5..cf9c7253b17 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs @@ -12,14 +12,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { public sealed partial class DependencyManager { - private void RestoreNugetPackages(List allNonBinaryFiles, IEnumerable allProjects, IEnumerable allSolutions, HashSet dllPaths) + private void RestoreNugetPackages(List allNonBinaryFiles, IEnumerable allProjects, IEnumerable allSolutions, HashSet dllLocations) { try { var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness); if (checkNugetFeedResponsiveness && !CheckFeeds(allNonBinaryFiles)) { - DownloadMissingPackages(allNonBinaryFiles, dllPaths, withNugetConfig: false); + DownloadMissingPackages(allNonBinaryFiles, dllLocations, withNugetConfig: false); return; } @@ -55,7 +55,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } nugetPackageDllPaths.ExceptWith(excludedPaths); - dllPaths.UnionWith(nugetPackageDllPaths); + dllLocations.UnionWith(nugetPackageDllPaths.Select(p => new AssemblyLookupLocation(p))); } catch (Exception exc) { @@ -72,10 +72,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .Paths .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d)) .ToList(); - dllPaths.UnionWith(paths); + dllLocations.UnionWith(paths.Select(p => new AssemblyLookupLocation(p))); LogAllUnusedPackages(dependencies); - DownloadMissingPackages(allNonBinaryFiles, dllPaths); + DownloadMissingPackages(allNonBinaryFiles, dllLocations); } /// @@ -148,7 +148,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString())); } - private void DownloadMissingPackages(List allFiles, ISet dllPaths, bool withNugetConfig = true) + private void DownloadMissingPackages(List allFiles, ISet dllLocations, bool withNugetConfig = true) { var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo); var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames(); @@ -206,7 +206,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString())); - dllPaths.Add(missingPackageDirectory.DirInfo.FullName); + dllLocations.Add(missingPackageDirectory.DirInfo.FullName); } private string[] GetAllNugetConfigs(List allFiles) => allFiles.SelectFileNamesByName("nuget.config").ToArray(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 8d63d7f42b2..7f8306c2d9d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -5,7 +5,6 @@ using System.IO; using System.Linq; using System.Security.Cryptography; using System.Text; -using System.Text.RegularExpressions; using System.Threading.Tasks; using Semmle.Util; @@ -46,7 +45,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// /// Performs C# dependency fetching. /// - /// Dependency fetching options + /// Path to directory containing source code. /// Logger for dependency fetching progress. public DependencyManager(string srcDir, ILogger logger) { @@ -91,9 +90,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching this.generatedSources = new(); var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj").ToList(); var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln").ToList(); - var dllPaths = allFiles.SelectFileNamesByExtension(".dll").ToHashSet(); + var dllLocations = allFiles.SelectFileNamesByExtension(".dll").Select(x => new AssemblyLookupLocation(x)).ToHashSet(); - logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllPaths.Count} DLLs."); + logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllLocations.Count} DLLs."); void startCallback(string s, bool silent) { @@ -122,12 +121,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching throw; } - RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllPaths); + RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllLocations); // Find DLLs in the .Net / Asp.Net Framework // This needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies. - var frameworkLocations = AddFrameworkDlls(dllPaths); + var frameworkLocations = AddFrameworkDlls(dllLocations); - assemblyCache = new AssemblyCache(dllPaths, frameworkLocations, logger); + assemblyCache = new AssemblyCache(dllLocations, frameworkLocations, logger); AnalyseSolutions(allSolutions); foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename)) @@ -192,7 +191,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching ]); } - private HashSet AddFrameworkDlls(HashSet dllPaths) + private HashSet AddFrameworkDlls(HashSet dllLocations) { var frameworkLocations = new HashSet(); @@ -200,9 +199,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var useSubfolders = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders); if (!string.IsNullOrWhiteSpace(frameworkReferences)) { - RemoveFrameworkNugetPackages(dllPaths); - RemoveNugetPackageReference(FrameworkPackageNames.AspNetCoreFramework, dllPaths); - RemoveNugetPackageReference(FrameworkPackageNames.WindowsDesktopFramework, dllPaths); + RemoveFrameworkNugetPackages(dllLocations); + RemoveNugetPackageReference(FrameworkPackageNames.AspNetCoreFramework, dllLocations); + RemoveNugetPackageReference(FrameworkPackageNames.WindowsDesktopFramework, dllLocations); var frameworkPaths = frameworkReferences.Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries); @@ -214,37 +213,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching continue; } - if (useSubfolders) - { - dllPaths.Add(path); - frameworkLocations.Add(path); - continue; - } - - try - { - var dlls = Directory.GetFiles(path, "*.dll", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive }); - if (dlls.Length == 0) - { - logger.LogError($"No DLLs found in specified framework reference path '{path}'."); - continue; - } - - dllPaths.UnionWith(dlls); - frameworkLocations.UnionWith(dlls); - } - catch (Exception e) - { - logger.LogError($"Error while searching for DLLs in '{path}': {e.Message}"); - } + dllLocations.Add(new AssemblyLookupLocation(path, _ => true, useSubfolders)); + frameworkLocations.Add(path); } return frameworkLocations; } - AddNetFrameworkDlls(dllPaths, frameworkLocations); - AddAspNetCoreFrameworkDlls(dllPaths, frameworkLocations); - AddMicrosoftWindowsDesktopDlls(dllPaths, frameworkLocations); + AddNetFrameworkDlls(dllLocations, frameworkLocations); + AddAspNetCoreFrameworkDlls(dllLocations, frameworkLocations); + AddMicrosoftWindowsDesktopDlls(dllLocations, frameworkLocations); return frameworkLocations; } @@ -284,7 +262,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet dllPaths, ISet frameworkLocations) + private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet dllLocations, ISet frameworkLocations) { var versionFolders = GetPackageVersionSubDirectories(frameworkPath); if (versionFolders.Length > 1) @@ -300,7 +278,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching selectedFrameworkFolder = frameworkPath; } - dllPaths.Add(selectedFrameworkFolder); + dllLocations.Add(selectedFrameworkFolder); frameworkLocations.Add(selectedFrameworkFolder); logger.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}."); } @@ -313,16 +291,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .ToArray(); } - private void RemoveFrameworkNugetPackages(ISet dllPaths, int fromIndex = 0) + private void RemoveFrameworkNugetPackages(ISet dllLocations, int fromIndex = 0) { var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks; for (var i = fromIndex; i < packagesInPrioOrder.Length; i++) { - RemoveNugetPackageReference(packagesInPrioOrder[i], dllPaths); + RemoveNugetPackageReference(packagesInPrioOrder[i], dllLocations); } } - private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLocations) + private void AddNetFrameworkDlls(ISet dllLocations, ISet frameworkLocations) { // Multiple dotnet framework packages could be present. // The order of the packages is important, we're adding the first one that is present in the nuget cache. @@ -342,8 +320,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching dotnetFrameworkVersionVariantCount += GetPackageVersionSubDirectories(fp.Path!).Length; } - SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllPaths, frameworkLocations); - RemoveFrameworkNugetPackages(dllPaths, frameworkPath.Index + 1); + SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllLocations, frameworkLocations); + RemoveFrameworkNugetPackages(dllLocations, frameworkPath.Index + 1); return; } @@ -368,14 +346,21 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - runtimeLocation ??= Runtime.ExecutingRuntime; + if (runtimeLocation is null) + { + runtimeLocation ??= Runtime.ExecutingRuntime; + dllLocations.Add(new AssemblyLookupLocation(runtimeLocation, name => !name.StartsWith("Semmle."))); + } + else + { + dllLocations.Add(runtimeLocation); + } logger.LogInfo($".NET runtime location selected: {runtimeLocation}"); - dllPaths.Add(runtimeLocation); frameworkLocations.Add(runtimeLocation); } - private void RemoveNugetPackageReference(string packagePrefix, ISet dllPaths) + private void RemoveNugetPackageReference(string packagePrefix, ISet dllLocations) { var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant(); if (packageFolder == null) @@ -384,10 +369,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } var packagePathPrefix = Path.Combine(packageFolder, packagePrefix.ToLowerInvariant()); - var toRemove = dllPaths.Where(s => s.StartsWith(packagePathPrefix, StringComparison.InvariantCultureIgnoreCase)); + var toRemove = dllLocations.Where(s => s.Path.StartsWith(packagePathPrefix, StringComparison.InvariantCultureIgnoreCase)); foreach (var path in toRemove) { - dllPaths.Remove(path); + dllLocations.Remove(path); logger.LogInfo($"Removed reference {path}"); } } @@ -397,7 +382,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return fileContent.IsNewProjectStructureUsed && fileContent.UseAspNetCoreDlls; } - private void AddAspNetCoreFrameworkDlls(ISet dllPaths, ISet frameworkLocations) + private void AddAspNetCoreFrameworkDlls(ISet dllLocations, ISet frameworkLocations) { if (!IsAspNetCoreDetected()) { @@ -407,23 +392,23 @@ namespace Semmle.Extraction.CSharp.DependencyFetching // First try to find ASP.NET Core assemblies in the NuGet packages if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework, packageDirectory) is string aspNetCorePackage) { - SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllPaths, frameworkLocations); + SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllLocations, frameworkLocations); return; } if (Runtime.AspNetCoreRuntime is string aspNetCoreRuntime) { logger.LogInfo($"ASP.NET runtime location selected: {aspNetCoreRuntime}"); - dllPaths.Add(aspNetCoreRuntime); + dllLocations.Add(aspNetCoreRuntime); frameworkLocations.Add(aspNetCoreRuntime); } } - private void AddMicrosoftWindowsDesktopDlls(ISet dllPaths, ISet frameworkLocations) + private void AddMicrosoftWindowsDesktopDlls(ISet dllLocations, ISet frameworkLocations) { if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework, packageDirectory) is string windowsDesktopApp) { - SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllPaths, frameworkLocations); + SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllLocations, frameworkLocations); } } diff --git a/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql b/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql index d59c60ec802..de95e0fcbe7 100644 --- a/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql +++ b/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/Types.ql @@ -2,4 +2,4 @@ import csharp from Class c where c.fromSource() -select c, c.getBaseClass().getFullyQualifiedName() +select c, c.getBaseClass().getFullyQualifiedNameDebug() diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected new file mode 100644 index 00000000000..772f640789e --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.expected @@ -0,0 +1,201 @@ +| [...]/Humanizer.dll | +| [...]/Microsoft.Bcl.AsyncInterfaces.dll | +| [...]/Microsoft.Build.Framework.dll | +| [...]/Microsoft.Build.dll | +| [...]/Microsoft.CSharp.dll | +| [...]/Microsoft.CodeAnalysis.CSharp.Workspaces.dll | +| [...]/Microsoft.CodeAnalysis.CSharp.dll | +| [...]/Microsoft.CodeAnalysis.VisualBasic.Workspaces.dll | +| [...]/Microsoft.CodeAnalysis.VisualBasic.dll | +| [...]/Microsoft.CodeAnalysis.Workspaces.dll | +| [...]/Microsoft.CodeAnalysis.dll | +| [...]/Microsoft.NET.StringTools.dll | +| [...]/Microsoft.VisualBasic.Core.dll | +| [...]/Microsoft.VisualBasic.dll | +| [...]/Microsoft.Win32.Primitives.dll | +| [...]/Microsoft.Win32.Registry.dll | +| [...]/Microsoft.Win32.SystemEvents.dll | +| [...]/Mono.Posix.NETStandard.dll | +| [...]/Newtonsoft.Json.dll | +| [...]/System.AppContext.dll | +| [...]/System.Buffers.dll | +| [...]/System.Collections.Concurrent.dll | +| [...]/System.Collections.Immutable.dll | +| [...]/System.Collections.NonGeneric.dll | +| [...]/System.Collections.Specialized.dll | +| [...]/System.Collections.dll | +| [...]/System.ComponentModel.Annotations.dll | +| [...]/System.ComponentModel.DataAnnotations.dll | +| [...]/System.ComponentModel.EventBasedAsync.dll | +| [...]/System.ComponentModel.Primitives.dll | +| [...]/System.ComponentModel.TypeConverter.dll | +| [...]/System.ComponentModel.dll | +| [...]/System.Composition.AttributedModel.dll | +| [...]/System.Composition.Convention.dll | +| [...]/System.Composition.Hosting.dll | +| [...]/System.Composition.Runtime.dll | +| [...]/System.Composition.TypedParts.dll | +| [...]/System.Configuration.ConfigurationManager.dll | +| [...]/System.Configuration.dll | +| [...]/System.Console.dll | +| [...]/System.Core.dll | +| [...]/System.Data.Common.dll | +| [...]/System.Data.DataSetExtensions.dll | +| [...]/System.Data.dll | +| [...]/System.Diagnostics.Contracts.dll | +| [...]/System.Diagnostics.Debug.dll | +| [...]/System.Diagnostics.DiagnosticSource.dll | +| [...]/System.Diagnostics.EventLog.dll | +| [...]/System.Diagnostics.FileVersionInfo.dll | +| [...]/System.Diagnostics.Process.dll | +| [...]/System.Diagnostics.StackTrace.dll | +| [...]/System.Diagnostics.TextWriterTraceListener.dll | +| [...]/System.Diagnostics.Tools.dll | +| [...]/System.Diagnostics.TraceSource.dll | +| [...]/System.Diagnostics.Tracing.dll | +| [...]/System.Drawing.Common.dll | +| [...]/System.Drawing.Primitives.dll | +| [...]/System.Drawing.dll | +| [...]/System.Dynamic.Runtime.dll | +| [...]/System.Formats.Asn1.dll | +| [...]/System.Formats.Tar.dll | +| [...]/System.Globalization.Calendars.dll | +| [...]/System.Globalization.Extensions.dll | +| [...]/System.Globalization.dll | +| [...]/System.IO.Compression.Brotli.dll | +| [...]/System.IO.Compression.FileSystem.dll | +| [...]/System.IO.Compression.ZipFile.dll | +| [...]/System.IO.Compression.dll | +| [...]/System.IO.FileSystem.AccessControl.dll | +| [...]/System.IO.FileSystem.DriveInfo.dll | +| [...]/System.IO.FileSystem.Primitives.dll | +| [...]/System.IO.FileSystem.Watcher.dll | +| [...]/System.IO.FileSystem.dll | +| [...]/System.IO.IsolatedStorage.dll | +| [...]/System.IO.MemoryMappedFiles.dll | +| [...]/System.IO.Pipelines.dll | +| [...]/System.IO.Pipes.AccessControl.dll | +| [...]/System.IO.Pipes.dll | +| [...]/System.IO.UnmanagedMemoryStream.dll | +| [...]/System.IO.dll | +| [...]/System.Linq.Expressions.dll | +| [...]/System.Linq.Parallel.dll | +| [...]/System.Linq.Queryable.dll | +| [...]/System.Linq.dll | +| [...]/System.Memory.dll | +| [...]/System.Net.Http.Json.dll | +| [...]/System.Net.Http.dll | +| [...]/System.Net.HttpListener.dll | +| [...]/System.Net.Mail.dll | +| [...]/System.Net.NameResolution.dll | +| [...]/System.Net.NetworkInformation.dll | +| [...]/System.Net.Ping.dll | +| [...]/System.Net.Primitives.dll | +| [...]/System.Net.Quic.dll | +| [...]/System.Net.Requests.dll | +| [...]/System.Net.Security.dll | +| [...]/System.Net.ServicePoint.dll | +| [...]/System.Net.Sockets.dll | +| [...]/System.Net.WebClient.dll | +| [...]/System.Net.WebHeaderCollection.dll | +| [...]/System.Net.WebProxy.dll | +| [...]/System.Net.WebSockets.Client.dll | +| [...]/System.Net.WebSockets.dll | +| [...]/System.Net.dll | +| [...]/System.Numerics.Vectors.dll | +| [...]/System.Numerics.dll | +| [...]/System.ObjectModel.dll | +| [...]/System.Private.CoreLib.dll | +| [...]/System.Private.DataContractSerialization.dll | +| [...]/System.Private.Uri.dll | +| [...]/System.Private.Xml.Linq.dll | +| [...]/System.Private.Xml.dll | +| [...]/System.Reflection.DispatchProxy.dll | +| [...]/System.Reflection.Emit.ILGeneration.dll | +| [...]/System.Reflection.Emit.Lightweight.dll | +| [...]/System.Reflection.Emit.dll | +| [...]/System.Reflection.Extensions.dll | +| [...]/System.Reflection.Metadata.dll | +| [...]/System.Reflection.MetadataLoadContext.dll | +| [...]/System.Reflection.Primitives.dll | +| [...]/System.Reflection.TypeExtensions.dll | +| [...]/System.Reflection.dll | +| [...]/System.Resources.Reader.dll | +| [...]/System.Resources.ResourceManager.dll | +| [...]/System.Resources.Writer.dll | +| [...]/System.Runtime.CompilerServices.Unsafe.dll | +| [...]/System.Runtime.CompilerServices.VisualC.dll | +| [...]/System.Runtime.Extensions.dll | +| [...]/System.Runtime.Handles.dll | +| [...]/System.Runtime.InteropServices.JavaScript.dll | +| [...]/System.Runtime.InteropServices.RuntimeInformation.dll | +| [...]/System.Runtime.InteropServices.dll | +| [...]/System.Runtime.Intrinsics.dll | +| [...]/System.Runtime.Loader.dll | +| [...]/System.Runtime.Numerics.dll | +| [...]/System.Runtime.Serialization.Formatters.dll | +| [...]/System.Runtime.Serialization.Json.dll | +| [...]/System.Runtime.Serialization.Primitives.dll | +| [...]/System.Runtime.Serialization.Xml.dll | +| [...]/System.Runtime.Serialization.dll | +| [...]/System.Runtime.dll | +| [...]/System.Security.AccessControl.dll | +| [...]/System.Security.Claims.dll | +| [...]/System.Security.Cryptography.Algorithms.dll | +| [...]/System.Security.Cryptography.Cng.dll | +| [...]/System.Security.Cryptography.Csp.dll | +| [...]/System.Security.Cryptography.Encoding.dll | +| [...]/System.Security.Cryptography.OpenSsl.dll | +| [...]/System.Security.Cryptography.Primitives.dll | +| [...]/System.Security.Cryptography.ProtectedData.dll | +| [...]/System.Security.Cryptography.X509Certificates.dll | +| [...]/System.Security.Cryptography.dll | +| [...]/System.Security.Permissions.dll | +| [...]/System.Security.Principal.Windows.dll | +| [...]/System.Security.Principal.dll | +| [...]/System.Security.SecureString.dll | +| [...]/System.Security.dll | +| [...]/System.ServiceModel.Web.dll | +| [...]/System.ServiceProcess.dll | +| [...]/System.Text.Encoding.CodePages.dll | +| [...]/System.Text.Encoding.Extensions.dll | +| [...]/System.Text.Encoding.dll | +| [...]/System.Text.Encodings.Web.dll | +| [...]/System.Text.Json.dll | +| [...]/System.Text.RegularExpressions.dll | +| [...]/System.Threading.Channels.dll | +| [...]/System.Threading.Overlapped.dll | +| [...]/System.Threading.Tasks.Dataflow.dll | +| [...]/System.Threading.Tasks.Extensions.dll | +| [...]/System.Threading.Tasks.Parallel.dll | +| [...]/System.Threading.Tasks.dll | +| [...]/System.Threading.Thread.dll | +| [...]/System.Threading.ThreadPool.dll | +| [...]/System.Threading.Timer.dll | +| [...]/System.Threading.dll | +| [...]/System.Transactions.Local.dll | +| [...]/System.Transactions.dll | +| [...]/System.ValueTuple.dll | +| [...]/System.Web.HttpUtility.dll | +| [...]/System.Web.dll | +| [...]/System.Windows.Extensions.dll | +| [...]/System.Windows.dll | +| [...]/System.Xml.Linq.dll | +| [...]/System.Xml.ReaderWriter.dll | +| [...]/System.Xml.Serialization.dll | +| [...]/System.Xml.XDocument.dll | +| [...]/System.Xml.XPath.XDocument.dll | +| [...]/System.Xml.XPath.dll | +| [...]/System.Xml.XmlDocument.dll | +| [...]/System.Xml.XmlSerializer.dll | +| [...]/System.Xml.dll | +| [...]/System.dll | +| [...]/WindowsBase.dll | +| [...]/mscorlib.dll | +| [...]/netstandard.dll | +| [...]/zh-Hant/Microsoft.CodeAnalysis.CSharp.Workspaces.resources.dll | +| [...]/zh-Hant/Microsoft.CodeAnalysis.CSharp.resources.dll | +| [...]/zh-Hant/Microsoft.CodeAnalysis.VisualBasic.Workspaces.resources.dll | +| [...]/zh-Hant/Microsoft.CodeAnalysis.VisualBasic.resources.dll | +| [...]/zh-Hant/Microsoft.CodeAnalysis.Workspaces.resources.dll | +| [...]/zh-Hant/Microsoft.CodeAnalysis.resources.dll | diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.ql b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.ql new file mode 100644 index 00000000000..3bb648c579a --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Assemblies.ql @@ -0,0 +1,21 @@ +import csharp + +private string getPath(Assembly a) { + not a.getCompilation().getOutputAssembly() = a and + exists(string s | s = a.getFile().getAbsolutePath() | + result = + "[...]" + + s.substring(s.indexOf("test-db/working/") + "test-db/working/".length() + 16 + + "/packages".length(), s.length()) + or + exists(string sub | sub = "csharp/tools/" + ["osx64", "linux64"] | + result = "[...]" + s.substring(s.indexOf(sub) + sub.length(), s.length()) + ) + or + result = s and + not exists(s.indexOf(["test-db/working/", "csharp/tools/"])) + ) +} + +from Assembly a +select getPath(a) diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Program.cs b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Program.cs new file mode 100644 index 00000000000..39a9e95bb6e --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/Program.cs @@ -0,0 +1,6 @@ +class Program +{ + static void Main(string[] args) + { + } +} \ No newline at end of file diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/test.py b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/test.py new file mode 100644 index 00000000000..a17966e148a --- /dev/null +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_executing_runtime/test.py @@ -0,0 +1,3 @@ +from create_database_utils import * + +run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"]) diff --git a/csharp/ql/lib/semmle/code/csharp/Element.qll b/csharp/ql/lib/semmle/code/csharp/Element.qll index 2c69912c993..a48241a1408 100644 --- a/csharp/ql/lib/semmle/code/csharp/Element.qll +++ b/csharp/ql/lib/semmle/code/csharp/Element.qll @@ -102,25 +102,8 @@ class NamedElement extends Element, @named_element { final predicate hasName(string name) { name = this.getName() } /** - * Gets the fully qualified name of this element, for example the - * fully qualified name of `M` on line 3 is `N.C.M` in + * DEPRECATED: Use `hasFullyQualifiedName` instead. * - * ```csharp - * namespace N { - * class C { - * void M(int i, string s) { } - * } - * } - * ``` - */ - cached - deprecated final string getQualifiedName() { - exists(string qualifier, string name | this.hasQualifiedName(qualifier, name) | - if qualifier = "" then result = name else result = qualifier + "." + name - ) - } - - /** * Gets the fully qualified name of this element, for example the * fully qualified name of `M` on line 3 is `N.C.M` in * @@ -135,21 +118,39 @@ class NamedElement extends Element, @named_element { * Unbound generic types, such as `IList`, are represented as * ``System.Collections.Generic.IList`1``. */ - cached - final string getFullyQualifiedName() { + deprecated final string getFullyQualifiedName() { exists(string qualifier, string name | this.hasFullyQualifiedName(qualifier, name) | if qualifier = "" then result = name else result = qualifier + "." + name ) } /** - * DEPRECATED: Use `hasFullyQualifiedName` instead. + * INTERNAL: Do not use. * - * Holds if this element has the qualified name `qualifier`.`name`. + * This is intended for DEBUG ONLY. + * Constructing the fully qualified name for all elements in a large codebase + * puts severe stress on the string pool. + * + * Gets the fully qualified name of this element, for example the + * fully qualified name of `M` on line 3 is `N.C.M` in + * + * ```csharp + * namespace N { + * class C { + * void M(int i, string s) { } + * } + * } + * ``` + * + * Unbound generic types, such as `IList`, are represented as + * ``System.Collections.Generic.IList`1``. */ - cached - deprecated predicate hasQualifiedName(string qualifier, string name) { - qualifier = "" and name = this.getName() + bindingset[this] + pragma[inline_late] + final string getFullyQualifiedNameDebug() { + exists(string qualifier, string name | this.hasFullyQualifiedName(qualifier, name) | + if qualifier = "" then result = name else result = qualifier + "." + name + ) } /** Holds if this element has the fully qualified name `qualifier`.`name`. */ diff --git a/csharp/ql/lib/semmle/code/csharp/Member.qll b/csharp/ql/lib/semmle/code/csharp/Member.qll index 1be091170e3..3427d4ea089 100644 --- a/csharp/ql/lib/semmle/code/csharp/Member.qll +++ b/csharp/ql/lib/semmle/code/csharp/Member.qll @@ -71,17 +71,11 @@ class Declaration extends NamedElement, @declaration { override string toString() { result = this.getName() } - deprecated override predicate hasQualifiedName(string qualifier, string name) { - QualifiedName::hasQualifiedName(this, qualifier, name) - } - override predicate hasFullyQualifiedName(string qualifier, string name) { QualifiedName::hasQualifiedName(this, qualifier, name) } /** - * DEPRECATED: Use `getFullyQualifiedNameWithTypes` instead. - * * Gets the fully qualified name of this declaration, including types, for example * the fully qualified name with types of `M` on line 3 is `N.C.M(int, string)` in * @@ -93,33 +87,13 @@ class Declaration extends NamedElement, @declaration { * } * ``` */ - deprecated string getQualifiedNameWithTypes() { - exists(string qual | - qual = this.getDeclaringType().getQualifiedName() and + deprecated string getFullyQualifiedNameWithTypes() { + exists(string fullqual, string qual, string name | + this.getDeclaringType().hasFullyQualifiedName(qual, name) and + fullqual = getQualifiedName(qual, name) and if this instanceof NestedType - then result = qual + "+" + this.toStringWithTypes() - else result = qual + "." + this.toStringWithTypes() - ) - } - - /** - * Gets the fully qualified name of this declaration, including types, for example - * the fully qualified name with types of `M` on line 3 is `N.C.M(int, string)` in - * - * ```csharp - * namespace N { - * class C { - * void M(int i, string s) { } - * } - * } - * ``` - */ - string getFullyQualifiedNameWithTypes() { - exists(string qual | - qual = this.getDeclaringType().getFullyQualifiedName() and - if this instanceof NestedType - then result = qual + "+" + this.toStringWithTypes() - else result = qual + "." + this.toStringWithTypes() + then result = fullqual + "+" + this.toStringWithTypes() + else result = fullqual + "." + this.toStringWithTypes() ) } @@ -263,17 +237,6 @@ class Member extends Modifiable, @member { /** Gets an access to this member. */ MemberAccess getAnAccess() { result.getTarget() = this } - /** - * DEPRECATED: Use `hasFullyQualifiedName` instead. - * - * Holds if this member has name `name` and is defined in type `type` - * with namespace `namespace`. - */ - cached - deprecated final predicate hasQualifiedName(string namespace, string type, string name) { - QualifiedName::hasQualifiedName(this, namespace, type, name) - } - /** * Holds if this member has name `name` and is defined in type `type` * with namespace `namespace`. diff --git a/csharp/ql/lib/semmle/code/csharp/Namespace.qll b/csharp/ql/lib/semmle/code/csharp/Namespace.qll index 002ce444b3f..383fe8ab2a8 100644 --- a/csharp/ql/lib/semmle/code/csharp/Namespace.qll +++ b/csharp/ql/lib/semmle/code/csharp/Namespace.qll @@ -38,16 +38,6 @@ class Namespace extends TypeContainer, Declaration, @namespace { parent_namespace(result, this) } - /** - * Holds if this namespace has the qualified name `qualifier`.`name`. - * - * For example if the qualified name is `System.Collections.Generic`, then - * `qualifier`=`System.Collections` and `name`=`Generic`. - */ - deprecated override predicate hasQualifiedName(string qualifier, string name) { - namespaceHasQualifiedName(this, qualifier, name) - } - /** * Holds if this namespace has the qualified name `qualifier`.`name`. * diff --git a/csharp/ql/lib/semmle/code/csharp/commons/QualifiedName.qll b/csharp/ql/lib/semmle/code/csharp/commons/QualifiedName.qll index eba0fb10c7c..417dc137027 100644 --- a/csharp/ql/lib/semmle/code/csharp/commons/QualifiedName.qll +++ b/csharp/ql/lib/semmle/code/csharp/commons/QualifiedName.qll @@ -219,3 +219,28 @@ predicate splitQualifiedName(string qualifiedName, string qualifier, string name name = qualifiedName ) } + +/** + * INTERNAL: Do not use. + * + * Gets the fully qualified name of this declaration, including types, for example + * the fully qualified name with types of `M` on line 3 is `N.C.M(int, string)` in + * + * ```csharp + * namespace N { + * class C { + * void M(int i, string s) { } + * } + * } + * ``` + */ +bindingset[d] +string getFullyQualifiedNameWithTypes(Declaration d) { + exists(string fullqual, string qual, string name | + d.getDeclaringType().hasFullyQualifiedName(qual, name) and + fullqual = getQualifiedName(qual, name) and + if d instanceof NestedType + then result = fullqual + "+" + d.toStringWithTypes() + else result = fullqual + "." + d.toStringWithTypes() + ) +} diff --git a/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll b/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll index 673bd1a5638..0d79eafdf5c 100644 --- a/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll +++ b/csharp/ql/lib/semmle/code/csharp/dataflow/SSA.qll @@ -120,7 +120,7 @@ module Ssa { result = prefix + "." + this.getAssignable() | if f.(Modifiable).isStatic() - then prefix = f.getDeclaringType().getFullyQualifiedName() + then prefix = f.getDeclaringType().getName() else prefix = "this" ) } diff --git a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll index 96f45e77655..82c2a7dcd3d 100644 --- a/csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll +++ b/csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll @@ -3,6 +3,7 @@ */ private import csharp +private import semmle.code.csharp.commons.QualifiedName private import semmle.code.csharp.frameworks.system.linq.Expressions private import codeql.dataflow.internal.FlowSummaryImpl private import codeql.dataflow.internal.AccessPathSyntax as AccessPath @@ -42,10 +43,18 @@ module Input implements InputSig string encodeContent(ContentSet c, string arg) { c = TElementContent() and result = "Element" and arg = "" or - exists(Field f | c = TFieldContent(f) and result = "Field" and arg = f.getFullyQualifiedName()) + exists(Field f, string qualifier, string name | + c = TFieldContent(f) and + f.hasFullyQualifiedName(qualifier, name) and + arg = getQualifiedName(qualifier, name) and + result = "Field" + ) or - exists(Property p | - c = TPropertyContent(p) and result = "Property" and arg = p.getFullyQualifiedName() + exists(Property p, string qualifier, string name | + c = TPropertyContent(p) and + p.hasFullyQualifiedName(qualifier, name) and + arg = getQualifiedName(qualifier, name) and + result = "Property" ) or exists(SyntheticField f | diff --git a/csharp/ql/lib/semmle/code/csharp/security/dataflow/ExternalAPIsQuery.qll b/csharp/ql/lib/semmle/code/csharp/security/dataflow/ExternalAPIsQuery.qll index 3075fe53a87..a0d0ada957a 100644 --- a/csharp/ql/lib/semmle/code/csharp/security/dataflow/ExternalAPIsQuery.qll +++ b/csharp/ql/lib/semmle/code/csharp/security/dataflow/ExternalAPIsQuery.qll @@ -139,13 +139,13 @@ class ExternalApiUsedWithUntrustedData extends TExternalApi { /** Gets a textual representation of this element. */ string toString() { - exists(Callable m, int index, string indexString | + exists(Callable m, int index, string indexString, string qualifier, string name | if index = -1 then indexString = "qualifier" else indexString = "param " + index | this = TExternalApiParameter(m, index) and + m.getDeclaringType().hasFullyQualifiedName(qualifier, name) and result = - m.getDeclaringType().getFullyQualifiedName() + "." + m.toStringWithTypes() + " [" + - indexString + "]" + getQualifiedName(qualifier, name) + "." + m.toStringWithTypes() + " [" + indexString + "]" ) } } diff --git a/csharp/ql/test/library-tests/attributes/AttributeElements.ql b/csharp/ql/test/library-tests/attributes/AttributeElements.ql index 17ce2ea3d93..679d7567ea5 100644 --- a/csharp/ql/test/library-tests/attributes/AttributeElements.ql +++ b/csharp/ql/test/library-tests/attributes/AttributeElements.ql @@ -1,9 +1,7 @@ import csharp -import semmle.code.csharp.commons.QualifiedName -from Attributable element, Attribute attribute, string qualifier, string name +from Attributable element, Attribute attribute where attribute = element.getAnAttribute() and - (attribute.fromSource() or element.(Assembly).getName() in ["attributes", "Assembly1"]) and - attribute.getType().hasFullyQualifiedName(qualifier, name) -select element, attribute, getQualifiedName(qualifier, name) + (attribute.fromSource() or element.(Assembly).getName() in ["attributes", "Assembly1"]) +select element, attribute, attribute.getType().getFullyQualifiedNameDebug() diff --git a/csharp/ql/test/library-tests/constructors/Destructors1.ql b/csharp/ql/test/library-tests/constructors/Destructors1.ql index 368980a290c..792d50da7bb 100644 --- a/csharp/ql/test/library-tests/constructors/Destructors1.ql +++ b/csharp/ql/test/library-tests/constructors/Destructors1.ql @@ -3,11 +3,10 @@ */ import csharp -import semmle.code.csharp.commons.QualifiedName from Destructor c, string qualifier, string name where c.getDeclaringType().hasFullyQualifiedName(qualifier, name) and qualifier = "Constructors" and name = "Class" -select c, c.getDeclaringType().getFullyQualifiedName() +select c, c.getDeclaringType().getFullyQualifiedNameDebug() diff --git a/csharp/ql/test/library-tests/csharp11/fileScoped.ql b/csharp/ql/test/library-tests/csharp11/fileScoped.ql index 3003fc801bf..33697255896 100644 --- a/csharp/ql/test/library-tests/csharp11/fileScoped.ql +++ b/csharp/ql/test/library-tests/csharp11/fileScoped.ql @@ -1,5 +1,4 @@ import csharp -private import semmle.code.csharp.commons.QualifiedName private predicate isInteresting(Type t) { ( @@ -20,10 +19,7 @@ query predicate typemodifiers(Type t, string modifier) { query predicate qualifiedtypes(Type t, string qualifiedName) { isInteresting(t) and - exists(string qualifier, string name | - t.hasFullyQualifiedName(qualifier, name) and - qualifiedName = getQualifiedName(qualifier, name) - ) + qualifiedName = t.getFullyQualifiedNameDebug() } query predicate filetypes(Type t) { diff --git a/csharp/ql/test/library-tests/csharp11/nativeInt.ql b/csharp/ql/test/library-tests/csharp11/nativeInt.ql index 80d7974de56..adbc062baf6 100644 --- a/csharp/ql/test/library-tests/csharp11/nativeInt.ql +++ b/csharp/ql/test/library-tests/csharp11/nativeInt.ql @@ -1,12 +1,10 @@ import csharp -import semmle.code.csharp.commons.QualifiedName -from LocalVariable v1, LocalVariable v2, Type t, string qualifier, string name +from LocalVariable v1, LocalVariable v2, Type t where v1.getFile().getStem() = "NativeInt" and v2.getFile().getStem() = "NativeInt" and t = v1.getType() and t = v2.getType() and - t.hasFullyQualifiedName(qualifier, name) and v1 != v2 -select v1, v2, getQualifiedName(qualifier, name) +select v1, v2, t.getFullyQualifiedNameDebug() diff --git a/csharp/ql/test/library-tests/csharp9/covariantReturn.ql b/csharp/ql/test/library-tests/csharp9/covariantReturn.ql index 6227ed18d6d..b4bab047322 100644 --- a/csharp/ql/test/library-tests/csharp9/covariantReturn.ql +++ b/csharp/ql/test/library-tests/csharp9/covariantReturn.ql @@ -1,13 +1,8 @@ import csharp -import semmle.code.csharp.commons.QualifiedName -from - Method m, Method overrider, string mnamespace, string mtype, string mname, string onamespace, - string otype, string oname +from Method m, Method overrider where m.getAnOverrider() = overrider and - m.getFile().getStem() = "CovariantReturn" and - m.hasFullyQualifiedName(mnamespace, mtype, mname) and - overrider.hasFullyQualifiedName(onamespace, otype, oname) -select getQualifiedName(mnamespace, mtype, mname), m.getReturnType().toString(), - getQualifiedName(onamespace, otype, oname), overrider.getReturnType().toString() + m.getFile().getStem() = "CovariantReturn" +select m.getFullyQualifiedNameDebug(), m.getReturnType().toString(), + overrider.getFullyQualifiedNameDebug(), overrider.getReturnType().toString() diff --git a/csharp/ql/test/library-tests/csharp9/foreach.ql b/csharp/ql/test/library-tests/csharp9/foreach.ql index cf6fcf2514a..343ecc556ab 100644 --- a/csharp/ql/test/library-tests/csharp9/foreach.ql +++ b/csharp/ql/test/library-tests/csharp9/foreach.ql @@ -1,5 +1,4 @@ import csharp -import semmle.code.csharp.commons.QualifiedName private string getLocation(Member m) { if m.fromSource() then result = m.getALocation().(SourceLocation).toString() else result = "-" @@ -9,13 +8,9 @@ private string getIsAsync(ForeachStmt f) { if f.isAsync() then result = "async" else result = "sync" } -from - ForeachStmt f, string qualifier1, string type1, string qualifier2, string type2, - string qualifier3, string type3 -where - f.getGetEnumerator().getDeclaringType().hasFullyQualifiedName(qualifier1, type1) and - f.getCurrent().getDeclaringType().hasFullyQualifiedName(qualifier2, type2) and - f.getMoveNext().getDeclaringType().hasFullyQualifiedName(qualifier3, type3) -select f, f.getElementType().toString(), getIsAsync(f), getQualifiedName(qualifier1, type1), - getLocation(f.getGetEnumerator()), getQualifiedName(qualifier2, type2), - getLocation(f.getCurrent()), getQualifiedName(qualifier3, type3), getLocation(f.getMoveNext()) +from ForeachStmt f +select f, f.getElementType().toString(), getIsAsync(f), + f.getGetEnumerator().getDeclaringType().getFullyQualifiedNameDebug(), + getLocation(f.getGetEnumerator()), f.getCurrent().getDeclaringType().getFullyQualifiedNameDebug(), + getLocation(f.getCurrent()), f.getMoveNext().getDeclaringType().getFullyQualifiedNameDebug(), + getLocation(f.getMoveNext()) diff --git a/csharp/ql/test/library-tests/csharp9/record.ql b/csharp/ql/test/library-tests/csharp9/record.ql index a2a9a9c0786..58cf579bac6 100644 --- a/csharp/ql/test/library-tests/csharp9/record.ql +++ b/csharp/ql/test/library-tests/csharp9/record.ql @@ -7,18 +7,10 @@ query predicate records(RecordClass t, string i, RecordCloneMethod clone) { t.fromSource() } -private string getMemberName(Member m) { - exists(string qualifier, string name | - m.getDeclaringType().hasFullyQualifiedName(qualifier, name) - | - result = getQualifiedName(qualifier, name) + "." + m.toStringWithTypes() - ) -} - query predicate members(RecordClass t, string ms, string l) { t.fromSource() and exists(Member m | t.hasMember(m) | - ms = getMemberName(m) and + ms = getFullyQualifiedNameWithTypes(m) and if m.fromSource() then l = m.getLocation().toString() else l = "no location" ) } diff --git a/csharp/ql/test/library-tests/csharp9/withExpr.ql b/csharp/ql/test/library-tests/csharp9/withExpr.ql index 6683d7c54f6..564cbe529aa 100644 --- a/csharp/ql/test/library-tests/csharp9/withExpr.ql +++ b/csharp/ql/test/library-tests/csharp9/withExpr.ql @@ -1,19 +1,11 @@ import csharp import semmle.code.csharp.commons.QualifiedName -private string getSignature(Method m) { - exists(string qualifier, string name | - m.getDeclaringType().hasFullyQualifiedName(qualifier, name) - | - result = getQualifiedName(qualifier, name) + "." + m.toStringWithTypes() - ) -} - query predicate withExpr(WithExpr with, string type, Expr expr, ObjectInitializer init, string clone) { type = with.getType().toStringWithTypes() and expr = with.getExpr() and init = with.getInitializer() and - clone = getSignature(with.getCloneMethod()) + clone = getFullyQualifiedNameWithTypes(with.getCloneMethod()) } query predicate withTarget(WithExpr with, RecordCloneMethod clone, Constructor ctor) { @@ -25,7 +17,7 @@ query predicate cloneOverrides(string b, string o) { exists(RecordCloneMethod base, RecordCloneMethod overrider | base.getDeclaringType().fromSource() and base.getAnOverrider() = overrider and - b = getSignature(base) and - o = getSignature(overrider) + b = getFullyQualifiedNameWithTypes(base) and + o = getFullyQualifiedNameWithTypes(overrider) ) } diff --git a/csharp/ql/test/library-tests/dispatch/GetADynamicTarget.ql b/csharp/ql/test/library-tests/dispatch/GetADynamicTarget.ql index cbde2d43ab8..b4c94a0b507 100644 --- a/csharp/ql/test/library-tests/dispatch/GetADynamicTarget.ql +++ b/csharp/ql/test/library-tests/dispatch/GetADynamicTarget.ql @@ -1,8 +1,9 @@ import csharp +import semmle.code.csharp.commons.QualifiedName import semmle.code.csharp.dispatch.Dispatch from DispatchCall call, Callable callable where callable = call.getADynamicTarget() and callable.fromSource() -select call, callable.getFullyQualifiedNameWithTypes() +select call, getFullyQualifiedNameWithTypes(callable) diff --git a/csharp/ql/test/library-tests/enums/Enums3.ql b/csharp/ql/test/library-tests/enums/Enums3.ql index 5cfbdc56193..b01ffb97e18 100644 --- a/csharp/ql/test/library-tests/enums/Enums3.ql +++ b/csharp/ql/test/library-tests/enums/Enums3.ql @@ -3,13 +3,11 @@ */ import csharp -import semmle.code.csharp.commons.QualifiedName -from EnumConstant c, string namespace, string name +from EnumConstant c where c.getName() = "Green" and c.getDeclaringType().hasFullyQualifiedName("Enums", "LongColor") and c.getType() = c.getDeclaringType() and - c.getValue() = "1" and - c.getDeclaringType().getBaseClass().hasFullyQualifiedName(namespace, name) -select c, getQualifiedName(namespace, name) + c.getValue() = "1" +select c, c.getDeclaringType().getBaseClass().getFullyQualifiedNameDebug() diff --git a/csharp/ql/test/library-tests/frameworks/system/Dispose/Dispose.ql b/csharp/ql/test/library-tests/frameworks/system/Dispose/Dispose.ql index 943245fa0a2..5fa2f337fcf 100644 --- a/csharp/ql/test/library-tests/frameworks/system/Dispose/Dispose.ql +++ b/csharp/ql/test/library-tests/frameworks/system/Dispose/Dispose.ql @@ -1,4 +1,5 @@ import csharp +import semmle.code.csharp.commons.QualifiedName import semmle.code.csharp.frameworks.System from ValueOrRefType t, Method m, boolean b @@ -6,4 +7,4 @@ where t.fromSource() and m = getInvokedDisposeMethod(t) and if implementsDispose(t) then b = true else b = false -select t, m.getFullyQualifiedNameWithTypes(), b +select t, getFullyQualifiedNameWithTypes(m), b diff --git a/csharp/ql/test/library-tests/frameworks/system/Equals/Equals.ql b/csharp/ql/test/library-tests/frameworks/system/Equals/Equals.ql index 91c04791ef3..e6dc4ae7549 100644 --- a/csharp/ql/test/library-tests/frameworks/system/Equals/Equals.ql +++ b/csharp/ql/test/library-tests/frameworks/system/Equals/Equals.ql @@ -1,4 +1,5 @@ import csharp +import semmle.code.csharp.commons.QualifiedName import semmle.code.csharp.frameworks.System from ValueOrRefType t, Method m, boolean b @@ -6,4 +7,4 @@ where t.fromSource() and m = getInvokedEqualsMethod(t) and if implementsEquals(t) then b = true else b = false -select t, m.getFullyQualifiedNameWithTypes(), b +select t, getFullyQualifiedNameWithTypes(m), b diff --git a/csharp/ql/test/library-tests/generics/Generics.ql b/csharp/ql/test/library-tests/generics/Generics.ql index 2f3aff0fd58..e75f1fdc908 100644 --- a/csharp/ql/test/library-tests/generics/Generics.ql +++ b/csharp/ql/test/library-tests/generics/Generics.ql @@ -231,18 +231,18 @@ query predicate test27(ConstructedType ct, UnboundGenericType ugt, UnboundGeneri query predicate test28(UnboundGeneric ug, string s) { ug.fromSource() and - s = ug.getFullyQualifiedNameWithTypes() + s = getFullyQualifiedNameWithTypes(ug) } query predicate test29(ConstructedGeneric cg, string s) { cg.fromSource() and - s = cg.getFullyQualifiedNameWithTypes() + s = getFullyQualifiedNameWithTypes(cg) } query predicate test30(Declaration d, string s) { d.fromSource() and d instanceof @generic and - s = d.getFullyQualifiedNameWithTypes() and + s = getFullyQualifiedNameWithTypes(d) and d != d.getUnboundDeclaration() and not d instanceof Generic } @@ -263,21 +263,17 @@ query predicate test33(ConstructedMethod cm, string s1, string s2) { exists(string namespace, string type, string name | cm.hasFullyQualifiedName(namespace, type, name) and s1 = getQualifiedName(namespace, type, name) ) and - cm.getFullyQualifiedNameWithTypes() = s2 + getFullyQualifiedNameWithTypes(cm) = s2 } query predicate test34(UnboundGeneric ug, string s1, string s2) { ug.fromSource() and - exists(string qualifier, string name | - ug.hasFullyQualifiedName(qualifier, name) and s1 = getQualifiedName(qualifier, name) - ) and - ug.getFullyQualifiedNameWithTypes() = s2 + s1 = ug.getFullyQualifiedNameDebug() and + s2 = getFullyQualifiedNameWithTypes(ug) } query predicate test35(UnboundGenericMethod gm, string s1, string s2) { gm.fromSource() and - exists(string namespace, string type, string name | - gm.hasFullyQualifiedName(namespace, type, name) and s1 = getQualifiedName(namespace, type, name) - ) and - gm.getFullyQualifiedNameWithTypes() = s2 + s1 = gm.getFullyQualifiedNameDebug() and + s2 = getFullyQualifiedNameWithTypes(gm) } diff --git a/csharp/ql/test/library-tests/overrides/Overrides22.ql b/csharp/ql/test/library-tests/overrides/Overrides22.ql index d2c5a9e4336..d6300d49ecd 100644 --- a/csharp/ql/test/library-tests/overrides/Overrides22.ql +++ b/csharp/ql/test/library-tests/overrides/Overrides22.ql @@ -1,4 +1,5 @@ import csharp +import semmle.code.csharp.commons.QualifiedName from Overridable v1, Overridable v2, string kind where @@ -9,4 +10,4 @@ where ) and v1.fromSource() and v2.fromSource() -select v1.getFullyQualifiedNameWithTypes(), v2.getFullyQualifiedNameWithTypes(), kind +select getFullyQualifiedNameWithTypes(v1), getFullyQualifiedNameWithTypes(v2), kind diff --git a/csharp/ql/test/library-tests/unification/Unification.ql b/csharp/ql/test/library-tests/unification/Unification.ql index 10c5e520921..f8c6c15377d 100644 --- a/csharp/ql/test/library-tests/unification/Unification.ql +++ b/csharp/ql/test/library-tests/unification/Unification.ql @@ -1,3 +1,4 @@ +import semmle.code.csharp.commons.QualifiedName import semmle.code.csharp.Unification class InterestingType extends @type { @@ -7,9 +8,9 @@ class InterestingType extends @type { } string toString() { - result = this.(Type).getFullyQualifiedNameWithTypes() + result = getFullyQualifiedNameWithTypes(this.(Type)) or - not exists(this.(Type).getFullyQualifiedNameWithTypes()) and + not exists(getFullyQualifiedNameWithTypes(this.(Type))) and result = this.(Type).toStringWithTypes() } diff --git a/docs/codeql/codeql-language-guides/customizing-library-models-for-csharp.rst b/docs/codeql/codeql-language-guides/customizing-library-models-for-csharp.rst index 09dcf36fc07..39b5ee30ee4 100644 --- a/docs/codeql/codeql-language-guides/customizing-library-models-for-csharp.rst +++ b/docs/codeql/codeql-language-guides/customizing-library-models-for-csharp.rst @@ -282,7 +282,7 @@ These are the same for both of the rows above as we are adding two summaries for - The second value ``Enumerable`` is the class (type) name. - The third value ``False`` is a flag that indicates whether or not the summary also applies to all overrides of the method. - The fourth value ``Select`` is the method name, along with the type parameters for the method. The names of the generic type parameters provided in the model must match the names of the generic type parameters in the method signature in the source code. -- The fifth value ``(System.Collections.Generic.IEnumerable,System.Func)`` is the method input type signature. The generics in the signature must match the generics in the method signature in the source code. +- The fifth value ``(System.Collections.Generic.IEnumerable,System.Func)`` is the method input type signature. The generics in the signature must match the generics in the method signature in the source code. The sixth value should be left empty and is out of scope for this documentation. The remaining values are used to define the ``access path``, the ``kind``, and the ``provenance`` (origin) of the summary definition. @@ -309,7 +309,7 @@ That is, the first row specifies that values can flow from the elements of the q Example: Add a ``neutral`` method ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This example shows how we can model a method as being neutral with respect to flow. We will also cover how to model a property by modeling the getter of the ``Now`` property of the ``DateTime`` class as neutral. +This example shows how we can model a method as being neutral with respect to flow. We will also cover how to model a property by modeling the getter of the ``Now`` property of the ``DateTime`` class as neutral. A neutral model is used to define that there is no flow through a method. .. code-block:: csharp @@ -346,13 +346,4 @@ The first four values identify the callable (in this case the getter of the ``No Threat models ------------- -.. include:: ../reusables/beta-note-threat-models.rst - -A threat model is a named class of dataflow sources that can be enabled or disabled independently. Threat models allow you to control the set of dataflow sources that you want to consider unsafe. For example, one codebase may only consider remote HTTP requests to be tainted, whereas another may also consider data from local files to be unsafe. You can use threat models to ensure that the relevant taint sources are used in a CodeQL analysis. - -The ``kind`` property of ``sourceModel`` determines which threat model a source is associated with. There are two main categories: - -- ``remote`` which represents requests and responses from the network. -- ``local`` which represents data from local files (``file``), command-line arguments (``commandargs``), database reads (``database``), and environment variables(``environment``). - -When running a CodeQL analysis, the ``remote`` threat model is included by default. You can optionally include other threat models as appropriate when using the CodeQL CLI and in GitHub code scanning. For more information, see `Analyzing your code with CodeQL queries `__ and `Customizing your advanced setup for code scanning `__. +.. include:: ../reusables/threat-model-description.rst diff --git a/docs/codeql/codeql-language-guides/customizing-library-models-for-java-and-kotlin.rst b/docs/codeql/codeql-language-guides/customizing-library-models-for-java-and-kotlin.rst index e8b69e20d81..dd7f352f6d0 100644 --- a/docs/codeql/codeql-language-guides/customizing-library-models-for-java-and-kotlin.rst +++ b/docs/codeql/codeql-language-guides/customizing-library-models-for-java-and-kotlin.rst @@ -297,13 +297,4 @@ The first four values identify the callable (in this case a method) to be modele Threat models ------------- -.. include:: ../reusables/beta-note-threat-models.rst - -A threat model is a named class of dataflow sources that can be enabled or disabled independently. Threat models allow you to control the set of dataflow sources that you want to consider unsafe. For example, one codebase may only consider remote HTTP requests to be tainted, whereas another may also consider data from local files to be unsafe. You can use threat models to ensure that the relevant taint sources are used in a CodeQL analysis. - -The ``kind`` property of the ``sourceModel`` determines which threat model a source is associated with. There are two main categories: - -- ``remote`` which represents requests and responses from the network. -- ``local`` which represents data from local files (``file``), command-line arguments (``commandargs``), database reads (``database``), and environment variables(``environment``). - -When running a CodeQL analysis, the ``remote`` threat model is included by default. You can optionally include other threat models as appropriate when using the CodeQL CLI and in GitHub code scanning. For more information, see `Analyzing your code with CodeQL queries `__ and `Customizing your advanced setup for code scanning `__. +.. include:: ../reusables/threat-model-description.rst diff --git a/docs/codeql/reusables/threat-model-description.rst b/docs/codeql/reusables/threat-model-description.rst new file mode 100644 index 00000000000..53a872487bf --- /dev/null +++ b/docs/codeql/reusables/threat-model-description.rst @@ -0,0 +1,10 @@ +.. include:: ../reusables/beta-note-threat-models.rst + +A threat model is a named class of dataflow sources that can be enabled or disabled independently. Threat models allow you to control the set of dataflow sources that you want to consider unsafe. For example, one codebase may only consider remote HTTP requests to be tainted, whereas another may also consider data from local files to be unsafe. You can use threat models to ensure that the relevant taint sources are used in a CodeQL analysis. + +The ``kind`` property of the ``sourceModel`` determines which threat model a source is associated with. There are two main categories: + +- ``remote`` which represents requests and responses from the network. +- ``local`` which represents data from local files (``file``), command-line arguments (``commandargs``), database reads (``database``), and environment variables(``environment``). + +When running a CodeQL analysis, the ``remote`` threat model is included by default. You can optionally include other threat models as appropriate when using the CodeQL CLI and in GitHub code scanning. For more information, see `Analyzing your code with CodeQL queries `__ and `Customizing your advanced setup for code scanning `__. diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index 1a60c2bf982..d7759326d57 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -388,6 +388,8 @@ func extractUniverseScope() { } // extractObjects extracts all objects declared in the given scope +// For more information on objects, see: +// https://github.com/golang/example/blob/master/gotypes/README.md#objects func extractObjects(tw *trap.Writer, scope *types.Scope, scopeLabel trap.Label) { for _, name := range scope.Names() { obj := scope.Lookup(name) @@ -440,6 +442,8 @@ func extractMethod(tw *trap.Writer, meth *types.Func) trap.Label { } // extractObject extracts a single object and emits it to the objects table. +// For more information on objects, see: +// https://github.com/golang/example/blob/master/gotypes/README.md#objects func extractObject(tw *trap.Writer, obj types.Object, lbl trap.Label) { name := obj.Name() isBuiltin := obj.Parent() == types.Universe @@ -487,6 +491,8 @@ func extractObject(tw *trap.Writer, obj types.Object, lbl trap.Label) { } // extractObjectTypes extracts type and receiver information for all objects +// For more information on objects, see: +// https://github.com/golang/example/blob/master/gotypes/README.md#objects func extractObjectTypes(tw *trap.Writer) { // calling `extractType` on a named type will extract all methods defined // on it, which will add new objects. Therefore we need to do this first @@ -497,11 +503,13 @@ func extractObjectTypes(tw *trap.Writer) { } changed = tw.ForEachObject(emitObjectType) if changed { - log.Printf("Warning: more objects were labeled while emitted object types") + log.Printf("Warning: more objects were labeled while emitting object types") } } // extractObjectType extracts type and receiver information for a given object +// For more information on objects, see: +// https://github.com/golang/example/blob/master/gotypes/README.md#objects func extractObjectType(tw *trap.Writer, obj types.Object, lbl trap.Label) { if tp := obj.Type(); tp != nil { extractType(tw, tp) @@ -790,7 +798,7 @@ func extractLocalScope(tw *trap.Writer, scope *types.Scope, parentScopeLabel tra func extractFileNode(tw *trap.Writer, nd *ast.File) { lbl := tw.Labeler.FileLabel() - extractExpr(tw, nd.Name, lbl, 0) + extractExpr(tw, nd.Name, lbl, 0, false) for i, decl := range nd.Decls { extractDecl(tw, decl, lbl, i) @@ -847,7 +855,7 @@ func emitScopeNodeInfo(tw *trap.Writer, nd ast.Node, lbl trap.Label) { } // extractExpr extracts AST information for the given expression and all its subexpressions -func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { +func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int, skipExtractingValue bool) { if expr == nil { return } @@ -900,7 +908,7 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { return } kind = dbscheme.EllipsisExpr.Index() - extractExpr(tw, expr.Elt, lbl, 0) + extractExpr(tw, expr.Elt, lbl, 0, false) case *ast.BasicLit: if expr == nil { return @@ -932,28 +940,28 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { return } kind = dbscheme.FuncLitExpr.Index() - extractExpr(tw, expr.Type, lbl, 0) + extractExpr(tw, expr.Type, lbl, 0, false) extractStmt(tw, expr.Body, lbl, 1) case *ast.CompositeLit: if expr == nil { return } kind = dbscheme.CompositeLitExpr.Index() - extractExpr(tw, expr.Type, lbl, 0) + extractExpr(tw, expr.Type, lbl, 0, false) extractExprs(tw, expr.Elts, lbl, 1, 1) case *ast.ParenExpr: if expr == nil { return } kind = dbscheme.ParenExpr.Index() - extractExpr(tw, expr.X, lbl, 0) + extractExpr(tw, expr.X, lbl, 0, false) case *ast.SelectorExpr: if expr == nil { return } kind = dbscheme.SelectorExpr.Index() - extractExpr(tw, expr.X, lbl, 0) - extractExpr(tw, expr.Sel, lbl, 1) + extractExpr(tw, expr.X, lbl, 0, false) + extractExpr(tw, expr.Sel, lbl, 1, false) case *ast.IndexExpr: if expr == nil { return @@ -974,8 +982,8 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { kind = dbscheme.IndexExpr.Index() } } - extractExpr(tw, expr.X, lbl, 0) - extractExpr(tw, expr.Index, lbl, 1) + extractExpr(tw, expr.X, lbl, 0, false) + extractExpr(tw, expr.Index, lbl, 1, false) case *ast.IndexListExpr: if expr == nil { return @@ -993,33 +1001,33 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { kind = dbscheme.GenericTypeInstantiationExpr.Index() } } - extractExpr(tw, expr.X, lbl, 0) + extractExpr(tw, expr.X, lbl, 0, false) extractExprs(tw, expr.Indices, lbl, 1, 1) case *ast.SliceExpr: if expr == nil { return } kind = dbscheme.SliceExpr.Index() - extractExpr(tw, expr.X, lbl, 0) - extractExpr(tw, expr.Low, lbl, 1) - extractExpr(tw, expr.High, lbl, 2) - extractExpr(tw, expr.Max, lbl, 3) + extractExpr(tw, expr.X, lbl, 0, false) + extractExpr(tw, expr.Low, lbl, 1, false) + extractExpr(tw, expr.High, lbl, 2, false) + extractExpr(tw, expr.Max, lbl, 3, false) case *ast.TypeAssertExpr: if expr == nil { return } kind = dbscheme.TypeAssertExpr.Index() - extractExpr(tw, expr.X, lbl, 0) + extractExpr(tw, expr.X, lbl, 0, false) // expr.Type can be `nil` if this is the `x.(type)` in a type switch. if expr.Type != nil { - extractExpr(tw, expr.Type, lbl, 1) + extractExpr(tw, expr.Type, lbl, 1, false) } case *ast.CallExpr: if expr == nil { return } kind = dbscheme.CallOrConversionExpr.Index() - extractExpr(tw, expr.Fun, lbl, 0) + extractExpr(tw, expr.Fun, lbl, 0, false) extractExprs(tw, expr.Args, lbl, 1, 1) if expr.Ellipsis.IsValid() { dbscheme.HasEllipsisTable.Emit(tw, lbl) @@ -1029,14 +1037,14 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { return } kind = dbscheme.StarExpr.Index() - extractExpr(tw, expr.X, lbl, 0) + extractExpr(tw, expr.X, lbl, 0, false) case *ast.KeyValueExpr: if expr == nil { return } kind = dbscheme.KeyValueExpr.Index() - extractExpr(tw, expr.Key, lbl, 0) - extractExpr(tw, expr.Value, lbl, 1) + extractExpr(tw, expr.Key, lbl, 0, false) + extractExpr(tw, expr.Value, lbl, 1, false) case *ast.UnaryExpr: if expr == nil { return @@ -1050,7 +1058,7 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { } kind = tp.Index() } - extractExpr(tw, expr.X, lbl, 0) + extractExpr(tw, expr.X, lbl, 0, false) case *ast.BinaryExpr: if expr == nil { return @@ -1065,16 +1073,17 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { log.Fatalf("unsupported binary operator %s", expr.Op) } kind = tp.Index() - extractExpr(tw, expr.X, lbl, 0) - extractExpr(tw, expr.Y, lbl, 1) + skipLeft := skipExtractingValueForLeftOperand(tw, expr) + extractExpr(tw, expr.X, lbl, 0, skipLeft) + extractExpr(tw, expr.Y, lbl, 1, false) } case *ast.ArrayType: if expr == nil { return } kind = dbscheme.ArrayTypeExpr.Index() - extractExpr(tw, expr.Len, lbl, 0) - extractExpr(tw, expr.Elt, lbl, 1) + extractExpr(tw, expr.Len, lbl, 0, false) + extractExpr(tw, expr.Elt, lbl, 1, false) case *ast.StructType: if expr == nil { return @@ -1103,8 +1112,8 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { return } kind = dbscheme.MapTypeExpr.Index() - extractExpr(tw, expr.Key, lbl, 0) - extractExpr(tw, expr.Value, lbl, 1) + extractExpr(tw, expr.Key, lbl, 0, false) + extractExpr(tw, expr.Value, lbl, 1, false) case *ast.ChanType: if expr == nil { return @@ -1114,13 +1123,15 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { log.Fatalf("unsupported channel direction %v", expr.Dir) } kind = tp.Index() - extractExpr(tw, expr.Value, lbl, 0) + extractExpr(tw, expr.Value, lbl, 0, false) default: log.Fatalf("unknown expression of type %T", expr) } dbscheme.ExprsTable.Emit(tw, lbl, kind, parent, idx) extractNodeLocation(tw, expr, lbl) - extractValueOf(tw, expr, lbl) + if !skipExtractingValue { + extractValueOf(tw, expr, lbl) + } } // extractExprs extracts AST information for a list of expressions, which are children of @@ -1130,7 +1141,7 @@ func extractExpr(tw *trap.Writer, expr ast.Expr, parent trap.Label, idx int) { // -1 for decreasing indices) func extractExprs(tw *trap.Writer, exprs []ast.Expr, parent trap.Label, idx int, dir int) { for _, expr := range exprs { - extractExpr(tw, expr, parent, idx) + extractExpr(tw, expr, parent, idx, false) idx += dir } } @@ -1195,11 +1206,11 @@ func extractFields(tw *trap.Writer, fields *ast.FieldList, parent trap.Label, id extractNodeLocation(tw, field, lbl) if field.Names != nil { for i, name := range field.Names { - extractExpr(tw, name, lbl, i+1) + extractExpr(tw, name, lbl, i+1, false) } } - extractExpr(tw, field.Type, lbl, 0) - extractExpr(tw, field.Tag, lbl, -1) + extractExpr(tw, field.Type, lbl, 0, false) + extractExpr(tw, field.Tag, lbl, -1, false) extractDoc(tw, field.Doc, lbl) idx += dir } @@ -1230,21 +1241,21 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { return } kind = dbscheme.LabeledStmtType.Index() - extractExpr(tw, stmt.Label, lbl, 0) + extractExpr(tw, stmt.Label, lbl, 0, false) extractStmt(tw, stmt.Stmt, lbl, 1) case *ast.ExprStmt: if stmt == nil { return } kind = dbscheme.ExprStmtType.Index() - extractExpr(tw, stmt.X, lbl, 0) + extractExpr(tw, stmt.X, lbl, 0, false) case *ast.SendStmt: if stmt == nil { return } kind = dbscheme.SendStmtType.Index() - extractExpr(tw, stmt.Chan, lbl, 0) - extractExpr(tw, stmt.Value, lbl, 1) + extractExpr(tw, stmt.Chan, lbl, 0, false) + extractExpr(tw, stmt.Value, lbl, 1, false) case *ast.IncDecStmt: if stmt == nil { return @@ -1256,7 +1267,7 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { } else { log.Fatalf("unsupported increment/decrement operator %v", stmt.Tok) } - extractExpr(tw, stmt.X, lbl, 0) + extractExpr(tw, stmt.X, lbl, 0, false) case *ast.AssignStmt: if stmt == nil { return @@ -1273,13 +1284,13 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { return } kind = dbscheme.GoStmtType.Index() - extractExpr(tw, stmt.Call, lbl, 0) + extractExpr(tw, stmt.Call, lbl, 0, false) case *ast.DeferStmt: if stmt == nil { return } kind = dbscheme.DeferStmtType.Index() - extractExpr(tw, stmt.Call, lbl, 0) + extractExpr(tw, stmt.Call, lbl, 0, false) case *ast.ReturnStmt: kind = dbscheme.ReturnStmtType.Index() extractExprs(tw, stmt.Results, lbl, 0, 1) @@ -1299,7 +1310,7 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { default: log.Fatalf("unsupported branch statement type %v", stmt.Tok) } - extractExpr(tw, stmt.Label, lbl, 0) + extractExpr(tw, stmt.Label, lbl, 0, false) case *ast.BlockStmt: if stmt == nil { return @@ -1313,7 +1324,7 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { } kind = dbscheme.IfStmtType.Index() extractStmt(tw, stmt.Init, lbl, 0) - extractExpr(tw, stmt.Cond, lbl, 1) + extractExpr(tw, stmt.Cond, lbl, 1, false) extractStmt(tw, stmt.Body, lbl, 2) extractStmt(tw, stmt.Else, lbl, 3) emitScopeNodeInfo(tw, stmt, lbl) @@ -1331,7 +1342,7 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { } kind = dbscheme.ExprSwitchStmtType.Index() extractStmt(tw, stmt.Init, lbl, 0) - extractExpr(tw, stmt.Tag, lbl, 1) + extractExpr(tw, stmt.Tag, lbl, 1, false) extractStmt(tw, stmt.Body, lbl, 2) emitScopeNodeInfo(tw, stmt, lbl) case *ast.TypeSwitchStmt: @@ -1360,7 +1371,7 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { } kind = dbscheme.ForStmtType.Index() extractStmt(tw, stmt.Init, lbl, 0) - extractExpr(tw, stmt.Cond, lbl, 1) + extractExpr(tw, stmt.Cond, lbl, 1, false) extractStmt(tw, stmt.Post, lbl, 2) extractStmt(tw, stmt.Body, lbl, 3) emitScopeNodeInfo(tw, stmt, lbl) @@ -1369,9 +1380,9 @@ func extractStmt(tw *trap.Writer, stmt ast.Stmt, parent trap.Label, idx int) { return } kind = dbscheme.RangeStmtType.Index() - extractExpr(tw, stmt.Key, lbl, 0) - extractExpr(tw, stmt.Value, lbl, 1) - extractExpr(tw, stmt.X, lbl, 2) + extractExpr(tw, stmt.Key, lbl, 0, false) + extractExpr(tw, stmt.Value, lbl, 1, false) + extractExpr(tw, stmt.X, lbl, 2, false) extractStmt(tw, stmt.Body, lbl, 3) emitScopeNodeInfo(tw, stmt, lbl) default: @@ -1426,8 +1437,8 @@ func extractDecl(tw *trap.Writer, decl ast.Decl, parent trap.Label, idx int) { } kind = dbscheme.FuncDeclType.Index() extractFields(tw, decl.Recv, lbl, -1, -1) - extractExpr(tw, decl.Name, lbl, 0) - extractExpr(tw, decl.Type, lbl, 1) + extractExpr(tw, decl.Name, lbl, 0, false) + extractExpr(tw, decl.Type, lbl, 1, false) extractStmt(tw, decl.Body, lbl, 2) extractDoc(tw, decl.Doc, lbl) extractTypeParamDecls(tw, decl.Type.TypeParams, lbl) @@ -1453,8 +1464,8 @@ func extractSpec(tw *trap.Writer, spec ast.Spec, parent trap.Label, idx int) { return } kind = dbscheme.ImportSpecType.Index() - extractExpr(tw, spec.Name, lbl, 0) - extractExpr(tw, spec.Path, lbl, 1) + extractExpr(tw, spec.Name, lbl, 0, false) + extractExpr(tw, spec.Path, lbl, 1, false) extractDoc(tw, spec.Doc, lbl) case *ast.ValueSpec: if spec == nil { @@ -1462,9 +1473,9 @@ func extractSpec(tw *trap.Writer, spec ast.Spec, parent trap.Label, idx int) { } kind = dbscheme.ValueSpecType.Index() for i, name := range spec.Names { - extractExpr(tw, name, lbl, -(1 + i)) + extractExpr(tw, name, lbl, -(1 + i), false) } - extractExpr(tw, spec.Type, lbl, 0) + extractExpr(tw, spec.Type, lbl, 0, false) extractExprs(tw, spec.Values, lbl, 1, 1) extractDoc(tw, spec.Doc, lbl) case *ast.TypeSpec: @@ -1476,9 +1487,9 @@ func extractSpec(tw *trap.Writer, spec ast.Spec, parent trap.Label, idx int) { } else { kind = dbscheme.TypeDefSpecType.Index() } - extractExpr(tw, spec.Name, lbl, 0) + extractExpr(tw, spec.Name, lbl, 0, false) extractTypeParamDecls(tw, spec.TypeParams, lbl) - extractExpr(tw, spec.Type, lbl, 1) + extractExpr(tw, spec.Type, lbl, 1, false) extractDoc(tw, spec.Doc, lbl) } dbscheme.SpecsTable.Emit(tw, lbl, kind, parent, idx) @@ -1907,7 +1918,7 @@ func flattenBinaryExprTree(tw *trap.Writer, e ast.Expr, parent trap.Label, idx i idx = flattenBinaryExprTree(tw, binaryexpr.X, parent, idx) idx = flattenBinaryExprTree(tw, binaryexpr.Y, parent, idx) } else { - extractExpr(tw, e, parent, idx) + extractExpr(tw, e, parent, idx, false) idx = idx + 1 } return idx @@ -1929,10 +1940,10 @@ func extractTypeParamDecls(tw *trap.Writer, fields *ast.FieldList, parent trap.L extractNodeLocation(tw, field, lbl) if field.Names != nil { for i, name := range field.Names { - extractExpr(tw, name, lbl, i+1) + extractExpr(tw, name, lbl, i+1, false) } } - extractExpr(tw, field.Type, lbl, 0) + extractExpr(tw, field.Type, lbl, 0, false) extractDoc(tw, field.Doc, lbl) idx += 1 } @@ -2011,3 +2022,24 @@ func setTypeParamParent(tp *types.TypeParam, newobj types.Object) { log.Fatalf("Parent of type parameter '%s %s' being set to a different value: '%s' vs '%s'", tp.String(), tp.Constraint().String(), obj, newobj) } } + +// skipExtractingValueForLeftOperand returns true if the left operand of `be` +// should not have its value extracted because it is an intermediate value in a +// string concatenation - specifically that the right operand is a string +// literal +func skipExtractingValueForLeftOperand(tw *trap.Writer, be *ast.BinaryExpr) bool { + // check `be` has string type + tpVal := tw.Package.TypesInfo.Types[be] + if tpVal.Value == nil || tpVal.Value.Kind() != constant.String { + return false + } + // check that the right operand of `be` is a basic literal + if _, isBasicLit := be.Y.(*ast.BasicLit); !isBasicLit { + return false + } + // check that the left operand of `be` is not a basic literal + if _, isBasicLit := be.X.(*ast.BasicLit); isBasicLit { + return false + } + return true +} diff --git a/go/ql/lib/change-notes/2024-03-11-addional-gopath-sanitizers.md b/go/ql/lib/change-notes/2024-03-11-addional-gopath-sanitizers.md new file mode 100644 index 00000000000..acfafc26cc5 --- /dev/null +++ b/go/ql/lib/change-notes/2024-03-11-addional-gopath-sanitizers.md @@ -0,0 +1,5 @@ + +* --- +category: minorAnalysis +--- +* Added strings.ReplaceAll, http.ParseMultipartForm sanitizers and remove path sanitizer. \ No newline at end of file diff --git a/go/ql/lib/semmle/go/internal/Locations.qll b/go/ql/lib/semmle/go/internal/Locations.qll index 498ac9d1170..8b0a206b883 100644 --- a/go/ql/lib/semmle/go/internal/Locations.qll +++ b/go/ql/lib/semmle/go/internal/Locations.qll @@ -11,12 +11,19 @@ newtype TLocation = TSynthLocation(string filepath, int startline, int startcolumn, int endline, int endcolumn) { any(DataFlow::Node n).hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn) and // avoid overlap with existing DB locations - not exists(File f | - locations_default(_, f, startline, startcolumn, endline, endcolumn) and - f.getAbsolutePath() = filepath - ) + not existingDBLocation(filepath, startline, startcolumn, endline, endcolumn) } +pragma[nomagic] +private predicate existingDBLocation( + string filepath, int startline, int startcolumn, int endline, int endcolumn +) { + exists(File f | + locations_default(_, f, startline, startcolumn, endline, endcolumn) and + f.getAbsolutePath() = filepath + ) +} + /** * A location as given by a file, a start line, a start column, * an end line, and an end column. diff --git a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll index 42edd470da2..5959bc0ffe8 100644 --- a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll +++ b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll @@ -5,6 +5,7 @@ import go import semmle.go.dataflow.barrierguardutil.RegexpCheck +import DataFlow /** * Provides extension points for customizing the taint tracking configuration for reasoning about @@ -74,13 +75,12 @@ module TaintedPath { } /** - * A call to `[file]path.Clean("/" + e)`, considered to sanitize `e` against path traversal. + * A call to `filepath.Clean("/" + e)`, considered to sanitize `e` against path traversal. */ class FilepathCleanSanitizer extends Sanitizer { FilepathCleanSanitizer() { exists(DataFlow::CallNode cleanCall, StringOps::Concatenation concatNode | - cleanCall = - any(Function f | f.hasQualifiedName(["path", "path/filepath"], "Clean")).getACall() and + cleanCall = any(Function f | f.hasQualifiedName("path/filepath", "Clean")).getACall() and concatNode = cleanCall.getArgument(0) and concatNode.getOperand(0).asExpr().(StringLit).getValue() = "/" and this = cleanCall.getResult() @@ -88,6 +88,16 @@ module TaintedPath { } } + /**An call to ParseMultipartForm creates multipart.Form and cleans multipart.Form.FileHeader.Filename using path.Base() */ + class MultipartClean extends Sanitizer { + MultipartClean() { + exists(DataFlow::FieldReadNode frn | + frn.getField().hasQualifiedName("mime/multipart", "FileHeader", "Filename") and + this = frn + ) + } + } + /** * A check of the form `!strings.Contains(nd, "..")`, considered as a sanitizer guard for * path traversal. @@ -106,6 +116,21 @@ module TaintedPath { } } + /** + * A replacement of the form `!strings.ReplaceAll(nd, "..")` or `!strings.ReplaceAll(nd, ".")`, considered as a sanitizer for + * path traversal. + */ + class DotDotReplace extends Sanitizer { + DotDotReplace() { + exists(DataFlow::CallNode cleanCall, DataFlow::Node valueNode | + cleanCall = any(Function f | f.hasQualifiedName("strings", "ReplaceAll")).getACall() and + valueNode = cleanCall.getArgument(1) and + valueNode.asExpr().(StringLit).getValue() = ["..", "."] and + this = cleanCall.getResult() + ) + } + } + /** * A node `nd` guarded by a check that ensures it is contained within some root folder, * considered as a sanitizer for path traversal. diff --git a/go/ql/test/extractor-tests/no-intermediate-strings/tst.expected b/go/ql/test/extractor-tests/no-intermediate-strings/tst.expected new file mode 100644 index 00000000000..9746d74fb90 --- /dev/null +++ b/go/ql/test/extractor-tests/no-intermediate-strings/tst.expected @@ -0,0 +1,11 @@ +| tst.go:4:6:4:8 | "a" | a | +| tst.go:4:6:4:14 | ...+... | | +| tst.go:4:6:4:20 | ...+... | | +| tst.go:4:6:4:26 | ...+... | | +| tst.go:4:6:4:32 | ...+... | | +| tst.go:4:6:4:38 | ...+... | abcdef | +| tst.go:4:12:4:14 | "b" | b | +| tst.go:4:18:4:20 | "c" | c | +| tst.go:4:24:4:26 | "d" | d | +| tst.go:4:30:4:32 | "e" | e | +| tst.go:4:36:4:38 | "f" | f | diff --git a/go/ql/test/extractor-tests/no-intermediate-strings/tst.go b/go/ql/test/extractor-tests/no-intermediate-strings/tst.go new file mode 100644 index 00000000000..c79c97a8e88 --- /dev/null +++ b/go/ql/test/extractor-tests/no-intermediate-strings/tst.go @@ -0,0 +1,5 @@ +package main + +func main() { + _ = "a" + "b" + "c" + "d" + "e" + "f" +} diff --git a/go/ql/test/extractor-tests/no-intermediate-strings/tst.ql b/go/ql/test/extractor-tests/no-intermediate-strings/tst.ql new file mode 100644 index 00000000000..6367ef51e70 --- /dev/null +++ b/go/ql/test/extractor-tests/no-intermediate-strings/tst.ql @@ -0,0 +1,17 @@ +import go + +string checkStringValue(Expr e) { + result = e.getStringValue() + or + not exists(e.getStringValue()) and result = "" +} + +from Expr e +where e.getType() instanceof StringType +// We should get string values for `"a"`, `"b"`, `"c"` and `"a" + "b" + "c" +// but not `"a" + "b"`. In the extractor we avoid storing the value of +// intermediate strings in string concatenations because in pathological cases +// this could lead to a quadratic blowup in the size of string values stored, +// which then causes performance problems when we iterate through all string +// values. +select e, checkStringValue(e) diff --git a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected index 34d4180a849..4b4748f0d9c 100644 --- a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected +++ b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected @@ -2,18 +2,19 @@ edges | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:13:18:13:30 | call to Query | provenance | | | TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:16:29:16:40 | tainted_path | provenance | | | TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:20:57:20:68 | tainted_path | provenance | | +| TaintedPath.go:13:18:13:30 | call to Query | TaintedPath.go:67:39:67:56 | ...+... | provenance | | | TaintedPath.go:20:57:20:68 | tainted_path | TaintedPath.go:20:28:20:69 | call to Join | provenance | | -| tst.go:14:2:14:39 | ... := ...[1] | tst.go:17:41:17:56 | selection of Filename | provenance | | +| TaintedPath.go:67:39:67:56 | ...+... | TaintedPath.go:67:28:67:57 | call to Clean | provenance | | nodes | TaintedPath.go:13:18:13:22 | selection of URL | semmle.label | selection of URL | | TaintedPath.go:13:18:13:30 | call to Query | semmle.label | call to Query | | TaintedPath.go:16:29:16:40 | tainted_path | semmle.label | tainted_path | | TaintedPath.go:20:28:20:69 | call to Join | semmle.label | call to Join | | TaintedPath.go:20:57:20:68 | tainted_path | semmle.label | tainted_path | -| tst.go:14:2:14:39 | ... := ...[1] | semmle.label | ... := ...[1] | -| tst.go:17:41:17:56 | selection of Filename | semmle.label | selection of Filename | +| TaintedPath.go:67:28:67:57 | call to Clean | semmle.label | call to Clean | +| TaintedPath.go:67:39:67:56 | ...+... | semmle.label | ...+... | subpaths #select | TaintedPath.go:16:29:16:40 | tainted_path | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:16:29:16:40 | tainted_path | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | | TaintedPath.go:20:28:20:69 | call to Join | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:20:28:20:69 | call to Join | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | -| tst.go:17:41:17:56 | selection of Filename | tst.go:14:2:14:39 | ... := ...[1] | tst.go:17:41:17:56 | selection of Filename | This path depends on a $@. | tst.go:14:2:14:39 | ... := ...[1] | user-provided value | +| TaintedPath.go:67:28:67:57 | call to Clean | TaintedPath.go:13:18:13:22 | selection of URL | TaintedPath.go:67:28:67:57 | call to Clean | This path depends on a $@. | TaintedPath.go:13:18:13:22 | selection of URL | user-provided value | diff --git a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go index e663fc3b11d..378cc3acea5 100644 --- a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go +++ b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go @@ -31,6 +31,10 @@ func handler(w http.ResponseWriter, r *http.Request) { w.Write(data) } + // GOOD: Sanitized by strings.ReplaceAll and replaces all .. with empty string + data, _ = ioutil.ReadFile(strings.ReplaceAll(tainted_path, "..", "")) + w.Write(data) + // GOOD: This can only read inside the provided safe path _, err := filepath.Rel("/home/user/safepath", tainted_path) if err == nil { @@ -53,10 +57,23 @@ func handler(w http.ResponseWriter, r *http.Request) { w.Write(data) } - // GOOD: Sanitized by [file]path.Clean with a prepended '/' forcing interpretation + // GOOD: Sanitized by filepath.Clean with a prepended '/' forcing interpretation // as an absolute path, so that Clean will throw away any leading `..` components. data, _ = ioutil.ReadFile(filepath.Clean("/" + tainted_path)) w.Write(data) + + // BAD: Sanitized by path.Clean with a prepended '/' forcing interpretation + // as an absolute path, however is not sufficient for Windows paths. data, _ = ioutil.ReadFile(path.Clean("/" + tainted_path)) w.Write(data) + + // GOOD: Multipart.Form.FileHeader.Filename sanitized by filepath.Base when calling ParseMultipartForm + r.ParseMultipartForm(32 << 20) + form := r.MultipartForm + files, ok := form.File["files"] + if !ok { + return + } + data, _ = ioutil.ReadFile(files[0].Filename) + w.Write(data) } diff --git a/java/ql/automodel/src/AutomodelApplicationModeCharacteristics.qll b/java/ql/automodel/src/AutomodelApplicationModeCharacteristics.qll index 045886985b8..3023677ece8 100644 --- a/java/ql/automodel/src/AutomodelApplicationModeCharacteristics.qll +++ b/java/ql/automodel/src/AutomodelApplicationModeCharacteristics.qll @@ -28,25 +28,27 @@ newtype TApplicationModeEndpoint = AutomodelJavaUtil::isFromSource(call) and exists(Argument argExpr | arg.asExpr() = argExpr and call = argExpr.getCall() and not argExpr.isVararg() - ) + ) and + not AutomodelJavaUtil::isUnexploitableType(arg.getType()) } or TInstanceArgument(Call call, DataFlow::Node arg) { AutomodelJavaUtil::isFromSource(call) and arg = DataFlow::getInstanceArgument(call) and - not call instanceof ConstructorCall + not call instanceof ConstructorCall and + not AutomodelJavaUtil::isUnexploitableType(arg.getType()) } or TImplicitVarargsArray(Call call, DataFlow::ImplicitVarargsArray arg, int idx) { AutomodelJavaUtil::isFromSource(call) and call = arg.getCall() and - idx = call.getCallee().getVaragsParameterIndex() + idx = call.getCallee().getVaragsParameterIndex() and + not AutomodelJavaUtil::isUnexploitableType(arg.getType()) } or - TMethodReturnValue(Call call) { + TMethodReturnValue(MethodCall call) { AutomodelJavaUtil::isFromSource(call) and - not call instanceof ConstructorCall + not AutomodelJavaUtil::isUnexploitableType(call.getType()) } or TOverriddenParameter(Parameter p, Method overriddenMethod) { AutomodelJavaUtil::isFromSource(p) and - not p.getCallable().callsConstructor(_) and p.getCallable().(Method).overrides(overriddenMethod) } @@ -163,7 +165,7 @@ class ImplicitVarargsArray extends CallArgument, TImplicitVarargsArray { * may be a source. */ class MethodReturnValue extends ApplicationModeEndpoint, TMethodReturnValue { - Call call; + MethodCall call; MethodReturnValue() { this = TMethodReturnValue(call) } diff --git a/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java b/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java index 4d6aff63fd0..9691cf86c15 100644 --- a/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java +++ b/java/ql/automodel/test/AutomodelApplicationModeExtraction/Test.java @@ -19,11 +19,11 @@ class Test { AtomicReference reference = new AtomicReference<>(); // uninteresting (parameterless constructor) reference.set( // $ sinkModelCandidate=set(Object):Argument[this] args[0] // $ negativeSinkExample=set(Object):Argument[0] // modeled as a flow step - ); // $ negativeSourceExample=set(Object):ReturnValue // return type is void + ); // not a source candidate (return type is void) } public static void callSupplier(Supplier supplier) { - supplier.get(); // $ sourceModelCandidate=get():ReturnValue + supplier.get(); // not a source candidate (lambda flow) } public static void copyFiles(Path source, Path target, CopyOption option) throws Exception { @@ -52,7 +52,7 @@ class Test { public static int compareFiles(File f1, File f2) { return f1.compareTo( // $ negativeSinkExample=compareTo(File):Argument[this] f2 // $ negativeSinkExample=compareTo(File):Argument[0] // modeled as not a sink - ); // $ negativeSourceExample=compareTo(File):ReturnValue // return type is int + ); // not a source candidate (return type is int) } public static void FilesWalkExample(Path p, FileVisitOption o) throws Exception { @@ -66,6 +66,7 @@ class Test { public static void WebSocketExample(URLConnection c) throws Exception { c.getInputStream(); // $ sinkModelCandidate=getInputStream():Argument[this] positiveSourceExample=getInputStream():ReturnValue(remote) // not a source candidate (manual modeling) + c.connect(); // $ sinkModelCandidate=connect():Argument[this] // not a source candidate (return type is void) } public static void fileFilterExample(File f, FileFilter ff) { @@ -102,10 +103,10 @@ class MoreTests { Files.delete( p // $ sinkModelCandidate=delete(Path):Argument[0] positiveSinkExample=delete(Path):Argument[0](path-injection) - ); // $ negativeSourceExample=delete(Path):ReturnValue // return type is void + ); // not a source candidate (return type is void) Files.deleteIfExists( p // $ sinkModelCandidate=deleteIfExists(Path):Argument[0] positiveSinkExample=deleteIfExists(Path):Argument[0](path-injection) - ); // $ negativeSourceExample=deleteIfExists(Path):ReturnValue // return type is boolean + ); // not a source candidate (return type is boolean) } } \ No newline at end of file diff --git a/java/ql/automodel/test/AutomodelFrameworkModeExtraction/com/github/codeql/test/MyWriter.java b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/com/github/codeql/test/MyWriter.java index b31ace21b4d..62bd773cc2e 100644 --- a/java/ql/automodel/test/AutomodelFrameworkModeExtraction/com/github/codeql/test/MyWriter.java +++ b/java/ql/automodel/test/AutomodelFrameworkModeExtraction/com/github/codeql/test/MyWriter.java @@ -2,7 +2,7 @@ package com.github.codeql.test; public class MyWriter extends java.io.Writer { @Override - public void write(char[] cbuf, int off, int len) { // $ sinkModelCandidate=write(char[],int,int):Argument[this] sourceModelCandidate=write(char[],int,int):Parameter[this] sourceModelCandidate=write(char[],int,int):Parameter[0] + public void write(char[] cbuf, int off, int len) { // $ sinkModelCandidate=write(char[],int,int):Argument[this] positiveSinkExample=write(char[],int,int):Argument[0](file-content-store) sourceModelCandidate=write(char[],int,int):Parameter[this] sourceModelCandidate=write(char[],int,int):Parameter[0] } @Override diff --git a/java/ql/lib/ext/generated/jdk.internal.access.foreign.model.yml b/java/ql/lib/ext/generated/jdk.internal.access.foreign.model.yml deleted file mode 100644 index 5cfddbd9303..00000000000 --- a/java/ql/lib/ext/generated/jdk.internal.access.foreign.model.yml +++ /dev/null @@ -1,10 +0,0 @@ -# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT. -extensions: - - addsTo: - pack: codeql/java-all - extensible: neutralModel - data: - - ["jdk.internal.access.foreign", "UnmapperProxy", "address", "()", "summary", "df-generated"] - - ["jdk.internal.access.foreign", "UnmapperProxy", "fileDescriptor", "()", "summary", "df-generated"] - - ["jdk.internal.access.foreign", "UnmapperProxy", "isSync", "()", "summary", "df-generated"] - - ["jdk.internal.access.foreign", "UnmapperProxy", "unmap", "()", "summary", "df-generated"] diff --git a/java/ql/lib/ext/generated/jdk.internal.access.model.yml b/java/ql/lib/ext/generated/jdk.internal.access.model.yml deleted file mode 100644 index 37d80349303..00000000000 --- a/java/ql/lib/ext/generated/jdk.internal.access.model.yml +++ /dev/null @@ -1,243 +0,0 @@ -# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT. -extensions: - - addsTo: - pack: codeql/java-all - extensible: summaryModel - data: - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", True, "registerCleanup", "(FileDescriptor,PhantomCleanable)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaIOFilePermissionAccess", True, "newPermPlusAltPath", "(FilePermission)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaIOFilePermissionAccess", True, "newPermUsingAltPath", "(FilePermission)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaIORandomAccessFileAccess", True, "openAndDelete", "(File,String)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "addEnableNativeAccess", "(Module)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "addOpensToAllUnnamed", "(Module,Set,Set)", "", "Argument[1].Element", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "addOpensToAllUnnamed", "(Module,Set,Set)", "", "Argument[2].Element", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "createOrGetClassLoaderValueMap", "(ClassLoader)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "defineModule", "(ClassLoader,ModuleDescriptor,URI)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "defineModule", "(ClassLoader,ModuleDescriptor,URI)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "definePackage", "(ClassLoader,String,Module)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "definePackage", "(ClassLoader,String,Module)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "defineUnnamedModule", "(ClassLoader)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "getBytesNoRepl", "(String,Charset)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "getBytesUTF8NoRepl", "(String)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "getServicesCatalog", "(ModuleLayer)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "join", "(String,String,String,String[],int)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "join", "(String,String,String,String[],int)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "join", "(String,String,String,String[],int)", "", "Argument[2]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "join", "(String,String,String,String[],int)", "", "Argument[3].ArrayElement", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "layers", "(ClassLoader)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "layers", "(ModuleLayer)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "newStringNoRepl", "(byte[],Charset)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "newStringUTF8NoRepl", "(byte[],int,int)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "newThreadWithAcc", "(Runnable,AccessControlContext)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "newThreadWithAcc", "(Runnable,AccessControlContext)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "setCause", "(Throwable,Throwable)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "stringConcatHelper", "(String,MethodType)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", True, "stringConcatHelper", "(String,MethodType)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "collectCoordinates", "(VarHandle,int,MethodHandle)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "dropCoordinates", "(VarHandle,int,Class[])", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "filterCoordinates", "(VarHandle,int,MethodHandle[])", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "filterValue", "(VarHandle,MethodHandle,MethodHandle)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "getMethodDescriptor", "(Object)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "getMethodType", "(Object)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "getName", "(Object)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "insertCoordinates", "(VarHandle,int,Object[])", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "nativeMethodHandle", "(NativeEntryPoint,MethodHandle)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "nativeMethodHandle", "(NativeEntryPoint,MethodHandle)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "permuteCoordinates", "(VarHandle,List,int[])", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", True, "permuteCoordinates", "(VarHandle,List,int[])", "", "Argument[1].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newConfiguration", "(ModuleFinder,Map)", "", "Argument[1].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newExports", "(Set,String)", "", "Argument[0].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newExports", "(Set,String)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newExports", "(Set,String,Set)", "", "Argument[0].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newExports", "(Set,String,Set)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newExports", "(Set,String,Set)", "", "Argument[2].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleBuilder", "(String,boolean,Set)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleBuilder", "(String,boolean,Set)", "", "Argument[2].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[2].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[3].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[4].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[5].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[6].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[7].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[8].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newModuleDescriptor", "(String,ModuleDescriptor$Version,Set,Set,Set,Set,Set,Set,Set,String,int)", "", "Argument[9]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newOpens", "(Set,String)", "", "Argument[0].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newOpens", "(Set,String)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newOpens", "(Set,String,Set)", "", "Argument[0].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newOpens", "(Set,String,Set)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newOpens", "(Set,String,Set)", "", "Argument[2].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newProvides", "(String,List)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newProvides", "(String,List)", "", "Argument[1].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newRequires", "(Set,String,ModuleDescriptor$Version)", "", "Argument[0].Element", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newRequires", "(Set,String,ModuleDescriptor$Version)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "newRequires", "(Set,String,ModuleDescriptor$Version)", "", "Argument[2]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", True, "packages", "(ModuleDescriptor$Builder)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "copyConstructor", "(Constructor)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "copyField", "(Field)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "copyMethod", "(Method)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getConstructorAccessor", "(Constructor)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getConstructorAnnotations", "(Constructor)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getConstructorParameterAnnotations", "(Constructor)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getConstructorSignature", "(Constructor)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getExecutableSharedParameterTypes", "(Executable)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getMethodAccessor", "(Method)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "getRoot", "(AccessibleObject)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "leafCopyMethod", "(Method)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "newConstructor", "(Class,Class[],Class[],int,int,String,byte[],byte[])", "", "Argument[1].ArrayElement", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "newConstructor", "(Class,Class[],Class[],int,int,String,byte[],byte[])", "", "Argument[2].ArrayElement", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "newConstructor", "(Class,Class[],Class[],int,int,String,byte[],byte[])", "", "Argument[5]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "newConstructor", "(Class,Class[],Class[],int,int,String,byte[],byte[])", "", "Argument[6]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "newConstructor", "(Class,Class[],Class[],int,int,String,byte[],byte[])", "", "Argument[7]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "setConstructorAccessor", "(Constructor,ConstructorAccessor)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", True, "setMethodAccessor", "(Method,MethodAccessor)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetHttpCookieAccess", True, "header", "(HttpCookie)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetHttpCookieAccess", True, "parse", "(String)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetInetAddressAccess", True, "addressBytes", "(Inet6Address)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetInetAddressAccess", True, "getByName", "(String,InetAddress)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetInetAddressAccess", True, "getOriginalHostName", "(InetAddress)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetURLAccess", True, "getHandler", "(URL)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetUriAccess", True, "create", "(String,String)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNetUriAccess", True, "create", "(String,String)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "acquireScope", "(Buffer,boolean)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "bufferSegment", "(Buffer)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "getBufferBase", "(ByteBuffer)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newDirectByteBuffer", "(long,int,Object,MemorySegmentProxy)", "", "Argument[2]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newDirectByteBuffer", "(long,int,Object,MemorySegmentProxy)", "", "Argument[3]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newHeapByteBuffer", "(byte[],int,int,MemorySegmentProxy)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newHeapByteBuffer", "(byte[],int,int,MemorySegmentProxy)", "", "Argument[3]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newMappedByteBuffer", "(UnmapperProxy,long,int,Object,MemorySegmentProxy)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newMappedByteBuffer", "(UnmapperProxy,long,int,Object,MemorySegmentProxy)", "", "Argument[3]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", True, "newMappedByteBuffer", "(UnmapperProxy,long,int,Object,MemorySegmentProxy)", "", "Argument[4]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess$ProtectionDomainCache", True, "get", "(ProtectionDomain)", "", "Argument[this]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess$ProtectionDomainCache", True, "put", "(ProtectionDomain,PermissionCollection)", "", "Argument[0]", "Argument[this]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess$ProtectionDomainCache", True, "put", "(ProtectionDomain,PermissionCollection)", "", "Argument[1]", "Argument[this]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess", True, "doIntersectionPrivilege", "(PrivilegedAction,AccessControlContext)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess", True, "doIntersectionPrivilege", "(PrivilegedAction,AccessControlContext,AccessControlContext)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess", True, "getProtectDomains", "(AccessControlContext)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySignatureAccess", True, "initSign", "(Signature,PrivateKey,AlgorithmParameterSpec,SecureRandom)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySignatureAccess", True, "initSign", "(Signature,PrivateKey,AlgorithmParameterSpec,SecureRandom)", "", "Argument[2]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySignatureAccess", True, "initSign", "(Signature,PrivateKey,AlgorithmParameterSpec,SecureRandom)", "", "Argument[3]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySignatureAccess", True, "initVerify", "(Signature,PublicKey,AlgorithmParameterSpec)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySignatureAccess", True, "initVerify", "(Signature,PublicKey,AlgorithmParameterSpec)", "", "Argument[2]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilCollectionAccess", True, "listFromTrustedArray", "(Object[])", "", "Argument[0].ArrayElement", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilCollectionAccess", True, "listFromTrustedArrayNullsAllowed", "(Object[])", "", "Argument[0].ArrayElement", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "entryFor", "(JarFile,String)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "entryNames", "(JarFile,CodeSource[])", "", "Argument[1].ArrayElement", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSource", "(JarFile,URL,String)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSource", "(JarFile,URL,String)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSource", "(JarFile,URL,String)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSource", "(JarFile,URL,String)", "", "Argument[2]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSources", "(JarFile,URL)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSources", "(JarFile,URL)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getCodeSources", "(JarFile,URL)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getManifestDigests", "(JarFile)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", True, "getTrustedAttributes", "(Manifest,String)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", True, "getBundle", "(String,Locale,Module)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", True, "getBundle", "(String,Locale,Module)", "", "Argument[1]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", True, "getParent", "(ResourceBundle)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", True, "setLocale", "(ResourceBundle,Locale)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", True, "setName", "(ResourceBundle,String)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", True, "setParent", "(ResourceBundle,ResourceBundle)", "", "Argument[1]", "Argument[0]", "taint", "df-generated"] - - addsTo: - pack: codeql/java-all - extensible: neutralModel - data: - - ["jdk.internal.access", "JavaBeansAccess", "getConstructorPropertiesValue", "(Constructor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaBeansAccess", "getReadMethod", "(Class,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOAccess", "charset", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOAccess", "console", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "close", "(FileDescriptor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "get", "(FileDescriptor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "getAppend", "(FileDescriptor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "getHandle", "(FileDescriptor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "registerCleanup", "(FileDescriptor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "set", "(FileDescriptor,int)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "setAppend", "(FileDescriptor,boolean)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "setHandle", "(FileDescriptor,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaIOFileDescriptorAccess", "unregisterCleanup", "(FileDescriptor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addEnableNativeAccessAllUnnamed", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addExports", "(Module,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addExports", "(Module,String,Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addExportsToAllUnnamed", "(Module,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addNonExportedPackages", "(ModuleLayer)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addOpens", "(Module,String,Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addOpensToAllUnnamed", "(Module,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addReads", "(Module,Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addReadsAllUnnamed", "(Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "addUses", "(Module,Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "bindToLoader", "(ModuleLayer,ClassLoader)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "blockedOn", "(Interruptible)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "casAnnotationType", "(Class,AnnotationType,AnnotationType)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "classData", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "decodeASCII", "(byte[],int,char[],int,int)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "defineClass", "(ClassLoader,Class,String,byte[],ProtectionDomain,boolean,int,Object)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "defineClass", "(ClassLoader,String,byte[],ProtectionDomain,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "exit", "(int)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "fastUUID", "(long,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "findBootstrapClassOrNull", "(String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "findNative", "(ClassLoader,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getAnnotationType", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getConstantPool", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getDeclaredAnnotationMap", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getDeclaredPublicMethods", "(Class,String,Class[])", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getEnumConstantsShared", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getRawClassAnnotations", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getRawClassTypeAnnotations", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "getRawExecutableTypeAnnotations", "(Executable)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "inflateBytesToChars", "(byte[],int,char[],int,int)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "invalidatePackageAccessCache", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "invokeFinalize", "(Object)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "isEnableNativeAccess", "(Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "isReflectivelyExported", "(Module,String,Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "isReflectivelyOpened", "(Module,String,Module)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "protectionDomain", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "registerShutdownHook", "(int,boolean,Runnable)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "stringConcatInitialCoder", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangAccess", "stringConcatMix", "(long,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", "ensureCustomized", "(MethodHandle)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", "generateHolderClasses", "(Stream)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", "getDeclaringClass", "(Object)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", "isNative", "(Object)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", "memoryAccessVarHandle", "(Class,boolean,long,ByteOrder)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangInvokeAccess", "newMemberName", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", "requires", "(ModuleDescriptor$Builder,Set,String,String)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangModuleAccess", "resolveAndBind", "(ModuleFinder,Collection,PrintStream)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangRefAccess", "runFinalization", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangRefAccess", "waitForReferenceProcessing", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", "getConstructorSlot", "(Constructor)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", "getExecutableTypeAnnotationBytes", "(Executable)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", "isTrustedFinalField", "(Field)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaLangReflectAccess", "newInstance", "(Constructor,Object[],Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNetInetAddressAccess", "addressValue", "(Inet4Address)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "force", "(FileDescriptor,long,boolean,long,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "getBufferAddress", "(ByteBuffer)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "getDirectBufferPool", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "isLoaded", "(long,boolean,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "load", "(long,boolean,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "pageSize", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "reserveMemory", "(long,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "unload", "(long,boolean,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "unmapper", "(ByteBuffer)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaNioAccess", "unreserveMemory", "(long,long)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaSecurityAccess", "getProtectionDomainCache", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySignatureAccess", "initVerify", "(Signature,Certificate,AlgorithmParameterSpec)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaSecuritySpecAccess", "clearEncodedKeySpec", "(EncodedKeySpec)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", "ensureInitialization", "(JarFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", "entries2", "(JarFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", "isInitializing", "()", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", "jarFileHasClassPathAttribute", "(JarFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilJarAccess", "setEagerValidation", "(JarFile,boolean)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilResourceBundleAccess", "newResourceBundle", "(Class)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "entries", "(ZipFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "entryNameStream", "(ZipFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "getExtraAttributes", "(ZipEntry)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "getManifestAndSignatureRelatedFiles", "(JarFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "getManifestName", "(JarFile,boolean)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "getManifestNum", "(JarFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "getMetaInfVersions", "(JarFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "setExtraAttributes", "(ZipEntry,int)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "startsWithLocHeader", "(ZipFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaUtilZipFileAccess", "stream", "(ZipFile)", "summary", "df-generated"] - - ["jdk.internal.access", "JavaxCryptoSpecAccess", "clearSecretKeySpec", "(SecretKeySpec)", "summary", "df-generated"] diff --git a/java/ql/lib/ext/generated/jdk.internal.misc.model.yml b/java/ql/lib/ext/generated/jdk.internal.misc.model.yml deleted file mode 100644 index 3ab4816a586..00000000000 --- a/java/ql/lib/ext/generated/jdk.internal.misc.model.yml +++ /dev/null @@ -1,11 +0,0 @@ -# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT. -extensions: - - addsTo: - pack: codeql/java-all - extensible: neutralModel - data: - - ["jdk.internal.misc", "Signal$Handler", "handle", "(Signal)", "summary", "df-generated"] - - ["jdk.internal.misc", "VM$BufferPool", "getCount", "()", "summary", "df-generated"] - - ["jdk.internal.misc", "VM$BufferPool", "getMemoryUsed", "()", "summary", "df-generated"] - - ["jdk.internal.misc", "VM$BufferPool", "getName", "()", "summary", "df-generated"] - - ["jdk.internal.misc", "VM$BufferPool", "getTotalCapacity", "()", "summary", "df-generated"] diff --git a/java/ql/lib/ext/generated/jdk.internal.org.objectweb.asm.model.yml b/java/ql/lib/ext/generated/jdk.internal.org.objectweb.asm.model.yml deleted file mode 100644 index 1121dda6781..00000000000 --- a/java/ql/lib/ext/generated/jdk.internal.org.objectweb.asm.model.yml +++ /dev/null @@ -1,15 +0,0 @@ -# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT. -extensions: - - addsTo: - pack: codeql/java-all - extensible: summaryModel - data: - - ["jdk.internal.org.objectweb.asm", "ClassVisitor", True, "visitAnnotation", "(String,boolean)", "", "Argument[0]", "ReturnValue", "taint", "df-generated"] - - ["jdk.internal.org.objectweb.asm", "ClassVisitor", True, "visitAnnotation", "(String,boolean)", "", "Argument[this]", "ReturnValue", "taint", "df-generated"] - - addsTo: - pack: codeql/java-all - extensible: neutralModel - data: - - ["jdk.internal.org.objectweb.asm", "ClassVisitor", "visit", "(int,int,String,String,String,String[])", "summary", "df-generated"] - - ["jdk.internal.org.objectweb.asm", "ClassVisitor", "visitAttribute", "(Attribute)", "summary", "df-generated"] - - ["jdk.internal.org.objectweb.asm", "ClassVisitor", "visitModule", "(String,int,String)", "summary", "df-generated"] diff --git a/java/ql/src/Advisory/Documentation/SpuriousJavadocParam.ql b/java/ql/src/Advisory/Documentation/SpuriousJavadocParam.ql index 691811687a7..43afd978f01 100644 --- a/java/ql/src/Advisory/Documentation/SpuriousJavadocParam.ql +++ b/java/ql/src/Advisory/Documentation/SpuriousJavadocParam.ql @@ -32,12 +32,24 @@ where ) or documentable instanceof ClassOrInterface and + not documentable instanceof Record and not exists(TypeVariable tv | tv.getGenericType() = documentable | "<" + tv.getName() + ">" = paramTag.getParamName() ) and msg = "@param tag \"" + paramTag.getParamName() + "\" does not match any actual type parameter of type \"" + documentable.getName() + "\"." + or + documentable instanceof Record and + not exists(TypeVariable tv | tv.getGenericType() = documentable | + "<" + tv.getName() + ">" = paramTag.getParamName() + ) and + not documentable.(Record).getCanonicalConstructor().getAParameter().getName() = + paramTag.getParamName() and + msg = + "@param tag \"" + paramTag.getParamName() + + "\" does not match any actual type parameter or record parameter of record \"" + + documentable.getName() + "\"." else // The tag has no value at all. msg = "This @param tag does not have a value." diff --git a/java/ql/src/change-notes/2024-04-02-javadoc-records.md b/java/ql/src/change-notes/2024-04-02-javadoc-records.md new file mode 100644 index 00000000000..e3859c9618b --- /dev/null +++ b/java/ql/src/change-notes/2024-04-02-javadoc-records.md @@ -0,0 +1,5 @@ +--- +category: minorAnalysis +--- +* The `java/unknown-javadoc-parameter` now accepts `@param` tags that apply to the parameters of a + record. diff --git a/java/ql/test/query-tests/SpuriousJavadocParam/Test.java b/java/ql/test/query-tests/SpuriousJavadocParam/Test.java index 84e05540c05..d8891afb756 100644 --- a/java/ql/test/query-tests/SpuriousJavadocParam/Test.java +++ b/java/ql/test/query-tests/SpuriousJavadocParam/Test.java @@ -120,5 +120,17 @@ public class Test { */ interface GenericInterface {} - // Diagnostic Matches: Incomplete inheritance relation for type java.lang.Object and supertype none + /** + * @param i exists + * @param k does not + */ + static record SomeRecord(int i, int j) {} + + /** + * @param exists + * @param does not + * @param i exists + * @param k does not + */ + static record GenericRecord(int i, int j) {} } diff --git a/java/ql/test/query-tests/SpuriousJavadocParam/options b/java/ql/test/query-tests/SpuriousJavadocParam/options new file mode 100644 index 00000000000..fc57fe025b9 --- /dev/null +++ b/java/ql/test/query-tests/SpuriousJavadocParam/options @@ -0,0 +1 @@ +//semmle-extractor-options: --javac-args -source 16 -target 16 diff --git a/java/ql/test/query-tests/SpuriousJavadocParam/test.expected b/java/ql/test/query-tests/SpuriousJavadocParam/test.expected index 42bdb93e60c..f184473ea4a 100644 --- a/java/ql/test/query-tests/SpuriousJavadocParam/test.expected +++ b/java/ql/test/query-tests/SpuriousJavadocParam/test.expected @@ -12,3 +12,6 @@ | Test.java:112:6:112:12 | @param | @param tag "" does not match any actual type parameter of type "GenericClass". | | Test.java:118:6:118:12 | @param | @param tag "T" does not match any actual type parameter of type "GenericInterface". | | Test.java:119:6:119:12 | @param | @param tag "" does not match any actual type parameter of type "GenericInterface". | +| Test.java:125:6:125:12 | @param | @param tag "k" does not match any actual type parameter or record parameter of record "SomeRecord". | +| Test.java:131:6:131:12 | @param | @param tag "" does not match any actual type parameter or record parameter of record "GenericRecord". | +| Test.java:133:6:133:12 | @param | @param tag "k" does not match any actual type parameter or record parameter of record "GenericRecord". | diff --git a/python/extractor/BUILD.bazel b/python/extractor/BUILD.bazel index 3b5a5b3a617..697bf8d49a4 100644 --- a/python/extractor/BUILD.bazel +++ b/python/extractor/BUILD.bazel @@ -5,7 +5,6 @@ py_binary( srcs = [ "make_zips.py", "python_tracer.py", - "unparse.py", ], data = [ "LICENSE-PSF.md", diff --git a/python/extractor/cli-integration-test/force-enable-library-extraction/disabled-test.sh b/python/extractor/cli-integration-test/force-enable-library-extraction/disabled-test.sh deleted file mode 100755 index 9d74cfaca4b..00000000000 --- a/python/extractor/cli-integration-test/force-enable-library-extraction/disabled-test.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -set -Eeuo pipefail # see https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ - -set -x - -CODEQL=${CODEQL:-codeql} - -SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$SCRIPTDIR" - -# start on clean slate -rm -rf dbs -mkdir dbs - -cd "$SCRIPTDIR" - -export CODEQL_EXTRACTOR_PYTHON_FORCE_ENABLE_LIBRARY_EXTRACTION_UNTIL_2_17_0= -$CODEQL database create dbs/normal --language python --source-root repo_dir/ - -export CODEQL_EXTRACTOR_PYTHON_FORCE_ENABLE_LIBRARY_EXTRACTION_UNTIL_2_17_0=1 -$CODEQL database create dbs/with-lib-extraction --language python --source-root repo_dir/ - -# --- - -set +x - -EXTRACTED_NORMAL=$(unzip -l dbs/normal/src.zip | wc -l) -EXTRACTED_WITH_LIB_EXTRACTION=$(unzip -l dbs/with-lib-extraction/src.zip | wc -l) - -exitcode=0 - -echo "EXTRACTED_NORMAL=$EXTRACTED_NORMAL" -echo "EXTRACTED_WITH_LIB_EXTRACTION=$EXTRACTED_WITH_LIB_EXTRACTION" - -if [[ ! $EXTRACTED_WITH_LIB_EXTRACTION -gt $EXTRACTED_NORMAL ]]; then - echo "ERROR: EXTRACTED_WITH_LIB_EXTRACTION not greater than EXTRACTED_NORMAL" - exitcode=1 -fi - -exit $exitcode diff --git a/python/extractor/cli-integration-test/force-enable-library-extraction/repo_dir/foo.py b/python/extractor/cli-integration-test/force-enable-library-extraction/repo_dir/foo.py deleted file mode 100644 index cf0cd77a108..00000000000 --- a/python/extractor/cli-integration-test/force-enable-library-extraction/repo_dir/foo.py +++ /dev/null @@ -1,3 +0,0 @@ -import pip - -print(42) diff --git a/python/extractor/cli-integration-test/ignore-venv/disabled-test.sh b/python/extractor/cli-integration-test/ignore-venv/disabled-test.sh deleted file mode 100755 index e1368008891..00000000000 --- a/python/extractor/cli-integration-test/ignore-venv/disabled-test.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash - -set -Eeuo pipefail # see https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ - -set -x - -CODEQL=${CODEQL:-codeql} - -SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$SCRIPTDIR" - -# start on clean slate -rm -rf dbs repo_dir/venv* -mkdir dbs - - -# set up venvs -cd repo_dir - -python3 -m venv venv -venv/bin/pip install flask - -python3 -m venv venv2 - -cd "$SCRIPTDIR" - -# In 2.16.0 we stop extracting libraries by default, so to test this functionality we -# need to force enable it. Once we release 2.17.0 and turn off library extraction for -# good, we can remove the part of this test ensuring that dependencies in an active -# venv are still extracted (since that will no longer be the case). -export CODEQL_EXTRACTOR_PYTHON_FORCE_ENABLE_LIBRARY_EXTRACTION_UNTIL_2_17_0=1 - -# Create DBs with venv2 active (that does not have flask installed) -source repo_dir/venv2/bin/activate - -export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE= -$CODEQL database create dbs/normal --language python --source-root repo_dir/ - -export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE=1 -$CODEQL database create dbs/no-venv-ignore --language python --source-root repo_dir/ - -# Create DB with venv active that has flask installed. We want to ensure that we're -# still able to resolve imports to flask, but don't want to extract EVERYTHING from -# within the venv. Important note is that the test-file in the repo_dir actually imports -# flask :D -source repo_dir/venv/bin/activate -export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE= -$CODEQL database create dbs/normal-with-flask-venv --language python --source-root repo_dir/ - -# --- - -set +x - -EXTRACTED_NORMAL=$(unzip -l dbs/normal/src.zip | wc -l) -EXTRACTED_NO_VENV_IGNORE=$(unzip -l dbs/no-venv-ignore/src.zip | wc -l) -EXTRACTED_ACTIVE_FLASK=$(unzip -l dbs/normal-with-flask-venv/src.zip | wc -l) - -exitcode=0 - -echo "EXTRACTED_NORMAL=$EXTRACTED_NORMAL" -echo "EXTRACTED_NO_VENV_IGNORE=$EXTRACTED_NO_VENV_IGNORE" -echo "EXTRACTED_ACTIVE_FLASK=$EXTRACTED_ACTIVE_FLASK" - -if [[ ! $EXTRACTED_NORMAL -lt $EXTRACTED_NO_VENV_IGNORE ]]; then - echo "ERROR: EXTRACTED_NORMAL not smaller EXTRACTED_NO_VENV_IGNORE" - exitcode=1 -fi - -if [[ ! $EXTRACTED_NORMAL -lt $EXTRACTED_ACTIVE_FLASK ]]; then - echo "ERROR: EXTRACTED_NORMAL not smaller EXTRACTED_ACTIVE_FLASK" - exitcode=1 -fi - -if [[ ! $EXTRACTED_ACTIVE_FLASK -lt $EXTRACTED_NO_VENV_IGNORE ]]; then - echo "ERROR: EXTRACTED_ACTIVE_FLASK not smaller EXTRACTED_NO_VENV_IGNORE" - exitcode=1 -fi - -exit $exitcode diff --git a/python/extractor/cli-integration-test/ignore-venv/test.sh b/python/extractor/cli-integration-test/ignore-venv/test.sh new file mode 100755 index 00000000000..8735203ec32 --- /dev/null +++ b/python/extractor/cli-integration-test/ignore-venv/test.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -Eeuo pipefail # see https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ + +set -x + +CODEQL=${CODEQL:-codeql} + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$SCRIPTDIR" + +# start on clean slate +rm -rf dbs repo_dir/venv* +mkdir dbs + + +# set up venvs +cd repo_dir + +# make venv with some package in it (so we show that our ignore logic is correct) +python3 -m venv venv +venv/bin/pip install flask + +cd "$SCRIPTDIR" + +export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE= +$CODEQL database create dbs/normal --language python --source-root repo_dir/ + +export CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE=1 +$CODEQL database create dbs/no-venv-ignore --language python --source-root repo_dir/ + +# --- + +set +x + +EXTRACTED_NORMAL=$(unzip -l dbs/normal/src.zip | wc -l) +EXTRACTED_NO_VENV_IGNORE=$(unzip -l dbs/no-venv-ignore/src.zip | wc -l) + +exitcode=0 + +echo "EXTRACTED_NORMAL=$EXTRACTED_NORMAL" +echo "EXTRACTED_NO_VENV_IGNORE=$EXTRACTED_NO_VENV_IGNORE" + +if [[ ! $EXTRACTED_NORMAL -lt $EXTRACTED_NO_VENV_IGNORE ]]; then + echo "ERROR: EXTRACTED_NORMAL not smaller EXTRACTED_NO_VENV_IGNORE" + exitcode=1 +fi + +exit $exitcode diff --git a/python/extractor/licenses.md b/python/extractor/licenses.md index f3a5c6cc458..d585a84626d 100644 --- a/python/extractor/licenses.md +++ b/python/extractor/licenses.md @@ -6,7 +6,6 @@ | `tsg-python/tree-sitter-python` | Y | MIT | Used in `tsg-python` to parse Python files | | `tsg-python` | Y | MIT / Apache | This is our own creation, so are free to choose what license it is covered by. | | `tree-sitter-graph` | N | MIT / Apache | Used in `tsg-python` to execute files written in the `tree-sitter-graph` language. | -| `unparse.py` | Y | PSF | Copied and adapted from `Tools/unparse.py` from the `cpython` source code, with attribution. | | `imp.py` | Y | PSF | Copied and adapted from `Lib/imp.py` from the `cpython` source code, with attribution. | | `semmle/data/*.trap` | Y | PSF | These files were derived from the C source code of the `cpython` project, and are used in our modelling of built-in objects. No attribution, currently. | | `semmle/thrift/parse.py` | Y | Apache | Includes a grammar based on https://github.com/apache/thrift/blob/master/doc/specs/idl.md, with comment stating this attribution. | diff --git a/python/extractor/make_zips.py b/python/extractor/make_zips.py index b91b1bf458d..faf595acc37 100755 --- a/python/extractor/make_zips.py +++ b/python/extractor/make_zips.py @@ -8,7 +8,6 @@ import optparse import compileall from python_tracer import getzipfilename -from unparse import strip_comments_and_docstrings # TO DO -- Add options to set destination directory and source directory @@ -84,9 +83,7 @@ def write_source(zipped, root, name, extensions=[".py"]): if ext not in extensions: continue path = os.path.join(dirpath, name) - temp = strip_comments_and_docstrings(path) - zipped.write(temp, os.path.relpath(path, root)) - os.remove(temp) + zipped.write(path, os.path.relpath(path, root)) def main(): parser = optparse.OptionParser(usage = "usage: %prog [install-dir]") diff --git a/python/extractor/tsg-python/Cargo.toml b/python/extractor/tsg-python/Cargo.toml index feecd254159..ac4d4093189 100644 --- a/python/extractor/tsg-python/Cargo.toml +++ b/python/extractor/tsg-python/Cargo.toml @@ -7,9 +7,14 @@ authors = ["Taus Brock-Nannestad "] edition = "2018" # When changing/updating these, the `Cargo.Bazel.lock` file has to be regenerated. -# Check out the documentation at https://bazelbuild.github.io/rules_rust/crate_universe.html#repinning--updating-dependencies -# for how to do so. The bazel repository for the tsg-python project is called `py_deps`, -# and instead of calling `bazel sync`, `./build --bazel sync` should be used instead, to always use the correct bazel version. +# Run `CARGO_BAZEL_REPIN=true CARGO_BAZEL_REPIN_ONLY=py_deps ./build --bazel sync --only=py_deps` +# in the `semmle-code` repository to do so. +# For more information, check out the documentation at +# https://bazelbuild.github.io/rules_rust/crate_universe.html#repinning--updating-dependencies +# In the future, the hope is to move this handling of the dependencies entirely into the `codeql` repository, +# but that depends on `rules_rust` being fully compatible with bzlmod, which they aren't yet +# (c.f. https://github.com/bazelbuild/rules_rust/issues/2452). +# Warning: The process takes >5min on my M1 mac, so do wait for a while. [dependencies] anyhow = "1.0" regex = "1" diff --git a/python/extractor/tsg-python/tsp/BUILD.bazel b/python/extractor/tsg-python/tsp/BUILD.bazel index 71319e894f6..e3389fce1cc 100644 --- a/python/extractor/tsg-python/tsp/BUILD.bazel +++ b/python/extractor/tsg-python/tsp/BUILD.bazel @@ -7,7 +7,7 @@ package(default_visibility = ["//visibility:public"]) # This will run the build script from the root of the workspace, and # collect the outputs. cargo_build_script( - name = "tsg-build", + name = "tsp-build", srcs = ["bindings/rust/build.rs"], data = glob([ "src/**", @@ -32,7 +32,7 @@ rust_library( proc_macro_deps = all_crate_deps( proc_macro = True, ), - deps = [":tsg-build"] + all_crate_deps( + deps = [":tsp-build"] + all_crate_deps( normal = True, ), ) diff --git a/python/extractor/unparse.py b/python/extractor/unparse.py deleted file mode 100644 index b12f1501592..00000000000 --- a/python/extractor/unparse.py +++ /dev/null @@ -1,709 +0,0 @@ -#Copied Tools.unparse.py with modifications. Copyright PSF. - -"Usage: unparse.py " -import sys -import ast -import tokenize -import io -import os -import shutil - -# Large float and imaginary literals get turned into infinities in the AST. -# We unparse those infinities to INFSTR. -INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) - -def interleave(inter, f, seq): - """Call f on each item in seq, calling inter() in between. - """ - seq = iter(seq) - try: - f(next(seq)) - except StopIteration: - pass - else: - for x in seq: - inter() - f(x) - -class Unparser: - """Methods in this class recursively traverse an AST and - output source code for the abstract syntax; original formatting - is disregarded. """ - - def __init__(self, tree, file = sys.stdout): - """Unparser(tree, file=sys.stdout) -> None. - Print the source for tree to file.""" - self.f = file - self._indent = 0 - self.dispatch(tree) - print("", file=self.f) - self.f.flush() - - def fill(self, text = ""): - "Indent a piece of text, according to the current indentation level" - self.f.write("\n"+" "*self._indent + text) - - def write(self, text): - "Append a piece of text to the current line." - self.f.write(text) - - def enter(self): - "Print ':', and increase the indentation." - self.write(":") - self._indent += 1 - - def leave(self): - "Decrease the indentation level." - self._indent -= 1 - - def dispatch(self, tree): - "Dispatcher function, dispatching tree type T to method _T." - if isinstance(tree, list): - for t in tree: - self.dispatch(t) - return - meth = getattr(self, "_"+tree.__class__.__name__) - meth(tree) - - def remove_docstring(self, t): - if hasattr(t, "docstring"): - return - if not t.body: - return - if not isinstance(t.body[0], ast.Expr): - return - if not isinstance(t.body[0].value, ast.Str): - return - t.body = t.body[1:] - - def add_pass(self, t): - if t.body: - #No pass needed - return - t.body = [ast.Pass()] - - ############### Unparsing methods ###################### - # There should be one method per concrete grammar type # - # Constructors should be grouped by sum type. Ideally, # - # this would follow the order in the grammar, but # - # currently doesn't. # - ######################################################## - - def _Module(self, tree): - self.remove_docstring(tree) - self.add_pass(tree) - for stmt in tree.body: - self.dispatch(stmt) - - # stmt - def _Expr(self, tree): - self.fill() - self.dispatch(tree.value) - - def _Import(self, t): - self.fill("import ") - interleave(lambda: self.write(", "), self.dispatch, t.names) - - def _ImportFrom(self, t): - self.fill("from ") - self.write("." * t.level) - if t.module: - self.write(t.module) - self.write(" import ") - interleave(lambda: self.write(", "), self.dispatch, t.names) - - def _Assign(self, t): - self.fill() - for target in t.targets: - self.dispatch(target) - self.write(" = ") - self.dispatch(t.value) - - def _AugAssign(self, t): - self.fill() - self.dispatch(t.target) - self.write(" "+self.binop[t.op.__class__.__name__]+"= ") - self.dispatch(t.value) - - def _AnnAssign(self, t): - self.fill() - if not t.simple and isinstance(t.target, ast.Name): - self.write('(') - self.dispatch(t.target) - if not t.simple and isinstance(t.target, ast.Name): - self.write(')') - self.write(": ") - self.dispatch(t.annotation) - if t.value: - self.write(" = ") - self.dispatch(t.value) - - def _Return(self, t): - self.fill("return") - if t.value: - self.write(" ") - self.dispatch(t.value) - - def _Pass(self, t): - self.fill("pass") - - def _Break(self, t): - self.fill("break") - - def _Continue(self, t): - self.fill("continue") - - def _Delete(self, t): - self.fill("del ") - interleave(lambda: self.write(", "), self.dispatch, t.targets) - - def _Assert(self, t): - self.fill("assert ") - self.dispatch(t.test) - if t.msg: - self.write(", ") - self.dispatch(t.msg) - - def _Global(self, t): - self.fill("global ") - interleave(lambda: self.write(", "), self.write, t.names) - - def _Nonlocal(self, t): - self.fill("nonlocal ") - interleave(lambda: self.write(", "), self.write, t.names) - - def _Await(self, t): - self.write("(") - self.write("await") - if t.value: - self.write(" ") - self.dispatch(t.value) - self.write(")") - - def _Yield(self, t): - self.write("(") - self.write("yield") - if t.value: - self.write(" ") - self.dispatch(t.value) - self.write(")") - - def _YieldFrom(self, t): - self.write("(") - self.write("yield from") - if t.value: - self.write(" ") - self.dispatch(t.value) - self.write(")") - - def _Raise(self, t): - self.fill("raise") - if not t.exc: - assert not t.cause - return - self.write(" ") - self.dispatch(t.exc) - if t.cause: - self.write(" from ") - self.dispatch(t.cause) - - def _Try(self, t): - self.fill("try") - self.enter() - self.dispatch(t.body) - self.leave() - for ex in t.handlers: - self.dispatch(ex) - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - if t.finalbody: - self.fill("finally") - self.enter() - self.dispatch(t.finalbody) - self.leave() - - def _ExceptHandler(self, t): - self.fill("except") - if t.type: - self.write(" ") - self.dispatch(t.type) - if t.name: - self.write(" as ") - self.write(t.name) - self.enter() - self.dispatch(t.body) - self.leave() - - def _ClassDef(self, t): - self.write("\n") - for deco in t.decorator_list: - self.fill("@") - self.dispatch(deco) - self.fill("class "+t.name) - self.write("(") - comma = False - for e in t.bases: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - for e in t.keywords: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - self.write(")") - - self.enter() - self.remove_docstring(t) - self.add_pass(t) - self.dispatch(t.body) - self.leave() - - def _FunctionDef(self, t): - self.__FunctionDef_helper(t, "def") - - def _AsyncFunctionDef(self, t): - self.__FunctionDef_helper(t, "async def") - - def __FunctionDef_helper(self, t, fill_suffix): - self.write("\n") - for deco in t.decorator_list: - self.fill("@") - self.dispatch(deco) - def_str = fill_suffix+" "+t.name + "(" - self.fill(def_str) - self.dispatch(t.args) - self.write(")") - if t.returns: - self.write(" -> ") - self.dispatch(t.returns) - self.enter() - self.remove_docstring(t) - self.add_pass(t) - self.dispatch(t.body) - self.leave() - - def _For(self, t): - self.__For_helper("for ", t) - - def _AsyncFor(self, t): - self.__For_helper("async for ", t) - - def __For_helper(self, fill, t): - self.fill(fill) - self.dispatch(t.target) - self.write(" in ") - self.dispatch(t.iter) - self.enter() - self.dispatch(t.body) - self.leave() - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _If(self, t): - self.fill("if ") - self.dispatch(t.test) - self.enter() - self.dispatch(t.body) - self.leave() - # collapse nested ifs into equivalent elifs. - while (t.orelse and len(t.orelse) == 1 and - isinstance(t.orelse[0], ast.If)): - t = t.orelse[0] - self.fill("elif ") - self.dispatch(t.test) - self.enter() - self.dispatch(t.body) - self.leave() - # final else - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _While(self, t): - self.fill("while ") - self.dispatch(t.test) - self.enter() - self.dispatch(t.body) - self.leave() - if t.orelse: - self.fill("else") - self.enter() - self.dispatch(t.orelse) - self.leave() - - def _With(self, t): - self.fill("with ") - interleave(lambda: self.write(", "), self.dispatch, t.items) - self.enter() - self.dispatch(t.body) - self.leave() - - def _AsyncWith(self, t): - self.fill("async with ") - interleave(lambda: self.write(", "), self.dispatch, t.items) - self.enter() - self.dispatch(t.body) - self.leave() - - # expr - def _Bytes(self, t): - self.write(repr(t.s)) - - def _Str(self, tree): - s = repr(tree.s).encode("ascii", errors="backslashreplace").decode("ascii") - self.write(s) - - def _JoinedStr(self, t): - self.write("f") - string = io.StringIO() - self._fstring_JoinedStr(t, string.write) - self.write(repr(string.getvalue())) - - def _FormattedValue(self, t): - self.write("f") - string = io.StringIO() - self._fstring_FormattedValue(t, string.write) - self.write(repr(string.getvalue())) - - def _fstring_JoinedStr(self, t, write): - for value in t.values: - meth = getattr(self, "_fstring_" + type(value).__name__) - meth(value, write) - - def _fstring_Str(self, t, write): - value = t.s.replace("{", "{{").replace("}", "}}") - write(value) - - def _fstring_Constant(self, t, write): - assert isinstance(t.value, str) - value = t.value.replace("{", "{{").replace("}", "}}") - write(value) - - def _fstring_FormattedValue(self, t, write): - write("{") - expr = io.StringIO() - Unparser(t.value, expr) - expr = expr.getvalue().rstrip("\n") - if expr.startswith("{"): - write(" ") # Separate pair of opening brackets as "{ {" - write(expr) - if t.conversion != -1: - conversion = chr(t.conversion) - assert conversion in "sra" - write("!%s" % conversion) - if t.format_spec: - write(":") - meth = getattr(self, "_fstring_" + type(t.format_spec).__name__) - meth(t.format_spec, write) - write("}") - - def _Name(self, t): - self.write(t.id) - - def _write_constant(self, value): - if isinstance(value, (float, complex)): - self.write(repr(value).replace("inf", INFSTR)) - else: - self.write(repr(value)) - - def _Constant(self, t): - value = t.value - if isinstance(value, tuple): - self.write("(") - if len(value) == 1: - self._write_constant(value[0]) - self.write(",") - else: - interleave(lambda: self.write(", "), self._write_constant, value) - self.write(")") - else: - self._write_constant(t.value) - - def _NameConstant(self, t): - self.write(repr(t.value)) - - def _Num(self, t): - # Substitute overflowing decimal literal for AST infinities. - self.write(repr(t.n).replace("inf", INFSTR)) - - def _List(self, t): - self.write("[") - interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write("]") - - def _ListComp(self, t): - self.write("[") - self.dispatch(t.elt) - for gen in t.generators: - self.dispatch(gen) - self.write("]") - - def _GeneratorExp(self, t): - self.write("(") - self.dispatch(t.elt) - for gen in t.generators: - self.dispatch(gen) - self.write(")") - - def _SetComp(self, t): - self.write("{") - self.dispatch(t.elt) - for gen in t.generators: - self.dispatch(gen) - self.write("}") - - def _DictComp(self, t): - self.write("{") - self.dispatch(t.key) - self.write(": ") - self.dispatch(t.value) - for gen in t.generators: - self.dispatch(gen) - self.write("}") - - def _comprehension(self, t): - if hasattr(t, "is_async") and t.is_async: - self.write(" async for ") - else: - self.write(" for ") - self.dispatch(t.target) - self.write(" in ") - self.dispatch(t.iter) - for if_clause in t.ifs: - self.write(" if ") - self.dispatch(if_clause) - - def _IfExp(self, t): - self.write("(") - self.dispatch(t.body) - self.write(" if ") - self.dispatch(t.test) - self.write(" else ") - self.dispatch(t.orelse) - self.write(")") - - def _Set(self, t): - assert(t.elts) # should be at least one element - self.write("{") - interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write("}") - - def _Dict(self, t): - self.write("{") - def write_key_value_pair(k, v): - self.dispatch(k) - self.write(": ") - self.dispatch(v) - - def write_item(item): - k, v = item - if k is None: - # for dictionary unpacking operator in dicts {**{'y': 2}} - # see PEP 448 for details - self.write("**") - self.dispatch(v) - else: - write_key_value_pair(k, v) - interleave(lambda: self.write(", "), write_item, zip(t.keys, t.values)) - self.write("}") - - def _Tuple(self, t): - self.write("(") - if len(t.elts) == 1: - elt = t.elts[0] - self.dispatch(elt) - self.write(",") - else: - interleave(lambda: self.write(", "), self.dispatch, t.elts) - self.write(")") - - unop = {"Invert":"~", "Not": "not", "UAdd":"+", "USub":"-"} - def _UnaryOp(self, t): - self.write("(") - self.write(self.unop[t.op.__class__.__name__]) - self.write(" ") - self.dispatch(t.operand) - self.write(")") - - binop = { "Add":"+", "Sub":"-", "Mult":"*", "MatMult":"@", "Div":"/", "Mod":"%", - "LShift":"<<", "RShift":">>", "BitOr":"|", "BitXor":"^", "BitAnd":"&", - "FloorDiv":"//", "Pow": "**"} - def _BinOp(self, t): - self.write("(") - self.dispatch(t.left) - self.write(" " + self.binop[t.op.__class__.__name__] + " ") - self.dispatch(t.right) - self.write(")") - - cmpops = {"Eq":"==", "NotEq":"!=", "Lt":"<", "LtE":"<=", "Gt":">", "GtE":">=", - "Is":"is", "IsNot":"is not", "In":"in", "NotIn":"not in"} - def _Compare(self, t): - self.write("(") - self.dispatch(t.left) - for o, e in zip(t.ops, t.comparators): - self.write(" " + self.cmpops[o.__class__.__name__] + " ") - self.dispatch(e) - self.write(")") - - boolops = {ast.And: 'and', ast.Or: 'or'} - def _BoolOp(self, t): - self.write("(") - s = " %s " % self.boolops[t.op.__class__] - interleave(lambda: self.write(s), self.dispatch, t.values) - self.write(")") - - def _Attribute(self,t): - self.dispatch(t.value) - # Special case: 3.__abs__() is a syntax error, so if t.value - # is an integer literal then we need to either parenthesize - # it or add an extra space to get 3 .__abs__(). - if isinstance(t.value, ast.Num) and isinstance(t.value.n, int): - self.write(" ") - self.write(".") - self.write(t.attr) - - def _Call(self, t): - self.dispatch(t.func) - self.write("(") - comma = False - for e in t.args: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - for e in t.keywords: - if comma: self.write(", ") - else: comma = True - self.dispatch(e) - self.write(")") - - def _Subscript(self, t): - self.dispatch(t.value) - self.write("[") - self.dispatch(t.slice) - self.write("]") - - def _Starred(self, t): - self.write("*") - self.dispatch(t.value) - - # slice - def _Ellipsis(self, t): - self.write("...") - - def _Index(self, t): - self.dispatch(t.value) - - def _Slice(self, t): - if t.lower: - self.dispatch(t.lower) - self.write(":") - if t.upper: - self.dispatch(t.upper) - if t.step: - self.write(":") - self.dispatch(t.step) - - def _ExtSlice(self, t): - interleave(lambda: self.write(', '), self.dispatch, t.dims) - - # argument - def _arg(self, t): - self.write(t.arg) - if t.annotation: - self.write(": ") - self.dispatch(t.annotation) - - # others - def _arguments(self, t): - first = True - # normal arguments - defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults - for a, d in zip(t.args, defaults): - if first:first = False - else: self.write(", ") - self.dispatch(a) - if d: - self.write("=") - self.dispatch(d) - - # varargs, or bare '*' if no varargs but keyword-only arguments present - if t.vararg or t.kwonlyargs: - if first:first = False - else: self.write(", ") - self.write("*") - if t.vararg: - self.write(t.vararg.arg) - if t.vararg.annotation: - self.write(": ") - self.dispatch(t.vararg.annotation) - - # keyword-only arguments - if t.kwonlyargs: - for a, d in zip(t.kwonlyargs, t.kw_defaults): - if first:first = False - else: self.write(", ") - self.dispatch(a), - if d: - self.write("=") - self.dispatch(d) - - # kwargs - if t.kwarg: - if first:first = False - else: self.write(", ") - self.write("**"+t.kwarg.arg) - if t.kwarg.annotation: - self.write(": ") - self.dispatch(t.kwarg.annotation) - - def _keyword(self, t): - if t.arg is None: - self.write("**") - else: - self.write(t.arg) - self.write("=") - self.dispatch(t.value) - - def _Lambda(self, t): - self.write("(") - self.write("lambda ") - self.dispatch(t.args) - self.write(": ") - self.dispatch(t.body) - self.write(")") - - def _alias(self, t): - self.write(t.name) - if t.asname: - self.write(" as "+t.asname) - - def _withitem(self, t): - self.dispatch(t.context_expr) - if t.optional_vars: - self.write(" as ") - self.dispatch(t.optional_vars) - -def roundtrip(filename, outpath): - with open(filename, "rb") as pyfile: - encoding = tokenize.detect_encoding(pyfile.readline)[0] - with open(filename, "r", encoding=encoding) as pyfile: - source = pyfile.read() - tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST) - with open(outpath, "w", encoding=encoding) as output: - Unparser(tree, output) - -def strip_comments_and_docstrings(path): - tmp = path + ".tmp" - if path.endswith(".py"): - roundtrip(path, tmp) - else: - shutil.copy(path, tmp) - return tmp diff --git a/python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll b/python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll index 34e48439271..2f4c74ea9e4 100644 --- a/python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll +++ b/python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll @@ -22,7 +22,7 @@ private import semmle.python.dataflow.new.FlowSummary /** * A remote flow source originating from a CSV source row. */ -private class RemoteFlowSourceFromCsv extends RemoteFlowSource { +private class RemoteFlowSourceFromCsv extends RemoteFlowSource::Range { RemoteFlowSourceFromCsv() { this = ModelOutput::getASourceNode("remote").asSource() } override string getSourceType() { result = "Remote flow (from model)" } diff --git a/python/ql/test/library-tests/web/zope/Test.expected b/python/ql/test/library-tests/web/zope/Test.expected deleted file mode 100644 index bb3ca1f441e..00000000000 --- a/python/ql/test/library-tests/web/zope/Test.expected +++ /dev/null @@ -1,3 +0,0 @@ -| 12 | ControlFlowNode for implementer | class implementer | ../../../query-tests/Security/lib/zope/interface/__init__.py:5 | -| 13 | ControlFlowNode for IThing | class IThing | test.py:4 | -| 14 | ControlFlowNode for Thing | class Thing | test.py:9 | diff --git a/python/ql/test/library-tests/web/zope/Test.ql b/python/ql/test/library-tests/web/zope/Test.ql deleted file mode 100644 index ca705ac292e..00000000000 --- a/python/ql/test/library-tests/web/zope/Test.ql +++ /dev/null @@ -1,10 +0,0 @@ -import python -import semmle.python.TestUtils - -from ControlFlowNode f, Value v, ControlFlowNode x -where - exists(ExprStmt s | s.getValue().getAFlowNode() = f) and - f.pointsTo(v, x) and - f.getLocation().getFile().getBaseName() = "test.py" -select f.getLocation().getStartLine(), f.toString(), v.toString(), - remove_library_prefix(x.getLocation()) diff --git a/python/ql/test/library-tests/web/zope/options b/python/ql/test/library-tests/web/zope/options deleted file mode 100644 index 7fb713d5924..00000000000 --- a/python/ql/test/library-tests/web/zope/options +++ /dev/null @@ -1 +0,0 @@ -semmle-extractor-options: --max-import-depth=3 -p ../../../query-tests/Security/lib/ diff --git a/python/ql/test/library-tests/web/zope/test.py b/python/ql/test/library-tests/web/zope/test.py deleted file mode 100644 index 64ac9138d7e..00000000000 --- a/python/ql/test/library-tests/web/zope/test.py +++ /dev/null @@ -1,14 +0,0 @@ - -from zope.interface import Interface, implementer - -class IThing(Interface): - pass - - -@implementer(IThing) -class Thing(object): - pass - -implementer -IThing -Thing diff --git a/python/ql/test/query-tests/Security/lib/Crypto/Cipher/AES.py b/python/ql/test/query-tests/Security/lib/Crypto/Cipher/AES.py deleted file mode 100644 index bcab58405ea..00000000000 --- a/python/ql/test/query-tests/Security/lib/Crypto/Cipher/AES.py +++ /dev/null @@ -1,3 +0,0 @@ - -def new(*args): - pass diff --git a/python/ql/test/query-tests/Security/lib/Crypto/Cipher/ARC4.py b/python/ql/test/query-tests/Security/lib/Crypto/Cipher/ARC4.py deleted file mode 100644 index bcab58405ea..00000000000 --- a/python/ql/test/query-tests/Security/lib/Crypto/Cipher/ARC4.py +++ /dev/null @@ -1,3 +0,0 @@ - -def new(*args): - pass diff --git a/python/ql/test/query-tests/Security/lib/Crypto/Cipher/__init__.py b/python/ql/test/query-tests/Security/lib/Crypto/Cipher/__init__.py deleted file mode 100644 index ab22f65be6e..00000000000 --- a/python/ql/test/query-tests/Security/lib/Crypto/Cipher/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = ['AES', 'ARC4'] diff --git a/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/DSA.py b/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/DSA.py deleted file mode 100644 index e002a77697e..00000000000 --- a/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/DSA.py +++ /dev/null @@ -1,2 +0,0 @@ -def generate(bits): - pass diff --git a/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/RSA.py b/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/RSA.py deleted file mode 100644 index e002a77697e..00000000000 --- a/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/RSA.py +++ /dev/null @@ -1,2 +0,0 @@ -def generate(bits): - pass diff --git a/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/__init__.py b/python/ql/test/query-tests/Security/lib/Crypto/PublicKey/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/Crypto/__init__.py b/python/ql/test/query-tests/Security/lib/Crypto/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/airspeed.py b/python/ql/test/query-tests/Security/lib/airspeed.py deleted file mode 100644 index aea81c72b71..00000000000 --- a/python/ql/test/query-tests/Security/lib/airspeed.py +++ /dev/null @@ -1,3 +0,0 @@ -class Template: - def __init__(self, content, filename=""): - pass diff --git a/python/ql/test/query-tests/Security/lib/base64.py b/python/ql/test/query-tests/Security/lib/base64.py deleted file mode 100644 index a2b97ca63ac..00000000000 --- a/python/ql/test/query-tests/Security/lib/base64.py +++ /dev/null @@ -1,2 +0,0 @@ -def decodestring(s): - return None diff --git a/python/ql/test/query-tests/Security/lib/bottle.py b/python/ql/test/query-tests/Security/lib/bottle.py deleted file mode 100644 index fe42507852c..00000000000 --- a/python/ql/test/query-tests/Security/lib/bottle.py +++ /dev/null @@ -1,69 +0,0 @@ - -class Bottle(object): - - def route(self, path=None, method='GET', **options): - pass - - def get(self, path=None, method='GET', **options): - """ Equals :meth:`route`. """ - return self.route(path, method, **options) - - def post(self, path=None, method='POST', **options): - """ Equals :meth:`route` with a ``POST`` method parameter. """ - return self.route(path, method, **options) - - def put(self, path=None, method='PUT', **options): - """ Equals :meth:`route` with a ``PUT`` method parameter. """ - return self.route(path, method, **options) - - def delete(self, path=None, method='DELETE', **options): - """ Equals :meth:`route` with a ``DELETE`` method parameter. """ - return self.route(path, method, **options) - - def error(self, code=500): - """ Decorator: Register an output handler for a HTTP error code""" - def wrapper(handler): - self.error_handler[int(code)] = handler - return handler - return wrapper - -#Use same wrapper logic as the original `bottle` code. - -def make_default_app_wrapper(name): - """ Return a callable that relays calls to the current default app. """ - - @functools.wraps(getattr(Bottle, name)) - def wrapper(*a, **ka): - return getattr(app(), name)(*a, **ka) - - return wrapper - -route = make_default_app_wrapper('route') -get = make_default_app_wrapper('get') -post = make_default_app_wrapper('post') -put = make_default_app_wrapper('put') -delete = make_default_app_wrapper('delete') -patch = make_default_app_wrapper('patch') -error = make_default_app_wrapper('error') -mount = make_default_app_wrapper('mount') -hook = make_default_app_wrapper('hook') -install = make_default_app_wrapper('install') -uninstall = make_default_app_wrapper('uninstall') -url = make_default_app_wrapper('get_url') - -class LocalProxy(object): - pass - -class LocalRequest(LocalProxy): - pass - -class LocalResponse(LocalProxy): - pass - - -request = LocalRequest() -response = LocalResponse() - - -def redirect(url, code=None): - pass diff --git a/python/ql/test/query-tests/Security/lib/cheetah/Template/__init__.py b/python/ql/test/query-tests/Security/lib/cheetah/Template/__init__.py deleted file mode 100644 index 04b862eafea..00000000000 --- a/python/ql/test/query-tests/Security/lib/cheetah/Template/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -class Template(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/cherrypy/__init__.py b/python/ql/test/query-tests/Security/lib/cherrypy/__init__.py deleted file mode 100644 index a1ae780e3b5..00000000000 --- a/python/ql/test/query-tests/Security/lib/cherrypy/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ - - -from ._helper import expose, popargs, url - -class _ThreadLocalProxy(object): - def __getattr__(self, name): - pass - - -request = _ThreadLocalProxy('request') -response = _ThreadLocalProxy('response') - -def quickstart(root=None, script_name='', config=None): - """Mount the given root, start the builtin server (and engine), then block.""" - pass diff --git a/python/ql/test/query-tests/Security/lib/cherrypy/_helper.py b/python/ql/test/query-tests/Security/lib/cherrypy/_helper.py deleted file mode 100644 index fc3f3604a57..00000000000 --- a/python/ql/test/query-tests/Security/lib/cherrypy/_helper.py +++ /dev/null @@ -1,31 +0,0 @@ -def expose(func=None, alias=None): - """Expose the function or class. - Optionally provide an alias or set of aliases. - """ - def expose_(func): - func.exposed = True - return func - - return expose_ - - -def popargs(*args, **kwargs): - """Decorate _cp_dispatch.""" - - def decorated(cls_or_self=None, vpath=None): - if inspect.isclass(cls_or_self): - # cherrypy.popargs is a class decorator - return cls - - # We're in the actual function - self = cls_or_self - if vpath: - return getattr(self, vpath.pop(0), None) - else: - return self - - return decorated - -def url(path='', qs='', script_name=None, base=None, relative=None): - #Do some opaque stuff here... - return new_url diff --git a/python/ql/test/query-tests/Security/lib/chevron.py b/python/ql/test/query-tests/Security/lib/chevron.py deleted file mode 100644 index 012ba58fdb8..00000000000 --- a/python/ql/test/query-tests/Security/lib/chevron.py +++ /dev/null @@ -1,6 +0,0 @@ - - -def render(template='', data={}, partials_path='.', partials_ext='mustache', - partials_dict={}, padding='', def_ldel='{{', def_rdel='}}', - scopes=None): - pass diff --git a/python/ql/test/query-tests/Security/lib/cryptography/__init__.py b/python/ql/test/query-tests/Security/lib/cryptography/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/__init__.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/__init__.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/__init__.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/dsa.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/dsa.py deleted file mode 100644 index b4c37f6b540..00000000000 --- a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/dsa.py +++ /dev/null @@ -1,4 +0,0 @@ - -def generate_private_key(key_size, backend): - pass - diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/ec.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/ec.py deleted file mode 100644 index d2eb4b0f1af..00000000000 --- a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/ec.py +++ /dev/null @@ -1,22 +0,0 @@ - -def generate_private_key(curve, backend): - pass - -class SECT571R1(object): - name = "sect571r1" - key_size = 570 - - -class SECP384R1(object): - name = "secp384r1" - key_size = 384 - - -class SECP224R1(object): - name = "secp224r1" - key_size = 224 - - -class SECT163K1(object): - name = "sect163k1" - key_size = 163 \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/rsa.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/rsa.py deleted file mode 100644 index 3a5c91734c2..00000000000 --- a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/asymmetric/rsa.py +++ /dev/null @@ -1,3 +0,0 @@ - -def generate_private_key(public_exponent, key_size, backend): - pass diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/__init__.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/__init__.py deleted file mode 100644 index f37f14e88f8..00000000000 --- a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ - -class Cipher(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/algorithms.py b/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/algorithms.py deleted file mode 100644 index e423804e4f5..00000000000 --- a/python/ql/test/query-tests/Security/lib/cryptography/hazmat/primitives/ciphers/algorithms.py +++ /dev/null @@ -1,3 +0,0 @@ - -class ARC4(object): - name = "RC4" diff --git a/python/ql/test/query-tests/Security/lib/dill.py b/python/ql/test/query-tests/Security/lib/dill.py deleted file mode 100644 index 410fa21087e..00000000000 --- a/python/ql/test/query-tests/Security/lib/dill.py +++ /dev/null @@ -1,2 +0,0 @@ -def loads(*args, **kwargs): - return None diff --git a/python/ql/test/query-tests/Security/lib/django/__init__.py b/python/ql/test/query-tests/Security/lib/django/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/django/conf/__init__.py b/python/ql/test/query-tests/Security/lib/django/conf/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/django/conf/urls.py b/python/ql/test/query-tests/Security/lib/django/conf/urls.py deleted file mode 100644 index 49a248e16aa..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/conf/urls.py +++ /dev/null @@ -1,7 +0,0 @@ -# https://docs.djangoproject.com/en/1.11/_modules/django/conf/urls/#url -def url(regex, view, kwargs=None, name=None): - pass - - -def patterns(*urls): - pass diff --git a/python/ql/test/query-tests/Security/lib/django/db/__init__.py b/python/ql/test/query-tests/Security/lib/django/db/__init__.py deleted file mode 100644 index 3e291c5731d..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/db/__init__.py +++ /dev/null @@ -1 +0,0 @@ -connection = object() diff --git a/python/ql/test/query-tests/Security/lib/django/db/models/__init__.py b/python/ql/test/query-tests/Security/lib/django/db/models/__init__.py deleted file mode 100644 index eb9c72adc45..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/db/models/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -class Model: - pass diff --git a/python/ql/test/query-tests/Security/lib/django/db/models/expressions.py b/python/ql/test/query-tests/Security/lib/django/db/models/expressions.py deleted file mode 100644 index d7e0d1c27b6..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/db/models/expressions.py +++ /dev/null @@ -1,2 +0,0 @@ -class RawSQL: - pass diff --git a/python/ql/test/query-tests/Security/lib/django/http/__init__.py b/python/ql/test/query-tests/Security/lib/django/http/__init__.py deleted file mode 100644 index f2ac6d2c55b..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/http/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .response import * -from .request import HttpRequest diff --git a/python/ql/test/query-tests/Security/lib/django/http/request.py b/python/ql/test/query-tests/Security/lib/django/http/request.py deleted file mode 100644 index 93afdaf29b7..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/http/request.py +++ /dev/null @@ -1,2 +0,0 @@ -class HttpRequest(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/django/http/response.py b/python/ql/test/query-tests/Security/lib/django/http/response.py deleted file mode 100644 index 1110a3cde19..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/http/response.py +++ /dev/null @@ -1,46 +0,0 @@ -class HttpResponseBase(object): - status_code = 200 - - def __init__(self, content_type=None, status=None, reason=None, charset=None): - pass - - -class HttpResponse(HttpResponseBase): - def __init__(self, content=b"", *args, **kwargs): - super().__init__(*args, **kwargs) - # Content is a bytestring. See the `content` property methods. - self.content = content - - -class HttpResponseRedirectBase(HttpResponse): - def __init__(self, redirect_to, *args, **kwargs): - super().__init__(*args, **kwargs) - ... - - -class HttpResponsePermanentRedirect(HttpResponseRedirectBase): - status_code = 301 - - -class HttpResponseRedirect(HttpResponseRedirectBase): - status_code = 302 - - -class HttpResponseNotFound(HttpResponse): - status_code = 404 - - -class JsonResponse(HttpResponse): - - def __init__( - self, - data, - encoder=..., - safe=True, - json_dumps_params=None, - **kwargs - ): - # fake code to represent what is going on :) - kwargs.setdefault("content_type", "application/json") - data = str(data) - super().__init__(content=data, **kwargs) diff --git a/python/ql/test/query-tests/Security/lib/django/shortcuts.py b/python/ql/test/query-tests/Security/lib/django/shortcuts.py deleted file mode 100644 index 4783729178a..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/shortcuts.py +++ /dev/null @@ -1,8 +0,0 @@ -# see https://docs.djangoproject.com/en/1.11/_modules/django/shortcuts/#redirect -# https://github.com/django/django/blob/86908785076b2bbc31b908781da6b6ad1779b18b/django/shortcuts.py - -def render(request, template_name, context=None, content_type=None, status=None, using=None): - pass - -def redirect(to, *args, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/django/urls.py b/python/ql/test/query-tests/Security/lib/django/urls.py deleted file mode 100644 index dc28eece874..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/urls.py +++ /dev/null @@ -1,7 +0,0 @@ -from functools import partial - -def _path(route, view, kwargs=None, name=None, Pattern=None): - pass - -path = partial(_path, Pattern='RoutePattern (but this is a mock)') -re_path = partial(_path, Pattern='RegexPattern (but this is a mock)') diff --git a/python/ql/test/query-tests/Security/lib/django/views/__init__.py b/python/ql/test/query-tests/Security/lib/django/views/__init__.py deleted file mode 100644 index 47de98f1351..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/views/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# For django 2.x and 3.x -class View: - pass diff --git a/python/ql/test/query-tests/Security/lib/django/views/generic.py b/python/ql/test/query-tests/Security/lib/django/views/generic.py deleted file mode 100644 index edb71bed762..00000000000 --- a/python/ql/test/query-tests/Security/lib/django/views/generic.py +++ /dev/null @@ -1,3 +0,0 @@ -# For django 1.x -class View: - pass diff --git a/python/ql/test/query-tests/Security/lib/fabric/__init__.py b/python/ql/test/query-tests/Security/lib/fabric/__init__.py deleted file mode 100644 index fcdb406611e..00000000000 --- a/python/ql/test/query-tests/Security/lib/fabric/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .connection import Connection -from .group import Group, SerialGroup, ThreadingGroup -from .tasks import task diff --git a/python/ql/test/query-tests/Security/lib/fabric/api.py b/python/ql/test/query-tests/Security/lib/fabric/api.py deleted file mode 100644 index b26d5a04b6d..00000000000 --- a/python/ql/test/query-tests/Security/lib/fabric/api.py +++ /dev/null @@ -1,29 +0,0 @@ -# For the 1.x version - -def needs_host(func): - @wraps(func) - def inner(*args, **kwargs): - return func(*args, **kwargs) - return inner - - -def local(command, capture=False, shell=None): - pass - - -@needs_host -def run(command, shell=True, pty=True, combine_stderr=None, quiet=False, - warn_only=False, stdout=None, stderr=None, timeout=None, shell_escape=None, - capture_buffer_size=None): - pass - - -@needs_host -def sudo(command, shell=True, pty=True, combine_stderr=None, user=None, - quiet=False, warn_only=False, stdout=None, stderr=None, group=None, - timeout=None, shell_escape=None, capture_buffer_size=None): - pass - -# https://github.com/fabric/fabric/blob/1.14/fabric/tasks.py#L281 -def execute(task, *args, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/fabric/connection.py b/python/ql/test/query-tests/Security/lib/fabric/connection.py deleted file mode 100644 index e046aa4e744..00000000000 --- a/python/ql/test/query-tests/Security/lib/fabric/connection.py +++ /dev/null @@ -1,15 +0,0 @@ -from invoke import Context - -@decorator -def opens(method, self, *args, **kwargs): - self.open() - return method(self, *args, **kwargs) - -class Connection(Context): - - def open(self): - pass - - @opens - def run(self, command, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/fabric/group.py b/python/ql/test/query-tests/Security/lib/fabric/group.py deleted file mode 100644 index 131231d2268..00000000000 --- a/python/ql/test/query-tests/Security/lib/fabric/group.py +++ /dev/null @@ -1,11 +0,0 @@ -class Group(list): - def run(self, *args, **kwargs): - raise NotImplementedError - -class SerialGroup(Group): - def run(self, *args, **kwargs): - pass - -class ThreadingGroup(Group): - def run(self, *args, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/fabric/tasks.py b/python/ql/test/query-tests/Security/lib/fabric/tasks.py deleted file mode 100644 index 08968e68ab5..00000000000 --- a/python/ql/test/query-tests/Security/lib/fabric/tasks.py +++ /dev/null @@ -1,2 +0,0 @@ -def task(*args, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/falcon/__init__.py b/python/ql/test/query-tests/Security/lib/falcon/__init__.py deleted file mode 100644 index 5983e6a41a5..00000000000 --- a/python/ql/test/query-tests/Security/lib/falcon/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ - -from falcon.api import API -from falcon.request import Request -from falcon.response import Response diff --git a/python/ql/test/query-tests/Security/lib/falcon/api.py b/python/ql/test/query-tests/Security/lib/falcon/api.py deleted file mode 100644 index 2763d8ca3e1..00000000000 --- a/python/ql/test/query-tests/Security/lib/falcon/api.py +++ /dev/null @@ -1,14 +0,0 @@ - -"""Falcon API class.""" - -class API(object): - - def add_route(self, uri_template, resource, **kwargs): - pass - - def add_sink(self, sink, prefix=r'/'): - pass - - def add_error_handler(self, exception, handler=None): - pass - diff --git a/python/ql/test/query-tests/Security/lib/falcon/request.py b/python/ql/test/query-tests/Security/lib/falcon/request.py deleted file mode 100644 index 4e39d52d841..00000000000 --- a/python/ql/test/query-tests/Security/lib/falcon/request.py +++ /dev/null @@ -1,3 +0,0 @@ - -class Request(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/falcon/response.py b/python/ql/test/query-tests/Security/lib/falcon/response.py deleted file mode 100644 index d03bbee54ac..00000000000 --- a/python/ql/test/query-tests/Security/lib/falcon/response.py +++ /dev/null @@ -1,4 +0,0 @@ - - -class Response(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/flask/__init__.py b/python/ql/test/query-tests/Security/lib/flask/__init__.py deleted file mode 100644 index 73ce61e7ccf..00000000000 --- a/python/ql/test/query-tests/Security/lib/flask/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -from .globals import request -from .globals import current_app - -class Flask(object): - # Only some methods mocked, signature copied from - # https://flask.palletsprojects.com/en/1.1.x/api/#flask.Flask - def run(host=None, port=None, debug=None, load_dotenv=True, **options): - pass - - def make_response(rv): - pass - - def add_url_rule(rule, endpoint=None, view_func=None, provide_automatic_options=None, **options): - pass - -class Response(object): - pass - -def redirect(location, code=302, Response=None): - pass - -def make_response(rv): - if not args: - return current_app.response_class() - if len(args) == 1: - args = args[0] - return current_app.make_response(args) - -def escape(txt): - return Markup.escape(txt) - -def render_template_string(source, **context): - pass \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/lib/flask/globals.py b/python/ql/test/query-tests/Security/lib/flask/globals.py deleted file mode 100644 index 36320cb4886..00000000000 --- a/python/ql/test/query-tests/Security/lib/flask/globals.py +++ /dev/null @@ -1,6 +0,0 @@ - -class LocalProxy(object): - pass - -request = LocalProxy() -current_app = LocalProxy() diff --git a/python/ql/test/query-tests/Security/lib/flask/urls.py b/python/ql/test/query-tests/Security/lib/flask/urls.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/flask/views.py b/python/ql/test/query-tests/Security/lib/flask/views.py deleted file mode 100644 index 1b7007f20da..00000000000 --- a/python/ql/test/query-tests/Security/lib/flask/views.py +++ /dev/null @@ -1,6 +0,0 @@ -class View(object): - pass - - -class MethodView(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/invoke/__init__.py b/python/ql/test/query-tests/Security/lib/invoke/__init__.py deleted file mode 100644 index 1cb865d16ff..00000000000 --- a/python/ql/test/query-tests/Security/lib/invoke/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from .context import Context -from .tasks import task - -def run(command, **kwargs): - pass - -def sudo(command, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/invoke/context.py b/python/ql/test/query-tests/Security/lib/invoke/context.py deleted file mode 100644 index dff247b23f1..00000000000 --- a/python/ql/test/query-tests/Security/lib/invoke/context.py +++ /dev/null @@ -1,3 +0,0 @@ -class Context(object): - def run(self, command, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/invoke/tasks.py b/python/ql/test/query-tests/Security/lib/invoke/tasks.py deleted file mode 100644 index 08968e68ab5..00000000000 --- a/python/ql/test/query-tests/Security/lib/invoke/tasks.py +++ /dev/null @@ -1,2 +0,0 @@ -def task(*args, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/jinja2.py b/python/ql/test/query-tests/Security/lib/jinja2.py deleted file mode 100644 index 7c95c1417ab..00000000000 --- a/python/ql/test/query-tests/Security/lib/jinja2.py +++ /dev/null @@ -1,23 +0,0 @@ - -class Environment(object): - - def __init__(self, loader, autoescape): - pass - -class Template(object): - - def __init__(self, source, autoescape): - pass - -def select_autoescape(files=[]): - def autoescape(template_name): - pass - return autoescape - -class FileSystemLoader(object): - - def __init__(self, searchpath): - pass - -def from_string(source, globals=None, template_class=None): - pass \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/lib/libxml2/__init__.py b/python/ql/test/query-tests/Security/lib/libxml2/__init__.py deleted file mode 100644 index 057488829f4..00000000000 --- a/python/ql/test/query-tests/Security/lib/libxml2/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -def parseFile(filename): - return xmlDoc(_obj=None) - - -class xmlDoc(Object): - def __init__(self, _obj=None): - pass - - def xpathEval(self, expr): - pass diff --git a/python/ql/test/query-tests/Security/lib/lxml/__init__.py b/python/ql/test/query-tests/Security/lib/lxml/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/lxml/etree.py b/python/ql/test/query-tests/Security/lib/lxml/etree.py deleted file mode 100644 index 139553b0d6c..00000000000 --- a/python/ql/test/query-tests/Security/lib/lxml/etree.py +++ /dev/null @@ -1,37 +0,0 @@ -class _ElementTree(object): - def xpath(self, _path, namespaces=None, extensions=None, smart_strings=True, **_variables): - pass - - def xslt(self, _xslt, extensions=None, access_control=None, **_kw): - pass - - -class ETXPath(object): - def __init__(self, path, extensions=None, regexp=True, smart_strings=True): - pass - - -class XPath(object): - def __init__(self, path, namespaces=None, extensions=None, regexp=True, smart_strings=True): - pass - - -class XSLT(object): - def __init__(self, xslt_input, extensions=None, regexp=True, access_control=None): - pass - - -def parse(self, parser=None, base_url=None): - return _ElementTree() - - -def fromstring(self, text, parser=None, base_url=None): - pass - - -def fromstringlist(self, strings, parser=None): - pass - - -def XML(self, text, parser=None, base_url=None): - pass diff --git a/python/ql/test/query-tests/Security/lib/marshall.py b/python/ql/test/query-tests/Security/lib/marshall.py deleted file mode 100644 index 410fa21087e..00000000000 --- a/python/ql/test/query-tests/Security/lib/marshall.py +++ /dev/null @@ -1,2 +0,0 @@ -def loads(*args, **kwargs): - return None diff --git a/python/ql/test/query-tests/Security/lib/os/__init__.py b/python/ql/test/query-tests/Security/lib/os/__init__.py deleted file mode 100644 index 397e2eb12da..00000000000 --- a/python/ql/test/query-tests/Security/lib/os/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -def system(cmd, *args, **kwargs): - return None - -def popen(cmd, *args, **kwargs): - return None - -def chmod(path, mode): - pass - -def open(path, flags, mode): - pass - -def tempnam(*args): - pass - -def tmpnam(*args): - pass diff --git a/python/ql/test/query-tests/Security/lib/os/path.py b/python/ql/test/query-tests/Security/lib/os/path.py deleted file mode 100644 index d54092e80b0..00000000000 --- a/python/ql/test/query-tests/Security/lib/os/path.py +++ /dev/null @@ -1,5 +0,0 @@ -def join(a, *b): - return a + "/" + "/".join(b) - -def normpath(x): - return x diff --git a/python/ql/test/query-tests/Security/lib/paramiko/__init__.py b/python/ql/test/query-tests/Security/lib/paramiko/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/paramiko/client.py b/python/ql/test/query-tests/Security/lib/paramiko/client.py deleted file mode 100644 index 6343fbe78dd..00000000000 --- a/python/ql/test/query-tests/Security/lib/paramiko/client.py +++ /dev/null @@ -1,15 +0,0 @@ -class SSHClient(object): - def __init__(self, *args, **kwargs): - pass - - def set_missing_host_key_policy(self, *args, **kwargs): - pass - -class AutoAddPolicy(object): - pass - -class WarningPolicy(object): - pass - -class RejectPolicy(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/pickle.py b/python/ql/test/query-tests/Security/lib/pickle.py deleted file mode 100644 index 410fa21087e..00000000000 --- a/python/ql/test/query-tests/Security/lib/pickle.py +++ /dev/null @@ -1,2 +0,0 @@ -def loads(*args, **kwargs): - return None diff --git a/python/ql/test/query-tests/Security/lib/pyOpenSSL/SSL.py b/python/ql/test/query-tests/Security/lib/pyOpenSSL/SSL.py deleted file mode 100644 index eb135d2d5e0..00000000000 --- a/python/ql/test/query-tests/Security/lib/pyOpenSSL/SSL.py +++ /dev/null @@ -1,10 +0,0 @@ -SSLv2_METHOD = 1 -SSLv3_METHOD = 2 -SSLv23_METHOD = 3 -TLSv1_METHOD = 4 -TLSv1_1_METHOD = 5 -TLSv1_2_METHOD = 6 - -class Context(object): - def __init__(self, *args, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/pyOpenSSL/__init__.py b/python/ql/test/query-tests/Security/lib/pyOpenSSL/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/pyramid/__init__.py b/python/ql/test/query-tests/Security/lib/pyramid/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/pyramid/response.py b/python/ql/test/query-tests/Security/lib/pyramid/response.py deleted file mode 100644 index a482ac919ff..00000000000 --- a/python/ql/test/query-tests/Security/lib/pyramid/response.py +++ /dev/null @@ -1,2 +0,0 @@ -class Response(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/pyramid/view.py b/python/ql/test/query-tests/Security/lib/pyramid/view.py deleted file mode 100644 index c7487bf827b..00000000000 --- a/python/ql/test/query-tests/Security/lib/pyramid/view.py +++ /dev/null @@ -1,7 +0,0 @@ -# https://docs.pylonsproject.org/projects/pyramid/en/1.10-branch/_modules/pyramid/view.html#view_config -class view_config(object): - def __init__(self, **settings): - pass - - def __call__(self, wrapped): - pass diff --git a/python/ql/test/query-tests/Security/lib/requests.py b/python/ql/test/query-tests/Security/lib/requests.py deleted file mode 100644 index 01afdbb1208..00000000000 --- a/python/ql/test/query-tests/Security/lib/requests.py +++ /dev/null @@ -1,21 +0,0 @@ - -def get(url, params=None, **kwargs): - pass - -def options(url, **kwargs): - pass - -def head(url, **kwargs): - pass - -def post(url, data=None, json=None, **kwargs): - pass - -def put(url, data=None, **kwargs): - pass - -def patch(url, data=None, **kwargs): - pass - -def delete(url, **kwargs): - pass diff --git a/python/ql/test/query-tests/Security/lib/subprocess.py b/python/ql/test/query-tests/Security/lib/subprocess.py deleted file mode 100644 index efb2ba183f0..00000000000 --- a/python/ql/test/query-tests/Security/lib/subprocess.py +++ /dev/null @@ -1,2 +0,0 @@ -def Popen(*args, **kwargs): - return None \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/lib/tg.py b/python/ql/test/query-tests/Security/lib/tg.py deleted file mode 100644 index 976b7427b33..00000000000 --- a/python/ql/test/query-tests/Security/lib/tg.py +++ /dev/null @@ -1,30 +0,0 @@ - -def validate(validators): - def validator(func): - return func - -def expose(*args): - if cond: - return args[0] - def with_template(func): - func - return with_template - -class TGController(object): - pass - -class TurboGearsContextMember(object): - """Member of the TurboGears request context. - Provides access to turbogears context members - like request, response, template context and so on - """ - - def __init__(self, name): - self.__dict__['name'] = name - - def _current_obj(self): - return getattr(context, self.name) - - -request = TurboGearsContextMember(name="request") -response = TurboGearsContextMember(name="response") diff --git a/python/ql/test/query-tests/Security/lib/tornado/__init__.py b/python/ql/test/query-tests/Security/lib/tornado/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/tornado/web.py b/python/ql/test/query-tests/Security/lib/tornado/web.py deleted file mode 100644 index 41b91848992..00000000000 --- a/python/ql/test/query-tests/Security/lib/tornado/web.py +++ /dev/null @@ -1,2 +0,0 @@ -class RequestHandler(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/traceback.py b/python/ql/test/query-tests/Security/lib/traceback.py deleted file mode 100644 index 2a7c5e58847..00000000000 --- a/python/ql/test/query-tests/Security/lib/traceback.py +++ /dev/null @@ -1,2 +0,0 @@ -def format_exc(): - return None \ No newline at end of file diff --git a/python/ql/test/query-tests/Security/lib/twisted/__init__.py b/python/ql/test/query-tests/Security/lib/twisted/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/twisted/web/__init__.py b/python/ql/test/query-tests/Security/lib/twisted/web/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/twisted/web/http.py b/python/ql/test/query-tests/Security/lib/twisted/web/http.py deleted file mode 100644 index c7ac65eaf05..00000000000 --- a/python/ql/test/query-tests/Security/lib/twisted/web/http.py +++ /dev/null @@ -1,2 +0,0 @@ -class Request(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/twisted/web/resource.py b/python/ql/test/query-tests/Security/lib/twisted/web/resource.py deleted file mode 100644 index 1fe186864cb..00000000000 --- a/python/ql/test/query-tests/Security/lib/twisted/web/resource.py +++ /dev/null @@ -1,2 +0,0 @@ -class Resource(object): - pass diff --git a/python/ql/test/query-tests/Security/lib/yaml.py b/python/ql/test/query-tests/Security/lib/yaml.py deleted file mode 100644 index 51fe2fd1d42..00000000000 --- a/python/ql/test/query-tests/Security/lib/yaml.py +++ /dev/null @@ -1,2 +0,0 @@ -def load(s): - pass diff --git a/python/ql/test/query-tests/Security/lib/zope/__init__.py b/python/ql/test/query-tests/Security/lib/zope/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/ql/test/query-tests/Security/lib/zope/interface/__init__.py b/python/ql/test/query-tests/Security/lib/zope/interface/__init__.py deleted file mode 100644 index 423c7b58341..00000000000 --- a/python/ql/test/query-tests/Security/lib/zope/interface/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -class Interface(): - pass - - -class implementer: - - def __call__(self, ob): - ... - return ob diff --git a/ruby/ql/lib/codeql/ruby/security/UnsafeCodeConstructionQuery.qll b/ruby/ql/lib/codeql/ruby/security/UnsafeCodeConstructionQuery.qll index 4cf57f36071..32cc9a4f821 100644 --- a/ruby/ql/lib/codeql/ruby/security/UnsafeCodeConstructionQuery.qll +++ b/ruby/ql/lib/codeql/ruby/security/UnsafeCodeConstructionQuery.qll @@ -46,12 +46,6 @@ private module UnsafeCodeConstructionConfig implements DataFlow::ConfigSig { // override to require the path doesn't have unmatched return steps DataFlow::FlowFeature getAFeature() { result instanceof DataFlow::FeatureHasSourceCallContext } - - predicate allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet set) { - // allow implicit reads of array elements - isSink(node) and - set.isElementOfTypeOrUnknown("int") - } } /** diff --git a/ruby/ql/lib/codeql/ruby/security/UnsafeShellCommandConstructionQuery.qll b/ruby/ql/lib/codeql/ruby/security/UnsafeShellCommandConstructionQuery.qll index 7576702a2d4..b4e0b8b6bb5 100644 --- a/ruby/ql/lib/codeql/ruby/security/UnsafeShellCommandConstructionQuery.qll +++ b/ruby/ql/lib/codeql/ruby/security/UnsafeShellCommandConstructionQuery.qll @@ -49,12 +49,6 @@ private module UnsafeShellCommandConstructionConfig implements DataFlow::ConfigS // override to require the path doesn't have unmatched return steps DataFlow::FlowFeature getAFeature() { result instanceof DataFlow::FeatureHasSourceCallContext } - - predicate allowImplicitRead(DataFlow::Node node, DataFlow::ContentSet set) { - // allow implicit reads of array elements - isSink(node) and - set.isElementOfTypeOrUnknown("int") - } } /** diff --git a/ruby/ql/src/queries/security/cwe-078/UnsafeShellCommandConstruction.qhelp b/ruby/ql/src/queries/security/cwe-078/UnsafeShellCommandConstruction.qhelp index 4231f7cb0b4..8cedb57c277 100644 --- a/ruby/ql/src/queries/security/cwe-078/UnsafeShellCommandConstruction.qhelp +++ b/ruby/ql/src/queries/security/cwe-078/UnsafeShellCommandConstruction.qhelp @@ -20,10 +20,15 @@

- If possible, provide the dynamic arguments to the shell as an array + If possible, avoid concatenating shell strings to APIs such as system(..) to avoid interpretation by the shell.

+

+ Instead, provide the arguments to the shell command as separate arguments to the + API, such as system("echo", arg1, arg2). +

+

Alternatively, if the shell command must be constructed dynamically, then add code to ensure that special characters diff --git a/ruby/ql/src/queries/security/cwe-078/examples/unsafe-shell-command-construction_fixed.rb b/ruby/ql/src/queries/security/cwe-078/examples/unsafe-shell-command-construction_fixed.rb index cb8730eee09..b055f94eba9 100644 --- a/ruby/ql/src/queries/security/cwe-078/examples/unsafe-shell-command-construction_fixed.rb +++ b/ruby/ql/src/queries/security/cwe-078/examples/unsafe-shell-command-construction_fixed.rb @@ -1,6 +1,6 @@ module Utils def download(path) - # using an array to call `system` is safe + # using an API that doesn't interpret the path as a shell command system("wget", path) # OK end end \ No newline at end of file