use new meaning of super keyword

Consider boxed booleans to avoid false positives for XXE.ql

.github/workflows/check-change-note.yml

@@ -1,3 +1,5 @@
+name: Check change note
+
 on:
   pull_request_target:
     types: [labeled, unlabeled, opened, synchronize, reopened, ready_for_review]

.github/workflows/close-stale.yml

@@ -0,0 +1,30 @@
+name: Mark stale issues
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "30 1 * * *"
+
+jobs:
+  stale:
+    if: github.repository == 'github/codeql'
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'This issue is stale because it has been open 14 days with no activity. Comment or remove the `Stale` label in order to avoid having this issue closed in 7 days.'
+        close-issue-message: 'This issue was closed because it has been inactive for 7 days.'
+        days-before-stale: 14
+        days-before-close: 7
+        only-labels: awaiting-response
+
+        # do not mark PRs as stale
+        days-before-pr-stale: -1
+        days-before-pr-close: -1
+
+        # Uncomment for dry-run
+        # debug-only: true
+        # operations-per-run: 1000

.github/workflows/codeql-analysis.yml

@@ -19,13 +19,18 @@ jobs:
 
     runs-on: ubuntu-latest
 
+    permissions:
+      contents: read
+      security-events: write
+      pull-requests: read
+
     steps:
     - name: Checkout repository
       uses: actions/checkout@v2
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@main
       # Override language selection by uncommenting this and choosing your languages
       with:
         languages: csharp
@@ -34,7 +39,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@main
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -48,4 +53,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@main

.vscode/settings.json

@@ -1,3 +1,3 @@
 {
     "omnisharp.autoStart": false
-}
+}

CODEOWNERS

@@ -10,3 +10,10 @@
 /java/**/experimental/**/* @github/codeql-java @xcorail
 /javascript/**/experimental/**/* @github/codeql-javascript @xcorail
 /python/**/experimental/**/* @github/codeql-python @xcorail
+
+# Notify members of codeql-go about PRs to the shared data-flow library files
+/java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl.qll @github/codeql-java @github/codeql-go
+/java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl2.qll @github/codeql-java @github/codeql-go
+/java/ql/src/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll @github/codeql-java @github/codeql-go
+/java/ql/src/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll @github/codeql-java @github/codeql-go
+/java/ql/src/semmle/code/java/dataflow/internal/tainttracking2/TaintTrackingImpl.qll @github/codeql-java @github/codeql-go

CONTRIBUTING.md

@@ -38,7 +38,7 @@ If you have an idea for a query that you would like to share with other CodeQL u
 
     - The queries and libraries must be autoformatted, for example using the "Format Document" command in [CodeQL for Visual Studio Code](https://help.semmle.com/codeql/codeql-for-vscode/procedures/about-codeql-for-vscode.html).
 
-    If you prefer, you can use this [pre-commit hook](misc/scripts/pre-commit) that automatically checks whether your files are correctly formatted. See the [pre-commit hook installation guide](docs/install-pre-commit-hook.md) for instructions on how to install the hook.
+    If you prefer, you can use this [pre-commit hook](misc/scripts/pre-commit) that automatically checks whether your files are correctly formatted. See the [pre-commit hook installation guide](docs/pre-commit-hook-setup.md) for instructions on how to install the hook.
 
 4. **Compilation**
 

config/identical-files.json

@@ -5,6 +5,7 @@
     "java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl3.qll",
     "java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl4.qll",
     "java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl5.qll",
+    "java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl6.qll",
     "cpp/ql/src/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll",
     "cpp/ql/src/semmle/code/cpp/dataflow/internal/DataFlowImpl2.qll",
     "cpp/ql/src/semmle/code/cpp/dataflow/internal/DataFlowImpl3.qll",
@@ -36,6 +37,7 @@
     "cpp/ql/src/semmle/code/cpp/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
+    "cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
     "csharp/ql/src/semmle/code/csharp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "csharp/ql/src/semmle/code/csharp/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
     "csharp/ql/src/semmle/code/csharp/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
@@ -55,6 +57,10 @@
     "csharp/ql/src/semmle/code/csharp/dataflow/internal/DataFlowImplConsistency.qll",
     "python/ql/src/semmle/python/dataflow/new/internal/DataFlowImplConsistency.qll"
   ],
+  "DataFlow Java/C# Flow Summaries": [
+    "java/ql/src/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll",
+    "csharp/ql/src/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll"
+  ],
   "SsaReadPosition Java/C#": [
     "java/ql/src/semmle/code/java/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll",
     "csharp/ql/src/semmle/code/csharp/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll"
@@ -374,50 +380,49 @@
     "javascript/ql/src/semmle/javascript/XML.qll",
     "python/ql/src/semmle/python/xml/XML.qll"
   ],
-  "DuplicationProblems.qhelp": [
-    "cpp/ql/src/Metrics/Files/DuplicationProblems.qhelp",
-    "csharp/ql/src/Metrics/Files/DuplicationProblems.qhelp",
-    "javascript/ql/src/Metrics/DuplicationProblems.qhelp",
-    "python/ql/src/Metrics/DuplicationProblems.qhelp"
+  "DuplicationProblems.inc.qhelp": [
+    "cpp/ql/src/Metrics/Files/DuplicationProblems.inc.qhelp",
+    "javascript/ql/src/Metrics/DuplicationProblems.inc.qhelp",
+    "python/ql/src/Metrics/DuplicationProblems.inc.qhelp"
   ],
-  "CommentedOutCodeQuery.qhelp": [
-    "cpp/ql/src/Documentation/CommentedOutCodeQuery.qhelp",
-    "python/ql/src/Lexical/CommentedOutCodeQuery.qhelp",
-    "csharp/ql/src/Bad Practices/Comments/CommentedOutCodeQuery.qhelp",
-    "java/ql/src/Violations of Best Practice/Comments/CommentedOutCodeQuery.qhelp",
-    "javascript/ql/src/Comments/CommentedOutCodeQuery.qhelp"
+  "CommentedOutCodeQuery.inc.qhelp": [
+    "cpp/ql/src/Documentation/CommentedOutCodeQuery.inc.qhelp",
+    "python/ql/src/Lexical/CommentedOutCodeQuery.inc.qhelp",
+    "csharp/ql/src/Bad Practices/Comments/CommentedOutCodeQuery.inc.qhelp",
+    "java/ql/src/Violations of Best Practice/Comments/CommentedOutCodeQuery.inc.qhelp",
+    "javascript/ql/src/Comments/CommentedOutCodeQuery.inc.qhelp"
   ],
-  "FLinesOfCodeReferences.qhelp": [
-    "java/ql/src/Metrics/Files/FLinesOfCodeReferences.qhelp",
-    "javascript/ql/src/Metrics/FLinesOfCodeReferences.qhelp"
+  "FLinesOfCodeReferences.inc.qhelp": [
+    "java/ql/src/Metrics/Files/FLinesOfCodeReferences.inc.qhelp",
+    "javascript/ql/src/Metrics/FLinesOfCodeReferences.inc.qhelp"
   ],
-  "FCommentRatioCommon.qhelp": [
-    "java/ql/src/Metrics/Files/FCommentRatioCommon.qhelp",
-    "javascript/ql/src/Metrics/FCommentRatioCommon.qhelp"
+  "FCommentRatioCommon.inc.qhelp": [
+    "java/ql/src/Metrics/Files/FCommentRatioCommon.inc.qhelp",
+    "javascript/ql/src/Metrics/FCommentRatioCommon.inc.qhelp"
   ],
-  "FLinesOfCodeOverview.qhelp": [
-    "java/ql/src/Metrics/Files/FLinesOfCodeOverview.qhelp",
-    "javascript/ql/src/Metrics/FLinesOfCodeOverview.qhelp"
+  "FLinesOfCodeOverview.inc.qhelp": [
+    "java/ql/src/Metrics/Files/FLinesOfCodeOverview.inc.qhelp",
+    "javascript/ql/src/Metrics/FLinesOfCodeOverview.inc.qhelp"
   ],
-  "CommentedOutCodeMetricOverview.qhelp": [
-    "cpp/ql/src/Metrics/Files/CommentedOutCodeMetricOverview.qhelp",
-    "csharp/ql/src/Metrics/Files/CommentedOutCodeMetricOverview.qhelp",
-    "java/ql/src/Metrics/Files/CommentedOutCodeMetricOverview.qhelp",
-    "javascript/ql/src/Comments/CommentedOutCodeMetricOverview.qhelp",
-    "python/ql/src/Lexical/CommentedOutCodeMetricOverview.qhelp"
+  "CommentedOutCodeMetricOverview.inc.qhelp": [
+    "cpp/ql/src/Metrics/Files/CommentedOutCodeMetricOverview.inc.qhelp",
+    "csharp/ql/src/Metrics/Files/CommentedOutCodeMetricOverview.inc.qhelp",
+    "java/ql/src/Metrics/Files/CommentedOutCodeMetricOverview.inc.qhelp",
+    "javascript/ql/src/Comments/CommentedOutCodeMetricOverview.inc.qhelp",
+    "python/ql/src/Lexical/CommentedOutCodeMetricOverview.inc.qhelp"
   ],
-  "FLinesOfDuplicatedCodeCommon.qhelp": [
-    "cpp/ql/src/Metrics/Files/FLinesOfDuplicatedCodeCommon.qhelp",
-    "java/ql/src/Metrics/Files/FLinesOfDuplicatedCodeCommon.qhelp",
-    "javascript/ql/src/Metrics/FLinesOfDuplicatedCodeCommon.qhelp",
-    "python/ql/src/Metrics/FLinesOfDuplicatedCodeCommon.qhelp"
+  "FLinesOfDuplicatedCodeCommon.inc.qhelp": [
+    "cpp/ql/src/Metrics/Files/FLinesOfDuplicatedCodeCommon.inc.qhelp",
+    "java/ql/src/Metrics/Files/FLinesOfDuplicatedCodeCommon.inc.qhelp",
+    "javascript/ql/src/Metrics/FLinesOfDuplicatedCodeCommon.inc.qhelp",
+    "python/ql/src/Metrics/FLinesOfDuplicatedCodeCommon.inc.qhelp"
   ],
-  "CommentedOutCodeReferences.qhelp": [
-    "cpp/ql/src/Metrics/Files/CommentedOutCodeReferences.qhelp",
-    "csharp/ql/src/Metrics/Files/CommentedOutCodeReferences.qhelp",
-    "java/ql/src/Metrics/Files/CommentedOutCodeReferences.qhelp",
-    "javascript/ql/src/Comments/CommentedOutCodeReferences.qhelp",
-    "python/ql/src/Lexical/CommentedOutCodeReferences.qhelp"
+  "CommentedOutCodeReferences.inc.qhelp": [
+    "cpp/ql/src/Metrics/Files/CommentedOutCodeReferences.inc.qhelp",
+    "csharp/ql/src/Metrics/Files/CommentedOutCodeReferences.inc.qhelp",
+    "java/ql/src/Metrics/Files/CommentedOutCodeReferences.inc.qhelp",
+    "javascript/ql/src/Comments/CommentedOutCodeReferences.inc.qhelp",
+    "python/ql/src/Lexical/CommentedOutCodeReferences.inc.qhelp"
   ],
   "IDE Contextual Queries": [
     "cpp/ql/src/IDEContextual.qll",
@@ -429,6 +434,15 @@
   "SSA C#": [
     "csharp/ql/src/semmle/code/csharp/dataflow/internal/SsaImplCommon.qll",
     "csharp/ql/src/semmle/code/csharp/controlflow/internal/pressa/SsaImplCommon.qll",
-    "csharp/ql/src/semmle/code/csharp/dataflow/internal/basessa/SsaImplCommon.qll"
+    "csharp/ql/src/semmle/code/csharp/dataflow/internal/basessa/SsaImplCommon.qll",
+    "csharp/ql/src/semmle/code/cil/internal/SsaImplCommon.qll"
+  ],
+  "CryptoAlgorithms Python/JS": [
+    "javascript/ql/src/semmle/javascript/security/CryptoAlgorithms.qll",
+    "python/ql/src/semmle/crypto/Crypto.qll"
+  ],
+  "SensitiveDataHeuristics Python/JS": [
+    "javascript/ql/src/semmle/javascript/security/internal/SensitiveDataHeuristics.qll",
+    "python/ql/src/semmle/python/security/internal/SensitiveDataHeuristics.qll"
   ]
-}
+}

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -5,6 +5,7 @@ using System;
 using System.Linq;
 using Microsoft.Build.Construction;
 using System.Xml;
+using System.IO;
 
 namespace Semmle.Autobuild.Cpp.Tests
 {
@@ -43,6 +44,8 @@ namespace Semmle.Autobuild.Cpp.Tests
         public IDictionary<string, int> RunProcess = new Dictionary<string, int>();
         public IDictionary<string, string> RunProcessOut = new Dictionary<string, string>();
         public IDictionary<string, string> RunProcessWorkingDirectory = new Dictionary<string, string>();
+        public HashSet<string> CreateDirectories { get; } = new HashSet<string>();
+        public HashSet<(string, string)> DownloadFiles { get; } = new HashSet<(string, string)>();
 
         int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, out IList<string> stdOut)
         {
@@ -135,6 +138,14 @@ namespace Semmle.Autobuild.Cpp.Tests
 
         string IBuildActions.GetFullPath(string path) => path;
 
+        string? IBuildActions.GetFileName(string? path) => Path.GetFileName(path?.Replace('\\', '/'));
+
+        public string? GetDirectoryName(string? path)
+        {
+            var dir = Path.GetDirectoryName(path?.Replace('\\', '/'));
+            return dir is null ? path : path?.Substring(0, dir.Length);
+        }
+
         void IBuildActions.WriteAllText(string filename, string contents)
         {
         }
@@ -153,6 +164,18 @@ namespace Semmle.Autobuild.Cpp.Tests
                 s = s.Replace($"%{kvp.Key}%", kvp.Value);
             return s;
         }
+
+        public void CreateDirectory(string path)
+        {
+            if (!CreateDirectories.Contains(path))
+                throw new ArgumentException($"Missing CreateDirectory, {path}");
+        }
+
+        public void DownloadFile(string address, string fileName)
+        {
+            if (!DownloadFiles.Contains((address, fileName)))
+                throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");
+        }
     }
 
     /// <summary>
@@ -213,6 +236,7 @@ namespace Semmle.Autobuild.Cpp.Tests
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
             Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
+            Actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = "win64";
             Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";
             Actions.GetEnvironmentVariable["SEMMLE_JAVA_HOME"] = @"C:\odasa\tools\java";
             Actions.GetEnvironmentVariable["SEMMLE_PLATFORM_TOOLS"] = @"C:\odasa\tools";
@@ -273,7 +297,8 @@ namespace Semmle.Autobuild.Cpp.Tests
         [Fact]
         public void TestCppAutobuilderSuccess()
         {
-            Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\csharp\nuget\nuget.exe restore C:\Project\test.sln"] = 1;
+            Actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test.sln -DisableParallelProcessing"] = 1;
+            Actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test.sln -DisableParallelProcessing"] = 0;
             Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && C:\odasa\tools\odasa index --auto msbuild C:\Project\test.sln /p:UseSharedCompilation=false /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"" /p:MvcBuildViews=true"] = 0;
             Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = "";
             Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 1;
@@ -286,11 +311,13 @@ namespace Semmle.Autobuild.Cpp.Tests
             Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = true;
             Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.slx";
             Actions.EnumerateDirectories[@"C:\Project"] = "";
+            Actions.CreateDirectories.Add(@"C:\Project\.nuget");
+            Actions.DownloadFiles.Add(("https://dist.nuget.org/win-x86-commandline/latest/nuget.exe", @"C:\Project\.nuget\nuget.exe"));
 
             var autobuilder = CreateAutoBuilder(true);
             var solution = new TestSolution(@"C:\Project\test.sln");
             autobuilder.ProjectsOrSolutionsToBuild.Add(solution);
-            TestAutobuilderScript(autobuilder, 0, 2);
+            TestAutobuilderScript(autobuilder, 0, 3);
         }
     }
 }

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/Semmle.Autobuild.Cpp.Tests.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
     <Nullable>enable</Nullable>

cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
     <AssemblyName>Semmle.Autobuild.Cpp</AssemblyName>
     <RootNamespace>Semmle.Autobuild.Cpp</RootNamespace>
     <ApplicationIcon />
@@ -17,7 +17,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Build" Version="16.0.461" />
+    <PackageReference Include="Microsoft.Build" Version="16.9.0" />
   </ItemGroup>
 
   <ItemGroup>

cpp/change-notes/2021-03-01-fluent-interface-data-flow.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* The data-flow library now recognises more side-effects of method chaining (e.g. `someObject.setX(clean).setY(tainted).setZ...` having a side-effect on `someObject`), as well as other related circumstances where a function input is directly passed to its output. All queries that use data-flow analysis, including most security queries, may return more results accordingly.

cpp/change-notes/2021-03-11-failed-extractions.md

@@ -0,0 +1,2 @@
+codescanning
+* Added cpp/diagnostics/failed-extractions. This query gives information about which extractions did not run to completion.

cpp/change-notes/2021-03-11-overflow-abs.md

@@ -0,0 +1,2 @@
+lgtm
+* The `cpp/tainted-arithmetic`, `cpp/arithmetic-with-extreme-values`, and `cpp/uncontrolled-arithmetic` queries now recognize more functions as returning the absolute value of their input. As a result, they produce fewer false positives.

cpp/change-notes/2021-03-17-av-rule-79.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* The 'Resource not released in destructor' (cpp/resource-not-released-in-destructor) query has been improved to recognize more releases of resources.

cpp/change-notes/2021-04-06-assign-where-compare-meant.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* The 'Assignment where comparison was intended' (cpp/assign-where-compare-meant) query has been improved to flag fewer benign assignments in conditionals.

cpp/change-notes/2021-04-09-unsigned-difference-expression-compared-zero.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* The 'Unsigned difference expression compared to zero' (cpp/unsigned-difference-expression-compared-zero) query has been improved to produce fewer false positive results.

cpp/change-notes/2021-04-13-arithmetic-queries.md

@@ -0,0 +1,2 @@
+lgtm
+* The queries cpp/tainted-arithmetic, cpp/uncontrolled-arithmetic, and cpp/arithmetic-with-extreme-values have been improved to produce fewer false positives.

cpp/change-notes/2021-04-21-return-stack-allocated-object.md

@@ -0,0 +1,2 @@
+codescanning
+* The 'Pointer to stack object used as return value' (cpp/return-stack-allocated-object) query has been deprecated, and any uses should be replaced with `Returning stack-allocated memory` (cpp/return-stack-allocated-memory).

cpp/change-notes/2021-10-05-comparison-with-wider-type.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* The 'Comparison with wider type' (cpp/comparison-with-wider-type) query has been improved to produce fewer false positives.

cpp/change-notes/2021-26-04-more-sound-expr-might-overflow.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* The `exprMightOverflowPositively` and `exprMightOverflowNegatively` predicates from the `SimpleRangeAnalysis` library now recognize more expressions that might overflow.

cpp/ql/src/Best Practices/Magic Constants/MagicConstantsNumbers.qhelp

@@ -39,7 +39,7 @@ then replace all the relevant occurrences in the code.</p>
 </li>
 <li>
   Mats Henricson and Erik Nyquist, <i>Industrial Strength C++</i>, published by Prentice Hall PTR (1997). 
-  Chapter 5: Object Life Cycle, Rec 5.4 (<a href="http://mongers.org/industrial-c++/">PDF</a>).
+  Chapter 5: Object Life Cycle, Rec 5.4 (<a href="https://web.archive.org/web/20190919025638/https://mongers.org/industrial-c++/">PDF</a>).
 </li>
 <li>
   <a href="https://www.securecoding.cert.org/confluence/display/c/DCL06-C.+Use+meaningful+symbolic+constants+to+represent+literal+values">DCL06-C. Use meaningful symbolic constants to represent literal values</a>

cpp/ql/src/Best Practices/Magic Constants/MagicConstantsString.qhelp

@@ -38,7 +38,7 @@ constant.</p>
 </li>
 <li>
   Mats Henricson and Erik Nyquist, <i>Industrial Strength C++</i>, published by Prentice Hall PTR (1997). 
-  Chapter 5: Object Life Cycle, Rec 5.4 (<a href="http://mongers.org/industrial-c++/">PDF</a>).
+  Chapter 5: Object Life Cycle, Rec 5.4 (<a href="https://web.archive.org/web/20190919025638/https://mongers.org/industrial-c++/">PDF</a>).
 </li>
 <li>
   <a href="https://www.securecoding.cert.org/confluence/display/c/DCL06-C.+Use+meaningful+symbolic+constants+to+represent+literal+values">DCL06-C. Use meaningful symbolic constants to represent literal values</a>

cpp/ql/src/Best Practices/SloppyGlobal.qhelp

@@ -21,7 +21,7 @@ Review the purpose of the each global variable flagged by this rule and update e
 
 <li>
   Mats Henricson and Erik Nyquist, <i>Industrial Strength C++</i>, published by Prentice Hall PTR (1997). 
-  Chapter 1: Naming, Rec 1.1 (<a href="http://mongers.org/industrial-c++/">PDF</a>).
+  Chapter 1: Naming, Rec 1.1 (<a href="https://web.archive.org/web/20190919025638/https://mongers.org/industrial-c++/">PDF</a>).
 </li>
 <li>
   <a href="http://www.learncpp.com/cpp-tutorial/42-global-variables/">Global variables</a>.

cpp/ql/src/Best Practices/UseOfGoto.qhelp

@@ -45,7 +45,7 @@ this rule.
 </li>
 <li>
   Mats Henricson and Erik Nyquist, <i>Industrial Strength C++</i>, Rule 4.6. Prentice Hall PTR, 1997.
-  (<a href="http://mongers.org/industrial-c++/">PDF</a>).
+  (<a href="https://web.archive.org/web/20190919025638/https://mongers.org/industrial-c++/">PDF</a>).
 </li>
 <li>
   cplusplus.com: <a href="http://www.cplusplus.com/doc/tutorial/control/">Control Structures</a>.

cpp/ql/src/Critical/DeadCodeCondition.qhelp

@@ -9,7 +9,7 @@
 It is likely that these conditions indicate an error in the branching condition. 
 Alternatively, the conditions may have been left behind after debugging.</p>
 
-<include src="aliasAnalysisWarning.qhelp" />
+<include src="aliasAnalysisWarning.inc.qhelp" />
 </overview>
 
 <recommendation>

cpp/ql/src/Critical/DeadCodeFunction.qhelp

@@ -13,7 +13,7 @@ If left in the code base they increase object code size, decrease code comprehen
 This type of function may be part of the program's API and could be used by external programs.
 </p>
 
-<include src="callGraphWarning.qhelp" />
+<include src="callGraphWarning.inc.qhelp" />
 </overview>
 
 <recommendation>

cpp/ql/src/Critical/DescriptorMayNotBeClosed.qhelp

@@ -10,7 +10,7 @@ This query looks at functions that return file or socket descriptors, but may re
 This can occur when an operation performed on the open descriptor fails, and the function returns with an error before it closes the open resource. An improperly handled error could cause the function to leak resource descriptors. Failing to close resources in the function that opened them also makes it more difficult to detect leaks.
 </p> 
 
-<include src="dataFlowWarning.qhelp" />
+<include src="dataFlowWarning.inc.qhelp" />
 </overview>
 
 <recommendation>

cpp/ql/src/Critical/DescriptorNeverClosed.qhelp

@@ -10,7 +10,7 @@ This rule finds calls to <code>socket</code> where there is no corresponding <co
 Leaving descriptors open will cause a resource leak that will persist even after the program terminates.
 </p>
 
-<include src="aliasAnalysisWarning.qhelp" />
+<include src="aliasAnalysisWarning.inc.qhelp" />
 </overview>
 
 <recommendation>

cpp/ql/src/Critical/FileMayNotBeClosed.qhelp

@@ -10,7 +10,7 @@ This rule looks at functions that return a <code>FILE*</code>, but may return an
 This can occur when an operation performed on the open descriptor fails, and the function returns with an error before closing the open resource. An improperly handled error may cause the function to leak file descriptors.
 </p> 
 
-<include src="dataFlowWarning.qhelp" />
+<include src="dataFlowWarning.inc.qhelp" />
 
 </overview>
 <recommendation>

cpp/ql/src/Critical/FileNeverClosed.qhelp

@@ -10,7 +10,7 @@ This rule finds calls to <code>fopen</code> with no corresponding <code>fclose</
 Leaving files open will cause a resource leak that will persist even after the program terminates.
 </p>
 
-<include src="aliasAnalysisWarning.qhelp" />
+<include src="aliasAnalysisWarning.inc.qhelp" />
 
 </overview>
 <recommendation>

cpp/ql/src/Critical/GlobalUseBeforeInit.qhelp

@@ -10,7 +10,7 @@ Not all compilers generate code that zero-out memory, especially when optimizati
 is not compliant with the latest language standards. Accessing uninitialized memory will lead to undefined results.
 </p>
 
-<include src="dataFlowWarning.qhelp" />
+<include src="dataFlowWarning.inc.qhelp" />
 </overview>
 
 <recommendation>

cpp/ql/src/Critical/InconsistentNullnessTesting.qhelp

@@ -12,7 +12,7 @@ Dereferencing a null pointer and attempting to modify its contents can lead to a
 important system data (including the interrupt table in some architectures).
 </p>
 
-<include src="pointsToWarning.qhelp" />
+<include src="pointsToWarning.inc.qhelp" />
 </overview>
 
 <recommendation>

cpp/ql/src/Critical/InitialisationNotRun.qhelp

@@ -11,7 +11,7 @@ Uninitialized variables may contain any value, as not all compilers generate cod
 optimizations are enabled or the compiler is not compliant with the latest language standards.
 </p>
 
-<include src="callGraphWarning.qhelp" />
+<include src="callGraphWarning.inc.qhelp" />
 </overview>
 
 <recommendation>

cpp/ql/src/Critical/LateNegativeTest.qhelp

@@ -13,7 +13,7 @@ after. Otherwise, if the value is negative then the program will have failed
 before performing the test.
 </p>
 
-<include src="dataFlowWarning.qhelp" />
+<include src="dataFlowWarning.inc.qhelp" />
 </overview>
 
 <recommendation>

cpp/ql/src/Critical/MemoryMayNotBeFreed.qhelp

@@ -9,7 +9,7 @@
 This rule looks for functions that allocate memory, but may return without freeing it.  This can occur when an operation performed on the memory block fails, and the function returns with an error before freeing the allocated block.  This causes the function to leak memory and may eventually lead to software failure.
 </p> 
 
-<include src="dataFlowWarning.qhelp" />
+<include src="dataFlowWarning.inc.qhelp" />
 
 </overview>
 <recommendation>

cpp/ql/src/Critical/MemoryNeverFreed.qhelp

@@ -10,7 +10,7 @@ This rule finds calls to the <code>alloc</code> family of functions without a co
 This leads to memory leaks.
 </p>
 
-<include src="aliasAnalysisWarning.qhelp" />
+<include src="aliasAnalysisWarning.inc.qhelp" />
 
 </overview>
 <recommendation>

cpp/ql/src/Critical/MissingNegativityTest.qhelp

@@ -16,7 +16,7 @@ buffer overruns.
 The query looks only at the return values of functions that may return a negative value (not all functions).
 </p>
 
-<include src="dataFlowWarning.qhelp" />
+<include src="dataFlowWarning.inc.qhelp" />
 </overview>
 
 <recommendation>

cpp/ql/src/Critical/NewArrayDeleteMismatch.qhelp

@@ -63,7 +63,7 @@ destructors likely not be called (as previously noted), but the pointer will als
 potentially less of a serious issue than that posed by the first approach, but it should still be avoided.</li>
 </ul>
 
-<include src="pointsToWarning.qhelp" />
+<include src="pointsToWarning.inc.qhelp" />
 
 </overview>
 <recommendation>

cpp/ql/src/Critical/NewDeleteArrayMismatch.qhelp

@@ -18,7 +18,7 @@ an array (which could have header data specifying the length of the array) and w
 element of the 'array', which would likely lead to a segfault due to the invalid header data.
 </p>
 
-<include src="pointsToWarning.qhelp" />
+<include src="pointsToWarning.inc.qhelp" />
 
 </overview>
 <recommendation>

cpp/ql/src/Critical/OverflowCalculated.qhelp

@@ -19,7 +19,7 @@ the data being copied. Buffer overflows can result to anything from a segmentati
 if the array is on stack-allocated memory).
 </p>
 
-<include src="aliasAnalysisWarning.qhelp" />
+<include src="aliasAnalysisWarning.inc.qhelp" />
 </overview>
 
 <recommendation>

cpp/ql/src/Critical/OverflowDestination.qhelp

@@ -14,7 +14,7 @@ Buffer overflows can lead to anything from a segmentation fault to a security vu
 Ensure that the size parameter is derived from the size of the destination buffer, and 
 not the source buffer.</p>
 
-<include src="aliasAnalysisWarning.qhelp" />
+<include src="aliasAnalysisWarning.inc.qhelp" />
 </recommendation>
 
 

cpp/ql/src/Critical/ReturnStackAllocatedObject.qhelp

@@ -12,7 +12,7 @@ the contents of that memory become undefined after that. Clearly, using a pointe
 memory after the function has already returned will have undefined results.
 </p>
 
-<include src="pointsToWarning.qhelp" />
+<include src="pointsToWarning.inc.qhelp" />
 </overview>
 
 <recommendation>

cpp/ql/src/Critical/ReturnStackAllocatedObject.ql

@@ -7,6 +7,8 @@
  * @tags reliability
  *       security
  *       external/cwe/cwe-562
+ * @deprecated This query is not suitable for production use and has been deprecated. Use
+ *             cpp/return-stack-allocated-memory instead.
  */
 
 import semmle.code.cpp.pointsto.PointsTo

cpp/ql/src/Critical/ReturnValueIgnored.qhelp

@@ -32,7 +32,7 @@ Check the return value of functions that return status information.
 <references>
 
 <li>
-  M. Henricson and E. Nyquist, <i>Industrial Strength C++</i>, Chapter 12: Error handling. Prentice Hall PTR, 1997 (<a href="http://mongers.org/industrial-c++/">available online</a>).
+  M. Henricson and E. Nyquist, <i>Industrial Strength C++</i>, Chapter 12: Error handling. Prentice Hall PTR, 1997 (<a href="https://web.archive.org/web/20190919025638/https://mongers.org/industrial-c++/">available online</a>).
 </li>
 <li>
   The CERT C Secure Coding Standard: <a href="https://www.securecoding.cert.org/confluence/display/perl/EXP32-PL.+Do+not+ignore+function+return+values">EXP32-PL. Do not ignore function return values</a>.

cpp/ql/src/Critical/UseAfterFree.qhelp

@@ -12,7 +12,7 @@ from a segfault to memory corruption that would cause subsequent calls to the dy
 erratically, to a possible security vulnerability.
 </p>
 
-<include src="pointsToWarning.qhelp" />
+<include src="pointsToWarning.inc.qhelp" />
 
 </overview>
 <recommendation>

cpp/ql/src/Diagnostics/ExtractionErrors.ql

@@ -0,0 +1,16 @@
+/**
+ * @name Extraction errors
+ * @description List all extraction errors for files in the source code directory.
+ * @kind diagnostic
+ * @id cpp/diagnostics/extraction-errors
+ */
+
+import cpp
+import ExtractionErrors
+
+from ExtractionError error
+where
+  error instanceof ExtractionUnknownError or
+  exists(error.getFile().getRelativePath())
+select error, "Extraction failed in " + error.getFile() + " with error " + error.getErrorMessage(),
+  error.getSeverity()

cpp/ql/src/Diagnostics/ExtractionErrors.qll

@@ -0,0 +1,137 @@
+/**
+ * Provides a common hierarchy of all types of errors that can occur during extraction.
+ */
+
+import cpp
+
+/*
+ * A note about how the C/C++ extractor emits diagnostics:
+ * When the extractor frontend encounters an error, it emits a diagnostic message,
+ * that includes a message, location and severity.
+ * However, that process is best-effort and may fail (e.g. due to lack of memory).
+ * Thus, if the extractor emitted at least one diagnostic of severity discretionary
+ * error (or higher), it *also* emits a simple "There was an error during this compilation"
+ * error diagnostic, without location information.
+ * In the common case, this means that a compilation during which one or more errors happened also gets
+ * the catch-all diagnostic.
+ * This diagnostic has the empty string as file path.
+ * We filter out these useless diagnostics if there is at least one error-level diagnostic
+ * for the affected compilation in the database.
+ * Otherwise, we show it to indicate that something went wrong and that we
+ * don't know what exactly happened.
+ */
+
+/**
+ * An error that, if present, leads to a file being marked as non-successfully extracted.
+ */
+class ReportableError extends Diagnostic {
+  ReportableError() {
+    (
+      this instanceof CompilerDiscretionaryError or
+      this instanceof CompilerError or
+      this instanceof CompilerCatastrophe
+    ) and
+    // Filter for the catch-all diagnostic, see note above.
+    not this.getFile().getAbsolutePath() = ""
+  }
+}
+
+private newtype TExtractionError =
+  TReportableError(ReportableError err) or
+  TCompilationFailed(Compilation c, File f) {
+    f = c.getAFileCompiled() and not c.normalTermination()
+  } or
+  // Show the catch-all diagnostic (see note above) only if we haven't seen any other error-level diagnostic
+  // for that compilation
+  TUnknownError(CompilerError err) {
+    not exists(ReportableError e | e.getCompilation() = err.getCompilation())
+  }
+
+/**
+ * Superclass for the extraction error hierarchy.
+ */
+class ExtractionError extends TExtractionError {
+  /** Gets the string representation of the error. */
+  string toString() { none() }
+
+  /** Gets the error message for this error. */
+  string getErrorMessage() { none() }
+
+  /** Gets the file this error occured in. */
+  File getFile() { none() }
+
+  /** Gets the location this error occured in. */
+  Location getLocation() { none() }
+
+  /** Gets the SARIF severity of this error. */
+  int getSeverity() {
+    // Unfortunately, we can't distinguish between errors and fatal errors in SARIF,
+    // so all errors have severity 2.
+    result = 2
+  }
+}
+
+/**
+ * An unrecoverable extraction error, where extraction was unable to finish.
+ * This can be caused by a multitude of reasons, for example:
+ *  - hitting a frontend assertion
+ *  - crashing due to dereferencing an invalid pointer
+ *  - stack overflow
+ *  - out of memory
+ */
+class ExtractionUnrecoverableError extends ExtractionError, TCompilationFailed {
+  Compilation c;
+  File f;
+
+  ExtractionUnrecoverableError() { this = TCompilationFailed(c, f) }
+
+  override string toString() {
+    result = "Unrecoverable extraction error while compiling " + f.toString()
+  }
+
+  override string getErrorMessage() { result = "unrecoverable compilation failure." }
+
+  override File getFile() { result = f }
+
+  override Location getLocation() { result = f.getLocation() }
+}
+
+/**
+ * A recoverable extraction error.
+ * These are compiler errors from the frontend.
+ * Upon encountering one of these, we still continue extraction, but the
+ * database will be incomplete for that file.
+ */
+class ExtractionRecoverableError extends ExtractionError, TReportableError {
+  ReportableError err;
+
+  ExtractionRecoverableError() { this = TReportableError(err) }
+
+  override string toString() { result = "Recoverable extraction error: " + err }
+
+  override string getErrorMessage() { result = err.getFullMessage() }
+
+  override File getFile() { result = err.getFile() }
+
+  override Location getLocation() { result = err.getLocation() }
+}
+
+/**
+ * An unknown error happened during extraction.
+ * These are only displayed if we know that we encountered an error during extraction,
+ * but, for some reason, failed to emit a proper diagnostic with location information
+ * and error message.
+ */
+class ExtractionUnknownError extends ExtractionError, TUnknownError {
+  CompilerError err;
+
+  ExtractionUnknownError() { this = TUnknownError(err) }
+
+  override string toString() { result = "Unknown extraction error: " + err }
+
+  override string getErrorMessage() { result = err.getFullMessage() }
+
+  override File getFile() { result = err.getFile() }
+
+  override Location getLocation() { result = err.getLocation() }
+}

cpp/ql/src/Diagnostics/FailedExtractorInvocations.ql

@@ -0,0 +1,22 @@
+/**
+ * @name Failed extractor invocations
+ * @description Gives the command line of compilations for which extraction did not run to completion.
+ * @kind diagnostic
+ * @id cpp/diagnostics/failed-extractor-invocations
+ */
+
+import cpp
+
+class AnonymousCompilation extends Compilation {
+  override string toString() { result = "<compilation>" }
+}
+
+string describe(Compilation c) {
+  if c.getArgument(1) = "--mimic"
+  then result = "compiler invocation " + concat(int i | i > 1 | c.getArgument(i), " " order by i)
+  else result = "extractor invocation " + concat(int i | | c.getArgument(i), " " order by i)
+}
+
+from Compilation c
+where not c.normalTermination()
+select c, "Extraction aborted for " + describe(c), 2

cpp/ql/src/Diagnostics/SuccessfullyExtractedFiles.ql

@@ -0,0 +1,15 @@
+/**
+ * @name Successfully extracted files
+ * @description Lists all files in the source code directory that were extracted without encountering an error in the file.
+ * @kind diagnostic
+ * @id cpp/diagnostics/successfully-extracted-files
+ */
+
+import cpp
+import ExtractionErrors
+
+from File f
+where
+  not exists(ExtractionError e | e.getFile() = f) and
+  exists(f.getRelativePath())
+select f, ""

cpp/ql/src/Documentation/CommentedOutCode.qhelp

@@ -2,6 +2,6 @@
   "-//Semmle//qhelp//EN"
   "qhelp.dtd">
 <qhelp>
-  <include src="CommentedOutCodeQuery.qhelp" />
-  <include src="../Metrics/Files/CommentedOutCodeReferences.qhelp" />
+  <include src="CommentedOutCodeQuery.inc.qhelp" />
+  <include src="../Metrics/Files/CommentedOutCodeReferences.inc.qhelp" />
 </qhelp>

cpp/ql/src/Likely Bugs/Arithmetic/SignedOverflowCheck.ql

@@ -9,6 +9,8 @@
  * @id cpp/signed-overflow-check
  * @tags correctness
  *       security
+ *       external/cwe/cwe-128
+ *       external/cwe/cwe-190
  */
 
 import cpp

cpp/ql/src/Likely Bugs/Conversion/CastArrayPointerArithmetic.ql

@@ -7,12 +7,12 @@
  * @kind path-problem
  * @problem.severity warning
  * @precision high
+ * @id cpp/upcast-array-pointer-arithmetic
  * @tags correctness
  *       reliability
  *       security
  *       external/cwe/cwe-119
  *       external/cwe/cwe-843
- * @id cpp/upcast-array-pointer-arithmetic
  */
 
 import cpp

cpp/ql/src/Likely Bugs/Format/SnprintfOverflow.ql

@@ -8,6 +8,8 @@
  * @tags reliability
  *       correctness
  *       security
+ *       external/cwe/cwe-190
+ *       external/cwe/cwe-253
  */
 
 import cpp

cpp/ql/src/Likely Bugs/Format/WrongNumberOfFormatArguments.ql

@@ -9,6 +9,7 @@
  * @tags reliability
  *       correctness
  *       security
+ *       external/cwe/cwe-234
  *       external/cwe/cwe-685
  */
 

cpp/ql/src/Likely Bugs/Leap Year/Adding365DaysPerYear.qhelp

@@ -3,7 +3,7 @@
   "qhelp.dtd">
 <qhelp>
 <overview>
-  <include src="LeapYear.qhelp" />
+  <include src="LeapYear.inc.qhelp" />
 
 <p>When performing arithmetic operations on a variable that represents a date, leap years must be taken into account.
 It is not safe to assume that a year is 365 days long.</p>

cpp/ql/src/Likely Bugs/Leap Year/UncheckedLeapYearAfterYearModification.qhelp

@@ -3,7 +3,7 @@
   "qhelp.dtd">
 <qhelp>
 <overview>
-  <include src="LeapYear.qhelp" />
+  <include src="LeapYear.inc.qhelp" />
   
 <p>When performing arithmetic operations on a variable that represents a year, it is important to consider that the resulting value may not be a valid date.</p>
 <p>The typical example is doing simple year arithmetic (i.e. <code>date.year++</code>) without considering if the resulting value will be a valid date or not.</p>

cpp/ql/src/Likely Bugs/Leap Year/UncheckedReturnValueForTimeFunctions.qhelp

@@ -3,7 +3,7 @@
   "qhelp.dtd">
 <qhelp>
 <overview>
-  <include src="LeapYear.qhelp" />
+  <include src="LeapYear.inc.qhelp" />
   
 <p>When using a function that transforms a date structure, and the year on the input argument for the API has been manipulated, it is important to check for the return value of the function to make sure it succeeded.</p>
 <p>Otherwise, the function may have failed, and the output parameter may contain invalid data that can cause any number of problems on the affected system.</p>

cpp/ql/src/Likely Bugs/Leap Year/UnsafeArrayForDaysOfYear.qhelp

@@ -3,7 +3,7 @@
   "qhelp.dtd">
 <qhelp>
 <overview>
-  <include src="LeapYear.qhelp" />
+  <include src="LeapYear.inc.qhelp" />
 
 <p>This query helps to detect when a developer allocates an array or other fixed-length data structure such as <code>std::vector</code> with 365 elements – one for each day of the year.</p>
 <p>Since leap years have 366 days, there will be no allocated element on December 31st at the end of a leap year; which will lead to a buffer overflow on a leap year.</p>

cpp/ql/src/Likely Bugs/Likely Typos/AssignWhereCompareMeant.ql

@@ -54,7 +54,7 @@ class BooleanControllingAssignmentInExpr extends BooleanControllingAssignment {
   override predicate isWhitelisted() {
     this.getConversion().(ParenthesisExpr).isParenthesised()
     or
-    // whitelist this assignment if all comparison operations in the expression that this
+    // Allow this assignment if all comparison operations in the expression that this
     // assignment is part of, are not parenthesized. In that case it seems like programmer
     // is fine with unparenthesized comparison operands to binary logical operators, and
     // the parenthesis around this assignment was used to call it out as an assignment.
@@ -62,6 +62,21 @@ class BooleanControllingAssignmentInExpr extends BooleanControllingAssignment {
     forex(ComparisonOperation op | op = getComparisonOperand*(this.getParent+()) |
       not op.isParenthesised()
     )
+    or
+    // Match a pattern like:
+    // ```
+    // if((a = b) && use_value(a)) { ... }
+    // ```
+    // where the assignment is meant to update the value of `a` before it's used in some other boolean
+    // subexpression that is guarenteed to be evaluate _after_ the assignment.
+    this.isParenthesised() and
+    exists(LogicalAndExpr parent, Variable var, VariableAccess access |
+      var = this.getLValue().(VariableAccess).getTarget() and
+      access = var.getAnAccess() and
+      not access.isUsedAsLValue() and
+      parent.getRightOperand() = access.getParent*() and
+      parent.getLeftOperand() = this.getParent*()
+    )
   }
 }
 

cpp/ql/src/Likely Bugs/Likely Typos/MissingEnumCaseInSwitch.qhelp

@@ -26,7 +26,7 @@ indication that there may be cases unhandled by the <code>switch</code> statemen
   MSDN Library: <a href="https://docs.microsoft.com/en-us/cpp/cpp/switch-statement-cpp">switch statement (C++)</a>
 </li>
 <li>
-  M. Henricson and E. Nyquist, <i>Industrial Strength C++</i>, Chapter 4: Control Flow, Rec 4.5. Prentice Hall PTR, 1997 (<a href="http://mongers.org/industrial-c++/">available online</a>).
+  M. Henricson and E. Nyquist, <i>Industrial Strength C++</i>, Chapter 4: Control Flow, Rec 4.5. Prentice Hall PTR, 1997 (<a href="https://web.archive.org/web/20190919025638/https://mongers.org/industrial-c++/">available online</a>).
 </li>
 
 

cpp/ql/src/Likely Bugs/Memory Management/PointerOverflow.ql

@@ -8,6 +8,7 @@
  * @id cpp/pointer-overflow-check
  * @tags reliability
  *       security
+ *       external/cwe/cwe-758
  */
 
 import cpp

cpp/ql/src/Likely Bugs/Memory Management/ReturnStackAllocatedMemory.ql

@@ -13,6 +13,7 @@
 
 import cpp
 import semmle.code.cpp.dataflow.EscapesTree
+import semmle.code.cpp.models.interfaces.PointerWrapper
 import semmle.code.cpp.dataflow.DataFlow
 
 /**
@@ -39,6 +40,10 @@ predicate hasNontrivialConversion(Expr e) {
     e instanceof ParenthesisExpr
   )
   or
+  // A smart pointer can be stack-allocated while the data it points to is heap-allocated.
+  // So we exclude such "conversions" from this predicate.
+  e = any(PointerWrapper wrapper).getAnUnwrapperFunction().getACallToThisFunction()
+  or
   hasNontrivialConversion(e.getConversion())
 }
 

cpp/ql/src/Likely Bugs/OO/UnsafeUseOfThis.ql

@@ -10,6 +10,7 @@
  * @tags correctness
  *       language-features
  *       security
+ *       external/cwe/cwe-670
  */
 
 import cpp

cpp/ql/src/Likely Bugs/Underspecified Functions/TooFewArguments.ql

@@ -12,6 +12,8 @@
  * @tags correctness
  *       maintainability
  *       security
+ *       external/cwe/cwe-234
+ *       external/cwe/cwe-685
  */
 
 import cpp

cpp/ql/src/Metrics/Dependencies/ExternalDependencies.ql

@@ -5,7 +5,6 @@
  * @kind treemap
  * @treemap.warnOn highValues
  * @metricType externalDependency
- * @precision medium
  * @id cpp/external-dependencies
  * @tags modularity
  */

cpp/ql/src/Metrics/Files/FLinesOfCode.ql

@@ -7,7 +7,6 @@
  * @treemap.warnOn highValues
  * @metricType file
  * @metricAggregate avg sum max
- * @precision very-high
  * @id cpp/lines-of-code-in-files
  * @tags maintainability
  *       complexity

cpp/ql/src/Metrics/Files/FLinesOfCommentedOutCode.qhelp

@@ -2,6 +2,6 @@
   "-//Semmle//qhelp//EN"
   "qhelp.dtd">
 <qhelp>
-  <include src="CommentedOutCodeMetricOverview.qhelp" />
-  <include src="CommentedOutCodeReferences.qhelp" />
+  <include src="CommentedOutCodeMetricOverview.inc.qhelp" />
+  <include src="CommentedOutCodeReferences.inc.qhelp" />
 </qhelp>

cpp/ql/src/Metrics/Files/FLinesOfCommentedOutCode.ql

@@ -5,7 +5,6 @@
  * @treemap.warnOn highValues
  * @metricType file
  * @metricAggregate avg sum max
- * @precision high
  * @id cpp/lines-of-commented-out-code-in-files
  * @tags documentation
  */

cpp/ql/src/Metrics/Files/FLinesOfComments.ql

@@ -7,7 +7,6 @@
  * @treemap.warnOn lowValues
  * @metricType file
  * @metricAggregate avg sum max
- * @precision very-high
  * @id cpp/lines-of-comments-in-files
  * @tags maintainability
  *       documentation

cpp/ql/src/Metrics/Files/FLinesOfDuplicatedCode.qhelp

@@ -2,5 +2,5 @@
   "-//Semmle//qhelp//EN"
   "qhelp.dtd">
 <qhelp>
-<include src="FLinesOfDuplicatedCodeCommon.qhelp" />
+<include src="FLinesOfDuplicatedCodeCommon.inc.qhelp" />
 </qhelp>

cpp/ql/src/Metrics/Files/FLinesOfDuplicatedCode.ql

@@ -8,7 +8,6 @@
  * @treemap.warnOn highValues
  * @metricType file
  * @metricAggregate avg sum max
- * @precision high
  * @id cpp/duplicated-lines-in-files
  * @tags testability
  *       modularity

cpp/ql/src/Metrics/Files/FLinesOfDuplicatedCodeCommon.inc.qhelp

@@ -14,7 +14,7 @@ for a number of reasons.
 </p>
 
 </overview>
-<include src="DuplicationProblems.qhelp" />
+<include src="DuplicationProblems.inc.qhelp" />
 
 <recommendation>
 

cpp/ql/src/Metrics/Files/FNumberOfTests.ql

@@ -5,7 +5,6 @@
  * @treemap.warnOn lowValues
  * @metricType file
  * @metricAggregate avg sum max
- * @precision medium
  * @id cpp/tests-in-files
  * @tags maintainability
  */

cpp/ql/src/Metrics/Files/FTransitiveIncludes.qhelp

@@ -29,7 +29,7 @@ build time: the more included files, the longer the compilation time.</p>
   <a href="http://www.drdobbs.com/cpp/decoupling-c-header-files/212701130">Decoupling C Header Files</a>
 </li>
 <li>
-  <a href="https://wiki.hsr.ch/Prog3/files/overload72-FINAL_DesigningHeaderFiles.pdf">C++ Best Practice -
+  <a href="https://accu.org/journals/overload/14/72/griffiths_1995/">C++ Best Practice -
 Designing Header Files</a>
 </li>
 

cpp/ql/src/Metrics/Files/FTransitiveSourceIncludes.qhelp

@@ -35,7 +35,7 @@ they are contributing to unnecessarily long build times and creating artificial
   <a href="http://www.drdobbs.com/cpp/decoupling-c-header-files/212701130">Decoupling C Header Files</a>
 </li>
 <li>
-  <a href="https://wiki.hsr.ch/Prog3/files/overload72-FINAL_DesigningHeaderFiles.pdf">C++ Best Practice -
+  <a href="https://accu.org/journals/overload/14/72/griffiths_1995/">C++ Best Practice -
 Designing Header Files</a>
 </li>
 </references>

cpp/ql/src/Security/CWE/CWE-190/ComparisonWithWiderType.ql

@@ -49,7 +49,9 @@ where
   small = rel.getLesserOperand() and
   large = rel.getGreaterOperand() and
   rel = l.getCondition().getAChild*() and
-  upperBound(large).log2() > getComparisonSize(small) * 8 and
+  forall(Expr conv | conv = large.getConversion*() |
+    upperBound(conv).log2() > getComparisonSize(small) * 8
+  ) and
   // Ignore cases where the smaller type is int or larger
   // These are still bugs, but you should need a very large string or array to
   // trigger them. We will want to disable this for some applications, but it's

cpp/ql/src/Security/CWE/CWE-190/IntegerOverflowTainted.ql

@@ -28,6 +28,7 @@ predicate outOfBoundsExpr(Expr expr, string kind) {
 
 from Expr use, Expr origin, string kind
 where
+  not use.getUnspecifiedType() instanceof PointerType and
   outOfBoundsExpr(use, kind) and
   tainted(origin, use) and
   origin != use and

cpp/ql/src/Security/CWE/CWE-191/UnsignedDifferenceExpressionComparedZero.ql

@@ -12,29 +12,60 @@
 
 import cpp
 import semmle.code.cpp.commons.Exclusions
-import semmle.code.cpp.valuenumbering.GlobalValueNumbering
 import semmle.code.cpp.rangeanalysis.SimpleRangeAnalysis
+import semmle.code.cpp.rangeanalysis.RangeAnalysisUtils
 import semmle.code.cpp.controlflow.Guards
+import semmle.code.cpp.dataflow.DataFlow
 
-/** Holds if `sub` is guarded by a condition which ensures that `left >= right`. */
+/**
+ *  Holds if `sub` is guarded by a condition which ensures that
+ * `left >= right`.
+ */
 pragma[noinline]
 predicate isGuarded(SubExpr sub, Expr left, Expr right) {
-  exists(GuardCondition guard |
-    guard.controls(sub.getBasicBlock(), true) and
-    guard.ensuresLt(left, right, 0, sub.getBasicBlock(), false)
+  exists(GuardCondition guard, int k |
+    guard.controls(sub.getBasicBlock(), _) and
+    guard.ensuresLt(left, right, k, sub.getBasicBlock(), false) and
+    k >= 0
   )
 }
 
-/** Holds if `sub` will never be negative. */
-predicate nonNegative(SubExpr sub) {
-  not exprMightOverflowNegatively(sub.getFullyConverted())
+/**
+ * Holds if `n` is known or suspected to be less than or equal to
+ * `sub.getLeftOperand()`.
+ */
+predicate exprIsSubLeftOrLess(SubExpr sub, DataFlow::Node n) {
+  n.asExpr() = sub.getLeftOperand()
   or
-  // The subtraction is guarded by a check of the form `left >= right`.
-  exists(GVN left, GVN right |
-    // This is basically a poor man's version of a directional unbind operator.
-    strictcount([left, globalValueNumber(sub.getLeftOperand())]) = 1 and
-    strictcount([right, globalValueNumber(sub.getRightOperand())]) = 1 and
-    isGuarded(sub, left.getAnExpr(), right.getAnExpr())
+  exists(DataFlow::Node other |
+    // dataflow
+    exprIsSubLeftOrLess(sub, other) and
+    (
+      DataFlow::localFlowStep(n, other) or
+      DataFlow::localFlowStep(other, n)
+    )
+  )
+  or
+  exists(DataFlow::Node other |
+    // guard constraining `sub`
+    exprIsSubLeftOrLess(sub, other) and
+    isGuarded(sub, other.asExpr(), n.asExpr()) // other >= n
+  )
+  or
+  exists(DataFlow::Node other, float p, float q |
+    // linear access of `other`
+    exprIsSubLeftOrLess(sub, other) and
+    linearAccess(n.asExpr(), other.asExpr(), p, q) and // n = p * other + q
+    p <= 1 and
+    q <= 0
+  )
+  or
+  exists(DataFlow::Node other, float p, float q |
+    // linear access of `n`
+    exprIsSubLeftOrLess(sub, other) and
+    linearAccess(other.asExpr(), n.asExpr(), p, q) and // other = p * n + q
+    p >= 1 and
+    q >= 0
   )
 }
 
@@ -45,5 +76,6 @@ where
   ro.getLesserOperand().getValue().toInt() = 0 and
   ro.getGreaterOperand() = sub and
   sub.getFullyConverted().getUnspecifiedType().(IntegralType).isUnsigned() and
-  not nonNegative(sub)
+  exprMightOverflowNegatively(sub.getFullyConverted()) and // generally catches false positives involving constants
+  not exprIsSubLeftOrLess(sub, DataFlow::exprNode(sub.getRightOperand())) // generally catches false positives where there's a relation between the left and right operands
 select ro, "Unsigned subtraction can never be negative."

cpp/ql/src/Security/CWE/CWE-311/CleartextBufferWrite.qhelp

@@ -2,4 +2,4 @@
   "-//Semmle//qhelp//EN"
   "qhelp.dtd">
 <qhelp>
-<include src="CleartextStorage.qhelp" /></qhelp>
+<include src="CleartextStorage.inc.qhelp" /></qhelp>

cpp/ql/src/Security/CWE/CWE-311/CleartextFileWrite.qhelp

@@ -2,4 +2,4 @@
   "-//Semmle//qhelp//EN"
   "qhelp.dtd">
 <qhelp>
-<include src="CleartextStorage.qhelp" /></qhelp>
+<include src="CleartextStorage.inc.qhelp" /></qhelp>

cpp/ql/src/Security/CWE/CWE-428/UnsafeCreateProcessCall.ql

@@ -93,7 +93,7 @@ class QuotedCommandInCreateProcessFunctionConfiguration extends DataFlow2::Confi
 
 bindingset[s]
 predicate isQuotedOrNoSpaceApplicationNameOnCmd(string s) {
-  s.regexpMatch("\"([^\"])*\"(\\s|.)*") // The first element (path) is quoted
+  s.regexpMatch("\"([^\"])*\"[\\s\\S]*") // The first element (path) is quoted
   or
   s.regexpMatch("[^\\s]+") // There are no spaces in the string
 }

cpp/ql/src/Summary/LinesOfCode.ql

@@ -0,0 +1,11 @@
+/**
+ * @id cpp/summary/lines-of-code
+ * @name Total lines of C/C++ code in the database
+ * @description The total number of lines of C/C++ code across all files, including system headers, libraries, and auto-generated files. This is a useful metric of the size of a database. For all files that were seen during the build, this query counts the lines of code, excluding whitespace or comments.
+ * @kind metric
+ * @tags summary
+ */
+
+import cpp
+
+select sum(File f | f.fromSource() | f.getMetrics().getNumberOfLinesOfCode())

cpp/ql/src/experimental/Security/CWE/CWE-020/LateCheckOfFunctionArgument.c

@@ -0,0 +1,7 @@
+if(len<0) return 1;
+memset(dest, source, len); // GOOD: variable `len` checked before call
+
+...
+
+memset(dest, source, len); // BAD: variable `len` checked after call
+if(len<0) return 1;

cpp/ql/src/experimental/Security/CWE/CWE-020/LateCheckOfFunctionArgument.qhelp

@@ -0,0 +1,28 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+<overview>
+<p>Checking the function argument after calling the function itself. This situation looks suspicious and requires the attention of the developer. It may be necessary to add validation before calling the function</p>
+
+
+</overview>
+<recommendation>
+
+<p>We recommend checking before calling the function.</p>
+
+</recommendation>
+<example>
+<p>The following example demonstrates an erroneous and fixed use of function argument validation.</p>
+<sample src="LateCheckOfFunctionArgument.c" />
+
+</example>
+<references>
+
+<li>
+  CWE Common Weakness Enumeration:
+  <a href="https://cwe.mitre.org/data/definitions/20.html"> CWE-20: Improper Input Validation</a>.
+</li>
+
+</references>
+</qhelp>

cpp/ql/src/experimental/Security/CWE/CWE-020/LateCheckOfFunctionArgument.ql

@@ -0,0 +1,66 @@
+/**
+ * @name Late Check Of Function Argument
+ * @description --Checking the function argument after calling the function itself.
+ *              --This situation looks suspicious and requires the attention of the developer.
+ *              --It may be necessary to add validation before calling the function.
+ * @kind problem
+ * @id cpp/late-check-of-function-argument
+ * @problem.severity warning
+ * @precision medium
+ * @tags correctness
+ *       security
+ *       external/cwe/cwe-20
+ */
+
+import cpp
+import semmle.code.cpp.valuenumbering.GlobalValueNumbering
+
+/** Holds for a function `f` that has an argument at index `apos` used for positioning in a buffer. */
+predicate numberArgument(Function f, int apos) {
+  f.hasGlobalOrStdName("write") and apos = 2
+  or
+  f.hasGlobalOrStdName("read") and apos = 2
+  or
+  f.hasGlobalOrStdName("lseek") and apos = 1
+  or
+  f.hasGlobalOrStdName("memmove") and apos = 2
+  or
+  f.hasGlobalOrStdName("memset") and apos = 2
+  or
+  f.hasGlobalOrStdName("memcpy") and apos = 2
+  or
+  f.hasGlobalOrStdName("memcmp") and apos = 2
+  or
+  f.hasGlobalOrStdName("strncat") and apos = 2
+  or
+  f.hasGlobalOrStdName("strncpy") and apos = 2
+  or
+  f.hasGlobalOrStdName("strncmp") and apos = 2
+  or
+  f.hasGlobalOrStdName("snprintf") and apos = 1
+  or
+  f.hasGlobalOrStdName("strndup") and apos = 2
+}
+
+class IfCompareWithZero extends IfStmt {
+  IfCompareWithZero() { this.getCondition().(RelationalOperation).getAChild().getValue() = "0" }
+
+  Expr noZerroOperand() {
+    if this.getCondition().(RelationalOperation).getGreaterOperand().getValue() = "0"
+    then result = this.getCondition().(RelationalOperation).getLesserOperand()
+    else result = this.getCondition().(RelationalOperation).getGreaterOperand()
+  }
+}
+
+from FunctionCall fc, IfCompareWithZero ifc, int na
+where
+  numberArgument(fc.getTarget(), na) and
+  globalValueNumber(fc.getArgument(na)) = globalValueNumber(ifc.noZerroOperand()) and
+  dominates(fc, ifc) and
+  not exists(IfStmt ifc1 |
+    dominates(ifc1, fc) and
+    globalValueNumber(fc.getArgument(na)) = globalValueNumber(ifc1.getCondition().getAChild*())
+  )
+select fc,
+  "The value of argument '$@' appears to be checked after the call, rather than before it.",
+  fc.getArgument(na), fc.getArgument(na).toString()

cpp/ql/src/experimental/Security/CWE/CWE-1126/DeclarationOfVariableWithUnnecessarilyWideScope.c

@@ -0,0 +1,14 @@
+while(intIndex > 2)
+{
+  ...
+  intIndex--;
+  ...
+} // GOOD: correct cycle
+...
+while(intIndex > 2)
+{
+  ...
+  int intIndex;
+  intIndex--;
+  ...
+} // BAD: the variable used in the condition does not change.

cpp/ql/src/experimental/Security/CWE/CWE-1126/DeclarationOfVariableWithUnnecessarilyWideScope.qhelp

@@ -0,0 +1,26 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+<overview>
+<p>Using variables with the same name is dangerous. However, such a situation inside the while loop can create an infinite loop exhausting resources. Requires the attention of developers.</p>
+
+</overview>
+<recommendation>
+<p>We recommend not to use local variables inside a loop if their names are the same as the variables in the condition of this loop.</p>
+
+</recommendation>
+<example>
+<p>The following example demonstrates an erroneous and corrected use of a local variable within a loop.</p>
+<sample src="DeclarationOfVariableWithUnnecessarilyWideScope.c" />
+
+</example>
+<references>
+
+<li>
+  CERT C Coding Standard:
+  <a href="https://wiki.sei.cmu.edu/confluence/display/c/DCL01-C.+Do+not+reuse+variable+names+in+subscopes">DCL01-C. Do not reuse variable names in subscopes</a>.
+</li>
+
+</references>
+</qhelp>

cpp/ql/src/experimental/Security/CWE/CWE-1126/DeclarationOfVariableWithUnnecessarilyWideScope.ql

@@ -0,0 +1,60 @@
+/**
+ * @name Errors When Using Variable Declaration Inside Loop
+ * @description Using variables with the same name is dangerous.
+ *              However, such a situation inside the while loop can create an infinite loop exhausting resources.
+ *              Requires the attention of developers.
+ * @kind problem
+ * @id cpp/errors-when-using-variable-declaration-inside-loop
+ * @problem.severity warning
+ * @precision medium
+ * @tags correctness
+ *       security
+ *       external/cwe/cwe-1126
+ */
+
+import cpp
+
+/**
+ * Errors when using a variable declaration inside a loop.
+ */
+class DangerousWhileLoop extends WhileStmt {
+  Expr exp;
+  Declaration dl;
+
+  DangerousWhileLoop() {
+    this = dl.getParentScope().(BlockStmt).getParent*() and
+    exp = this.getCondition().getAChild*() and
+    not exp instanceof PointerFieldAccess and
+    not exp instanceof ValueFieldAccess and
+    exp.(VariableAccess).getTarget().getName() = dl.getName() and
+    not exp.getParent*() instanceof FunctionCall
+  }
+
+  Declaration getDeclaration() { result = dl }
+
+  /** Holds when there are changes to the variables involved in the condition. */
+  predicate isUseThisVariable() {
+    exists(Variable v |
+      this.getCondition().getAChild*().(VariableAccess).getTarget() = v and
+      (
+        exists(Assignment aexp |
+          this = aexp.getEnclosingStmt().getParentStmt*() and
+          (
+            aexp.getLValue().(ArrayExpr).getArrayBase().(VariableAccess).getTarget() = v
+            or
+            aexp.getLValue().(VariableAccess).getTarget() = v
+          )
+        )
+        or
+        exists(CrementOperation crm |
+          this = crm.getEnclosingStmt().getParentStmt*() and
+          crm.getOperand().(VariableAccess).getTarget() = v
+        )
+      )
+    )
+  }
+}
+
+from DangerousWhileLoop lp
+where not lp.isUseThisVariable()
+select lp.getDeclaration(), "A variable with this name is used in the $@ condition.", lp, "loop"

cpp/ql/src/experimental/Security/CWE/CWE-570/IncorrectAllocationErrorHandling.cpp

@@ -0,0 +1,43 @@
+// BAD: the allocation will throw an unhandled exception
+// instead of returning a null pointer.
+void bad1(std::size_t length) noexcept {
+  int* dest = new int[length];
+  if(!dest) {
+    return;
+  }
+  std::memset(dest, 0, length);
+  // ...
+}
+
+// BAD: the allocation won't throw an exception, but
+// instead return a null pointer.
+void bad2(std::size_t length) noexcept {
+  try {
+    int* dest = new(std::nothrow) int[length];
+    std::memset(dest, 0, length);
+    // ...
+  } catch(std::bad_alloc&) {
+    // ...
+  }
+}
+
+// GOOD: the allocation failure is handled appropiately.
+void good1(std::size_t length) noexcept {
+  try {
+    int* dest = new int[length];
+    std::memset(dest, 0, length);
+    // ...
+  } catch(std::bad_alloc&) {
+    // ...
+  }
+}
+
+// GOOD: the allocation failure is handled appropiately.
+void good2(std::size_t length) noexcept {
+  int* dest = new int[length];
+  if(!dest) {
+    return;
+  }
+  std::memset(dest, 0, length);
+  // ...
+}

cpp/ql/src/experimental/Security/CWE/CWE-570/IncorrectAllocationErrorHandling.qhelp

@@ -0,0 +1,31 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+<overview>
+<p>Different overloads of the <code>new</code> operator handle allocation failures in different ways.
+If <code>new T</code> fails for some type <code>T</code>, it throws a <code>std::bad_alloc</code> exception,
+but <code>new(std::nothrow) T</code> returns a null pointer. If the programmer does not use the corresponding
+method of error handling, allocation failure may go unhandled and could cause the program to behave in
+unexpected ways.</p>
+
+</overview>
+<recommendation>
+
+<p>Make sure that exceptions are handled appropriately if <code>new T</code> is used. On the other hand,
+make sure to handle the possibility of null pointers if <code>new(std::nothrow) T</code> is used.</p>
+
+</recommendation>
+<example>
+<sample src="IncorrectAllocationErrorHandling.cpp" />
+
+</example>
+<references>
+
+<li>
+  CERT C++ Coding Standard:
+<a href="https://wiki.sei.cmu.edu/confluence/display/cplusplus/MEM52-CPP.+Detect+and+handle+memory+allocation+errors">MEM52-CPP. Detect and handle memory allocation errors</a>.
+</li>
+
+</references>
+</qhelp>