denonify

JS: add a getFlowLabel method to the PathNode class

.codeqlmanifest.json

@@ -4,6 +4,7 @@
         "*/ql/lib/qlpack.yml",
         "*/ql/test/qlpack.yml",
         "*/ql/examples/qlpack.yml",
+        "*/ql/consistency-queries/qlpack.yml",
         "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
         "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
         "javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml",
@@ -14,8 +15,6 @@
         "misc/legacy-support/*/qlpack.yml",
         "misc/suite-helpers/qlpack.yml",
         "ruby/extractor-pack/codeql-extractor.yml",
-        "ruby/ql/consistency-queries/qlpack.yml",
-        "ql/ql/consistency-queries/qlpack.yml",
         "ql/extractor-pack/codeql-extractor.yml"
     ],
     "versionPolicies": {

.gitattributes

@@ -50,4 +50,9 @@
 *.pdb -text
 
 java/ql/test/stubs/**/*.java linguist-generated=true
-java/ql/test/experimental/stubs/**/*.java linguist-generated=true
+java/ql/test/experimental/stubs/**/*.java linguist-generated=true
+
+# Generated test files - these are synced from the standard JavaScript libraries using
+# `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
+javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
+javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge

.github/workflows/codeql-analysis.yml

@@ -27,6 +27,11 @@ jobs:
       pull-requests: read
 
     steps:
+    - name: Setup dotnet
+      uses: actions/setup-dotnet@v1
+      with:
+        dotnet-version: 6.0.101
+
     - name: Checkout repository
       uses: actions/checkout@v2
 
@@ -51,7 +56,7 @@ jobs:
     #    uses a compiled language
 
     - run: |
-       dotnet build csharp
+       dotnet build csharp /p:UseSharedCompilation=false
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@main

.github/workflows/csv-coverage-metrics.yml

@@ -0,0 +1,43 @@
+name: "Publish framework coverage as metrics"
+
+on:
+  schedule:
+    - cron: '5 0 * * *'
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/csv-coverage-metrics.yml"
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Setup CodeQL
+        uses: ./.github/actions/fetch-codeql
+      - name: Create empty database
+        run: |
+          DATABASE="${{ runner.temp }}/java-database"
+          PROJECT="${{ runner.temp }}/java-project"
+          mkdir -p "$PROJECT/src/tmp/empty"
+          echo "class Empty {}" >> "$PROJECT/src/tmp/empty/Empty.java"
+          codeql database create "$DATABASE" --language=java --source-root="$PROJECT" --command 'javac src/tmp/empty/Empty.java'
+      - name: Capture coverage information
+        run: |
+          DATABASE="${{ runner.temp }}/java-database"
+          codeql database analyze --format=sarif-latest --output=metrics.sarif -- "$DATABASE" ./java/ql/src/Metrics/Summaries/FrameworkCoverage.ql
+      - uses: actions/upload-artifact@v2
+        with:
+          name: metrics.sarif
+          path: metrics.sarif
+          retention-days: 20
+      - name: Upload SARIF file
+        uses: github/codeql-action/upload-sarif@v1
+        with:
+          sarif_file: metrics.sarif

.github/workflows/js-ml-tests.yml

@@ -0,0 +1,67 @@
+name: JS ML-powered queries tests
+
+on:
+  push:
+    paths:
+      - "javascript/ql/experimental/adaptivethreatmodeling/**"
+      - .github/workflows/js-ml-tests.yml
+    branches:
+      - main
+      - "rc/*"
+  pull_request:
+    paths:
+      - "javascript/ql/experimental/adaptivethreatmodeling/**"
+      - .github/workflows/js-ml-tests.yml
+
+defaults:
+  run:
+    working-directory: javascript/ql/experimental/adaptivethreatmodeling
+
+jobs:
+  qlformat:
+    name: Check QL formatting
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - name: Check QL formatting
+        run: |
+          find . "(" -name "*.ql" -or -name "*.qll" ")" -print0 | \
+            xargs -0 codeql query format --check-only
+
+  qlcompile:
+    name: Check QL compilation
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - name: Check QL compilation
+        run: |
+          codeql query compile \
+            --check-only \
+            --ram 5120 \
+            --additional-packs "${{ github.workspace }}" \
+            --threads=0 \
+            -- \
+            lib modelbuilding src
+
+  qltest:
+    name: Run QL tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - name: Run QL tests
+        run: |
+          codeql test run \
+            --threads=0 \
+            --ram 5120 \
+            --additional-packs "${{ github.workspace }}" \
+            -- \
+            test

.github/workflows/mad_regenerate-models.yml

@@ -0,0 +1,62 @@
+name: Regenerate framework models
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "30 2 * * *"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/mad_regenerate-models.yml"
+
+jobs:
+  regenerate-models:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # placeholder required for each axis, excluded below, replaced by the actual combinations (see include)
+        slug: ["placeholder"]
+        ref: ["placeholder"]
+        include:
+          - slug: "apache/commons-io"
+            ref: "8985de8fe74f6622a419b37a6eed0dbc484dc128"
+        exclude:
+          - slug: "placeholder"
+            ref: "placeholder"
+    steps:
+      - name: Clone self (github/codeql)
+        uses: actions/checkout@v2
+      - name: Setup CodeQL binaries
+        uses: ./.github/actions/fetch-codeql
+      - name: Clone repositories
+        uses: actions/checkout@v2
+        with:
+          path: repos/${{ matrix.ref }}
+          ref: ${{ matrix.ref }}
+          repository: ${{ matrix.slug }}
+      - name: Build database
+        env:
+          SLUG: ${{ matrix.slug }}
+          REF: ${{ matrix.ref }}
+        run: |
+          mkdir dbs
+          cd repos/${REF}
+          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
+          codeql database create --language=java ../../dbs/${SHORTNAME}
+      - name: Regenerate models in-place
+        env:
+          SLUG: ${{ matrix.slug }}
+        run: |
+          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
+          java/ql/src/utils/model-generator/RegenerateModels.py "${SLUG}" dbs/${SHORTNAME}
+      - name: Stage changes
+        run: |
+          find java -name "*.qll" -print0 | xargs -0 git add
+          git status
+          git diff --cached > models.patch
+      - uses: actions/upload-artifact@v2
+        with:
+          name: patch
+          path: models.patch
+          retention-days: 7

.github/workflows/ql-for-ql-build.yml

@@ -189,4 +189,11 @@ jobs:
         uses: github/codeql-action/analyze@erik-krogh/ql
         with: 
           category: "ql-for-ql-${{ matrix.folder }}"
+      - name: Copy sarif file to CWD
+        run: cp ../results/ql.sarif ./${{ matrix.folder }}.sarif
+      - name: Sarif as artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.folder }}.sarif
+          path: ${{ matrix.folder }}.sarif
 

.github/workflows/ruby-build.yml

@@ -50,7 +50,7 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             ruby/target
-          key: ${{ runner.os }}-ruby-rust-cargo-${{ hashFiles('ruby/**/Cargo.lock') }}
+          key: ${{ runner.os }}-ruby-rust-cargo-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/**/Cargo.lock') }}
       - name: Check formatting
         run: cargo fmt --all -- --check
       - name: Build

.github/workflows/ruby-qltest.yml

@@ -52,6 +52,14 @@ jobs:
           codeql dataset import -S ql/lib/upgrades/initial/ruby.dbscheme testdb empty.trap
           codeql dataset upgrade testdb --additional-packs ql/lib
           diff -q testdb/ruby.dbscheme ql/lib/ruby.dbscheme
+      - name: Check DB downgrade scripts
+        run: |
+          echo >empty.trap
+          rm -rf testdb; codeql dataset import -S ql/lib/ruby.dbscheme testdb empty.trap
+          codeql resolve upgrades --format=lines --allow-downgrades --additional-packs downgrades \
+           --dbscheme=ql/lib/ruby.dbscheme --target-dbscheme=downgrades/initial/ruby.dbscheme |
+           xargs codeql execute upgrades testdb
+          diff -q testdb/ruby.dbscheme downgrades/initial/ruby.dbscheme
   qltest:
     runs-on: ubuntu-latest
     strategy:

CODEOWNERS

@@ -13,6 +13,9 @@
 /python/**/experimental/**/* @github/codeql-python @xcorail
 /ruby/**/experimental/**/* @github/codeql-ruby @xcorail
 
+# ML-powered queries
+/javascript/ql/experimental/adaptivethreatmodeling/ @github/codeql-ml-powered-queries-reviewers
+
 # Notify members of codeql-go about PRs to the shared data-flow library files
 /java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl.qll @github/codeql-java @github/codeql-go
 /java/ql/src/semmle/code/java/dataflow/internal/DataFlowImpl2.qll @github/codeql-java @github/codeql-go
@@ -27,4 +30,4 @@
 /docs/query-*-style-guide.md @github/codeql-analysis-reviewers
 
 # QL for QL reviewers
-/ql/ @github/codeql-ql-for-ql-reviewers
+/ql/ @github/codeql-ql-for-ql-reviewers

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/Semmle.Autobuild.Cpp.Tests.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
     <Nullable>enable</Nullable>

cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <AssemblyName>Semmle.Autobuild.Cpp</AssemblyName>
     <RootNamespace>Semmle.Autobuild.Cpp</RootNamespace>
     <ApplicationIcon />

cpp/ql/lib/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 0.0.8
+
+### Deprecated APIs
+
+* The `codeql/cpp-upgrades` CodeQL pack has been removed. All upgrades scripts have been merged into the `codeql/cpp-all` CodeQL pack.
+
+### Minor Analysis Improvements
+
+* `FormatLiteral::getMaxConvertedLength` now uses range analysis to provide a
+  more accurate length for integers formatted with `%x`
+
 ## 0.0.7
 
 ## 0.0.6

cpp/ql/lib/change-notes/2022-01-11-remove-upgrades-packs.md

@@ -1,4 +0,0 @@
----
-category: deprecated
----
-* The `codeql/cpp-upgrades` CodeQL pack has been removed. All upgrades scripts have been merged into the `codeql/cpp-all` CodeQL pack.

cpp/ql/lib/change-notes/2022-01-14-hex-format-range-analysis.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* `FormatLiteral::getMaxConvertedLength` now uses range analysis to provide a
-  more accurate length for integers formatted with `%x`

cpp/ql/lib/change-notes/2022-02-02-detect-creat-in-openat.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The `cpp/world-writable-file-creation` query now only detects `open` and `openat` calls with the `O_CREAT` or `O_TMPFILE` flag.

cpp/ql/lib/change-notes/2022-02-02-detect-missing-mode-argument-for-open.md

@@ -0,0 +1,4 @@
+---
+category: newQuery
+---
+* Added a new query, `cpp/open-call-with-mode-argument`, to detect when `open` or `openat` is called with the `O_CREAT` or `O_TMPFILE` flag but when the `mode` argument is omitted.

cpp/ql/lib/change-notes/2022-03-03-template-implicit-copy.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The `Class::hasImplicitCopyConstructor` and `Class::hasImplicitCopyAssignmentOperator` methods now handle template instantiations more accurately. This should improve results for the `cpp/rule-of-two` query.

cpp/ql/lib/change-notes/released/0.0.8.md

@@ -0,0 +1,10 @@
+## 0.0.8
+
+### Deprecated APIs
+
+* The `codeql/cpp-upgrades` CodeQL pack has been removed. All upgrades scripts have been merged into the `codeql/cpp-all` CodeQL pack.
+
+### Minor Analysis Improvements
+
+* `FormatLiteral::getMaxConvertedLength` now uses range analysis to provide a
+  more accurate length for integers formatted with `%x`

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.0.7
+lastReleaseVersion: 0.0.8

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.0.8-dev
+version: 0.0.9-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/lib/semmle/code/cpp/Class.qll

@@ -285,17 +285,7 @@ class Class extends UserType {
   predicate hasImplicitCopyConstructor() {
     not this.implicitCopyConstructorDeleted() and
     forall(CopyConstructor cc | cc = this.getAMemberFunction() |
-      cc.isCompilerGenerated() and not cc.isDeleted() and not cc.isDefaulted()
-    ) and
-    (
-      not this instanceof ClassTemplateInstantiation
-      or
-      this.(ClassTemplateInstantiation).getTemplate().hasImplicitCopyConstructor()
-    ) and
-    (
-      not this instanceof PartialClassTemplateSpecialization
-      or
-      this.(PartialClassTemplateSpecialization).getPrimaryTemplate().hasImplicitCopyConstructor()
+      cc.isCompilerGenerated() and not cc.isDeleted()
     )
   }
 
@@ -310,19 +300,7 @@ class Class extends UserType {
   predicate hasImplicitCopyAssignmentOperator() {
     not this.implicitCopyAssignmentOperatorDeleted() and
     forall(CopyAssignmentOperator ca | ca = this.getAMemberFunction() |
-      ca.isCompilerGenerated() and not ca.isDeleted() and not ca.isDefaulted()
-    ) and
-    (
-      not this instanceof ClassTemplateInstantiation
-      or
-      this.(ClassTemplateInstantiation).getTemplate().hasImplicitCopyAssignmentOperator()
-    ) and
-    (
-      not this instanceof PartialClassTemplateSpecialization
-      or
-      this.(PartialClassTemplateSpecialization)
-          .getPrimaryTemplate()
-          .hasImplicitCopyAssignmentOperator()
+      ca.isCompilerGenerated() and not ca.isDeleted()
     )
   }
 
@@ -333,12 +311,6 @@ class Class extends UserType {
    * http://en.cppreference.com/w/cpp/language/copy_constructor#Deleted_implicitly-declared_copy_constructor
    */
   predicate implicitCopyConstructorDeleted() {
-    forex(CopyConstructor cc | cc = this.getAConstructor() |
-      cc.isDeleted()
-      or
-      not cc.isCompilerGenerated()
-    )
-    or
     // - T has non-static data members that cannot be copied (have deleted,
     //   inaccessible, or ambiguous copy constructors);
     exists(Type t | t = this.getAFieldSubobjectType().getUnspecifiedType() |
@@ -346,6 +318,34 @@ class Class extends UserType {
       // constructors are considered equal.
       this.cannotAccessCopyConstructorOnAny(t)
     )
+    or
+    // - T has direct or virtual base class that cannot be copied (has deleted,
+    //   inaccessible, or ambiguous copy constructors);
+    exists(Class c | c = this.getADirectOrVirtualBase() |
+      // Note: Overload resolution is not implemented -- all copy
+      // constructors are considered equal.
+      this.cannotAccessCopyConstructorOnThis(c)
+    )
+    or
+    // - T has direct or virtual base class with a deleted or inaccessible
+    //   destructor;
+    exists(Class base | base = this.getADirectOrVirtualBase() |
+      this.cannotAccessDestructor(base, this)
+    )
+    or
+    // - T has a user-defined move constructor or move assignment operator;
+    exists(MoveConstructor mc | mc = this.getAMemberFunction() | not mc.isCompilerGenerated())
+    or
+    exists(MoveAssignmentOperator ma | ma = this.getAMemberFunction() |
+      not ma.isCompilerGenerated()
+    )
+    or
+    // - T is a union and has a variant member with non-trivial copy
+    //   constructor (since C++11)
+    none() // Not implemented
+    or
+    // - T has a data member of rvalue reference type.
+    exists(Type t | t = this.getAFieldSubobjectType() | t instanceof RValueReferenceType)
   }
 
   /**
@@ -355,12 +355,34 @@ class Class extends UserType {
    * http://en.cppreference.com/w/cpp/language/copy_assignment#Deleted_implicitly-declared_copy_assignment_operator
    */
   predicate implicitCopyAssignmentOperatorDeleted() {
-    forex(CopyAssignmentOperator ca | ca = this.getAMemberFunction() |
-      ca.isDeleted()
-      or
-      not ca.isCompilerGenerated()
+    // - T has a user-declared move constructor;
+    exists(MoveConstructor mc | mc = this.getAMemberFunction() | not mc.isCompilerGenerated())
+    or
+    // - T has a user-declared move assignment operator.
+    exists(MoveAssignmentOperator ma | ma = this.getAMemberFunction() |
+      not ma.isCompilerGenerated()
     )
     or
+    // - T has a non-static data member of non-class type (or array thereof)
+    //   that is const;
+    exists(Type t | t = this.getAFieldSubobjectType() |
+      // The rule for this case refers only to non-class types only, but our
+      // implementation extends it to cover class types too. Class types are
+      // supposed to be covered by the rule below on data members that
+      // cannot be copy-assigned. Copy-assigning a const class-typed member
+      // would call an overload of type
+      // `const C& operator=(const C&) const;`. Such an overload is unlikely
+      // to exist because it contradicts the intention of "const": it allows
+      // assigning to a const object. But since we have not implemented the
+      // ability to distinguish between overloads, we cannot distinguish that
+      // overload from the ordinary `C& operator=(const C&);`. Instead, we
+      // require class types to be non-const in this clause.
+      /* not t instanceof Class and */ t.isConst()
+    )
+    or
+    // - T has a non-static data member of a reference type;
+    exists(Type t | t = this.getAFieldSubobjectType() | t instanceof ReferenceType)
+    or
     // - T has a non-static data member or a direct or virtual base class that
     //   cannot be copy-assigned (overload resolution for the copy assignment
     //   fails, or selects a deleted or inaccessible function);
@@ -369,6 +391,15 @@ class Class extends UserType {
       // operators are considered equal.
       this.cannotAccessCopyAssignmentOperatorOnAny(t)
     )
+    or
+    exists(Class c | c = this.getADirectOrVirtualBase() |
+      // Note: Overload resolution is not implemented -- all copy assignment
+      // operators are considered equal.
+      this.cannotAccessCopyAssignmentOperatorOnThis(c)
+    )
+    // - T is a union-like class, and has a variant member whose corresponding
+    //   assignment operator is non-trivial.
+    // Not implemented
   }
 
   /** Gets the destructor of this class, struct or union, if any. */

cpp/ql/lib/semmle/code/cpp/commons/unix/Constants.qll

@@ -11,10 +11,10 @@ import cpp
  */
 bindingset[input]
 int parseOctal(string input) {
-  input.charAt(0) = "0" and
+  input.regexpMatch("0[0-7]+") and
   result =
     strictsum(int ix |
-      ix in [0 .. input.length()]
+      ix in [1 .. input.length()]
     |
       8.pow(input.length() - (ix + 1)) * input.charAt(ix).toInt()
     )

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowUtil.qll

@@ -592,12 +592,14 @@ predicate simpleLocalFlowStep(Node nodeFrom, Node nodeTo) {
  * Holds if data flows from `source` to `sink` in zero or more local
  * (intra-procedural) steps.
  */
+pragma[inline]
 predicate localFlow(Node source, Node sink) { localFlowStep*(source, sink) }
 
 /**
  * Holds if data can flow from `e1` to `e2` in zero or more
  * local (intra-procedural) steps.
  */
+pragma[inline]
 predicate localExprFlow(Expr e1, Expr e2) { localFlow(exprNode(e1), exprNode(e2)) }
 
 /**

cpp/ql/lib/semmle/code/cpp/dataflow/internal/TaintTrackingUtil.qll

@@ -124,12 +124,14 @@ predicate localAdditionalTaintStep(DataFlow::Node nodeFrom, DataFlow::Node nodeT
  * Holds if taint may propagate from `source` to `sink` in zero or more local
  * (intra-procedural) steps.
  */
+pragma[inline]
 predicate localTaint(DataFlow::Node source, DataFlow::Node sink) { localTaintStep*(source, sink) }
 
 /**
  * Holds if taint can flow from `e1` to `e2` in zero or more
  * local (intra-procedural) steps.
  */
+pragma[inline]
 predicate localExprTaint(Expr e1, Expr e2) {
   localTaint(DataFlow::exprNode(e1), DataFlow::exprNode(e2))
 }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll

@@ -1032,12 +1032,14 @@ SideEffectInstruction getSideEffectFor(CallInstruction call, int argument) {
  * Holds if data flows from `source` to `sink` in zero or more local
  * (intra-procedural) steps.
  */
+pragma[inline]
 predicate localFlow(Node source, Node sink) { localFlowStep*(source, sink) }
 
 /**
  * Holds if data can flow from `i1` to `i2` in zero or more
  * local (intra-procedural) steps.
  */
+pragma[inline]
 predicate localInstructionFlow(Instruction e1, Instruction e2) {
   localFlow(instructionNode(e1), instructionNode(e2))
 }
@@ -1046,6 +1048,7 @@ predicate localInstructionFlow(Instruction e1, Instruction e2) {
  * Holds if data can flow from `e1` to `e2` in zero or more
  * local (intra-procedural) steps.
  */
+pragma[inline]
 predicate localExprFlow(Expr e1, Expr e2) { localFlow(exprNode(e1), exprNode(e2)) }
 
 private newtype TContent =

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/TaintTrackingUtil.qll

@@ -121,12 +121,14 @@ private predicate operandToInstructionTaintStep(Operand opFrom, Instruction inst
  * Holds if taint may propagate from `source` to `sink` in zero or more local
  * (intra-procedural) steps.
  */
+pragma[inline]
 predicate localTaint(DataFlow::Node source, DataFlow::Node sink) { localTaintStep*(source, sink) }
 
 /**
  * Holds if taint can flow from `i1` to `i2` in zero or more
  * local (intra-procedural) steps.
  */
+pragma[inline]
 predicate localInstructionTaint(Instruction i1, Instruction i2) {
   localTaint(DataFlow::instructionNode(i1), DataFlow::instructionNode(i2))
 }
@@ -135,6 +137,7 @@ predicate localInstructionTaint(Instruction i1, Instruction i2) {
  * Holds if taint can flow from `e1` to `e2` in zero or more
  * local (intra-procedural) steps.
  */
+pragma[inline]
 predicate localExprTaint(Expr e1, Expr e2) {
   localTaint(DataFlow::exprNode(e1), DataFlow::exprNode(e2))
 }

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedElement.qll

@@ -567,6 +567,13 @@ newtype TTranslatedElement =
   } or
   // The initialization of a base class from within a constructor.
   TTranslatedConstructorBaseInit(ConstructorBaseInit init) { not ignoreExpr(init) } or
+  // Workaround for a case where no base constructor is generated but a targetless base
+  // constructor call is present.
+  TTranslatedConstructorBareInit(ConstructorInit init) {
+    not ignoreExpr(init) and
+    not init instanceof ConstructorBaseInit and
+    not init instanceof ConstructorFieldInit
+  } or
   // The destruction of a base class from within a destructor.
   TTranslatedDestructorBaseDestruction(DestructorBaseDestruction destruction) {
     not ignoreExpr(destruction)

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedFunction.qll

@@ -573,6 +573,11 @@ class TranslatedConstructorInitList extends TranslatedElement, InitializationCon
       baseInit = func.(Constructor).getInitializer(id) and
       result = getTranslatedConstructorBaseInit(baseInit)
     )
+    or
+    exists(ConstructorInit bareInit |
+      bareInit = func.(Constructor).getInitializer(id) and
+      result = getTranslatedConstructorBareInit(bareInit)
+    )
   }
 
   override Instruction getFirstInstruction() {

cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedInitialization.qll

@@ -917,3 +917,36 @@ class TranslatedDestructorBaseDestruction extends TranslatedBaseStructorCall,
 
   final override string toString() { result = "destroy base: " + call.toString() }
 }
+
+/**
+ * A constructor base init call where no base constructor has been generated.
+ *
+ * Workaround for an extractor issue.
+ */
+class TranslatedConstructorBareInit extends TranslatedElement, TTranslatedConstructorBareInit {
+  ConstructorInit init;
+
+  TranslatedConstructorBareInit() { this = TTranslatedConstructorBareInit(init) }
+
+  override Locatable getAST() { result = init }
+
+  final override string toString() { result = "construct base (no constructor)" }
+
+  override Instruction getFirstInstruction() { result = getParent().getChildSuccessor(this) }
+
+  override predicate hasInstruction(Opcode opcode, InstructionTag tag, CppType resultType) {
+    none()
+  }
+
+  override TranslatedElement getChild(int id) { none() }
+
+  override Function getFunction() { result = getParent().getFunction() }
+
+  override Instruction getInstructionSuccessor(InstructionTag tag, EdgeKind kind) { none() }
+
+  override Instruction getChildSuccessor(TranslatedElement child) { none() }
+}
+
+TranslatedConstructorBareInit getTranslatedConstructorBareInit(ConstructorInit init) {
+  result.getAST() = init
+}

cpp/ql/src/CHANGELOG.md

@@ -1,3 +1,22 @@
+## 0.0.8
+
+### New Queries
+
+* The `security` tag has been added to the `cpp/return-stack-allocated-memory` query. As a result, its results will now appear by default.
+* The "Uncontrolled data in arithmetic expression" (cpp/uncontrolled-arithmetic) query has been enhanced to reduce false positive results and its @precision increased to high.
+* A new `cpp/very-likely-overruning-write` query has been added to the default query suite for C/C++. The query reports some results that were formerly flagged by `cpp/overruning-write`.
+
+### Minor Analysis Improvements
+
+* Fix an issue with the `cpp/declaration-hides-variable` query where it would report variables that are unnamed in a database.
+* The `cpp/cleartext-storage-file` query has been upgraded with non-local taint flow and has been converted to a `path-problem` query.
+* The `cpp/return-stack-allocated-memory` query has been improved to produce fewer false positives. The
+  query has also been converted to a `path-problem` query.
+* The "Cleartext transmission of sensitive information" (`cpp/cleartext-transmission`) query has been improved in several ways to reduce false positive results.
+* The "Potential improper null termination" (`cpp/improper-null-termination`) query now produces fewer false positive results around control flow branches and loops.
+* Added exception for GLib's gboolean to cpp/ambiguously-signed-bit-field.
+  This change reduces the number of false positives in the query.
+
 ## 0.0.7
 
 ## 0.0.6

cpp/ql/src/Likely Bugs/Memory Management/ReturnStackAllocatedMemory.ql

@@ -14,6 +14,9 @@
  */
 
 import cpp
+// We don't actually use the global value numbering library in this query, but without it we end up
+// recomputing the IR.
+private import semmle.code.cpp.valuenumbering.GlobalValueNumbering
 import semmle.code.cpp.ir.IR
 import semmle.code.cpp.ir.dataflow.DataFlow::DataFlow
 

cpp/ql/src/Security/CWE/CWE-311/CleartextTransmission.ql

@@ -5,7 +5,7 @@
  * @kind path-problem
  * @problem.severity warning
  * @security-severity 7.5
- * @precision medium
+ * @precision high
  * @id cpp/cleartext-transmission
  * @tags security
  *       external/cwe/cwe-319
@@ -14,8 +14,8 @@
 import cpp
 import semmle.code.cpp.security.SensitiveExprs
 import semmle.code.cpp.dataflow.TaintTracking
-import semmle.code.cpp.valuenumbering.GlobalValueNumbering
 import semmle.code.cpp.models.interfaces.FlowSource
+import semmle.code.cpp.commons.File
 import DataFlow::PathGraph
 
 /**
@@ -121,24 +121,32 @@ abstract class NetworkSendRecv extends FunctionCall {
   NetworkSendRecv() {
     this.getTarget() = target and
     // exclude calls based on the socket...
-    not exists(GVN g |
-      g = globalValueNumber(target.getSocketExpr(this)) and
+    not exists(DataFlow::Node src, DataFlow::Node dest |
+      DataFlow::localFlow(src, dest) and
+      dest.asExpr() = target.getSocketExpr(this) and
       (
         // literal constant
-        globalValueNumber(any(Literal l)) = g
+        src.asExpr() instanceof Literal
         or
         // variable (such as a global) initialized to a literal constant
         exists(Variable v |
           v.getInitializer().getExpr() instanceof Literal and
-          g = globalValueNumber(v.getAnAccess())
+          src.asExpr() = v.getAnAccess()
         )
         or
         // result of a function call with literal inputs (likely constant)
+        forex(Expr arg | arg = src.asExpr().(FunctionCall).getAnArgument() | arg instanceof Literal)
+        or
+        // variable called `stdin`, `stdout` or `stderr`
+        src.asExpr().(VariableAccess).getTarget().getName() = ["stdin", "stdout", "stderr"]
+        or
+        // open of `"/dev/tty"`
         exists(FunctionCall fc |
-          forex(Expr arg | arg = fc.getAnArgument() | arg instanceof Literal) and
-          g = globalValueNumber(fc)
+          fopenCall(fc) and
+          fc.getAnArgument().getValue() = "/dev/tty" and
+          src.asExpr() = fc
         )
-        // (this is far from exhaustive)
+        // (this is not exhaustive)
       )
     )
   }
@@ -160,6 +168,16 @@ class NetworkRecv extends NetworkSendRecv {
   override Recv target;
 }
 
+pragma[noinline]
+predicate encryptionFunction(Function f) {
+  f.getName().toLowerCase().regexpMatch(".*(crypt|encode|decode|hash|securezero).*")
+}
+
+pragma[noinline]
+predicate encryptionType(UserType t) {
+  t.getName().toLowerCase().regexpMatch(".*(crypt|encode|decode|hash|securezero).*")
+}
+
 /**
  * An expression that is an argument or return value from an encryption /
  * decryption call. This is quite inclusive to minimize false positives, for
@@ -169,10 +187,7 @@ class NetworkRecv extends NetworkSendRecv {
 class Encrypted extends Expr {
   Encrypted() {
     exists(FunctionCall fc |
-      fc.getTarget()
-          .getName()
-          .toLowerCase()
-          .regexpMatch(".*(crypt|encode|decode|hash|securezero).*") and
+      encryptionFunction(fc.getTarget()) and
       (
         this = fc or
         this = fc.getAnArgument()
@@ -181,7 +196,7 @@ class Encrypted extends Expr {
     or
     exists(Type t |
       this.getType().refersTo(t) and
-      t.getName().toLowerCase().regexpMatch(".*(crypt|encode|decode|hash|securezero).*")
+      encryptionType(t)
     )
   }
 }

cpp/ql/src/Security/CWE/CWE-732/DoNotCreateWorldWritable.ql

@@ -12,17 +12,16 @@
 
 import cpp
 import FilePermissions
-import semmle.code.cpp.commons.unix.Constants
 
 predicate worldWritableCreation(FileCreationExpr fc, int mode) {
   mode = localUmask(fc).mask(fc.getMode()) and
-  sets(mode, s_iwoth())
+  setsAnyBits(mode, UnixConstants::s_iwoth())
 }
 
 predicate setWorldWritable(FunctionCall fc, int mode) {
   fc.getTarget().getName() = ["chmod", "fchmod", "_chmod", "_wchmod"] and
   mode = fc.getArgument(1).getValue().toInt() and
-  sets(mode, s_iwoth())
+  setsAnyBits(mode, UnixConstants::s_iwoth())
 }
 
 from Expr fc, int mode, string message

cpp/ql/src/Security/CWE/CWE-732/FilePermissions.qll

@@ -1,5 +1,49 @@
 import cpp
-import semmle.code.cpp.commons.unix.Constants
+import semmle.code.cpp.commons.unix.Constants as UnixConstants
+
+/**
+ * Gets the number corresponding to the contents of `input` in base-16.
+ * Note: the first two characters of `input` must be `0x`. For example:
+ * `parseHex("0x123abc") = 1194684`.
+ */
+bindingset[input]
+int parseHex(string input) {
+  exists(string lowerCaseInput | lowerCaseInput = input.toLowerCase() |
+    lowerCaseInput.regexpMatch("0x[0-9a-f]+") and
+    result =
+      strictsum(int ix |
+        ix in [2 .. input.length()]
+      |
+        16.pow(input.length() - (ix + 1)) * "0123456789abcdef".indexOf(lowerCaseInput.charAt(ix))
+      )
+  )
+}
+
+/**
+ * Gets the value defined by the `O_CREAT` macro if the macro
+ * exists and if every definition defines the same value.
+ */
+int o_creat() {
+  result =
+    unique(int v |
+      exists(Macro m | m.getName() = "O_CREAT" |
+        v = parseHex(m.getBody()) or v = UnixConstants::parseOctal(m.getBody())
+      )
+    )
+}
+
+/**
+ * Gets the value defined by the `O_TMPFILE` macro if the macro
+ * exists and if every definition defines the same value.
+ */
+int o_tmpfile() {
+  result =
+    unique(int v |
+      exists(Macro m | m.getName() = "O_TMPFILE" |
+        v = parseHex(m.getBody()) or v = UnixConstants::parseOctal(m.getBody())
+      )
+    )
+}
 
 bindingset[n, digit]
 private string octalDigit(int n, int digit) {
@@ -20,11 +64,17 @@ string octalFileMode(int mode) {
   else result = "[non-standard mode: decimal " + mode + "]"
 }
 
+/**
+ * Holds if the bitmask `value` sets the bits in `flag`.
+ */
+bindingset[value, flag]
+predicate setsFlag(int value, int flag) { value.bitAnd(flag) = flag }
+
 /**
  * Holds if the bitmask `mask` sets any of the bit fields in `fields`.
  */
 bindingset[mask, fields]
-predicate sets(int mask, int fields) { mask.bitAnd(fields) != 0 }
+predicate setsAnyBits(int mask, int fields) { mask.bitAnd(fields) != 0 }
 
 /**
  * Gets the value that `fc` sets the umask to, if `fc` is a call to
@@ -83,16 +133,24 @@ abstract class FileCreationExpr extends FunctionCall {
   abstract int getMode();
 }
 
-class OpenCreationExpr extends FileCreationExpr {
+abstract class FileCreationWithOptionalModeExpr extends FileCreationExpr {
+  abstract predicate hasModeArgument();
+}
+
+class OpenCreationExpr extends FileCreationWithOptionalModeExpr {
   OpenCreationExpr() {
-    this.getTarget().getName() = ["open", "_open", "_wopen"] and
-    sets(this.getArgument(1).getValue().toInt(), o_creat())
+    this.getTarget().hasGlobalOrStdName(["open", "_open", "_wopen"]) and
+    exists(int flag | flag = this.getArgument(1).getValue().toInt() |
+      setsFlag(flag, o_creat()) or setsFlag(flag, o_tmpfile())
+    )
   }
 
   override Expr getPath() { result = this.getArgument(0) }
 
+  override predicate hasModeArgument() { exists(this.getArgument(2)) }
+
   override int getMode() {
-    if exists(this.getArgument(2))
+    if this.hasModeArgument()
     then result = this.getArgument(2).getValue().toInt()
     else
       // assume anything is permitted
@@ -108,20 +166,35 @@ class CreatCreationExpr extends FileCreationExpr {
   override int getMode() { result = this.getArgument(1).getValue().toInt() }
 }
 
-class OpenatCreationExpr extends FileCreationExpr {
+class OpenatCreationExpr extends FileCreationWithOptionalModeExpr {
   OpenatCreationExpr() {
-    this.getTarget().getName() = "openat" and
-    this.getNumberOfArguments() = 4
+    this.getTarget().hasGlobalOrStdName("openat") and
+    exists(int flag | flag = this.getArgument(2).getValue().toInt() |
+      setsFlag(flag, o_creat()) or setsFlag(flag, o_tmpfile())
+    )
   }
 
   override Expr getPath() { result = this.getArgument(1) }
 
-  override int getMode() { result = this.getArgument(3).getValue().toInt() }
+  override predicate hasModeArgument() { exists(this.getArgument(3)) }
+
+  override int getMode() {
+    if this.hasModeArgument()
+    then result = this.getArgument(3).getValue().toInt()
+    else
+      // assume anything is permitted
+      result = 0.bitNot()
+  }
 }
 
 private int fopenMode() {
   result =
-    s_irusr().bitOr(s_irgrp()).bitOr(s_iroth()).bitOr(s_iwusr()).bitOr(s_iwgrp()).bitOr(s_iwoth())
+    UnixConstants::s_irusr()
+        .bitOr(UnixConstants::s_irgrp())
+        .bitOr(UnixConstants::s_iroth())
+        .bitOr(UnixConstants::s_iwusr())
+        .bitOr(UnixConstants::s_iwgrp())
+        .bitOr(UnixConstants::s_iwoth())
 }
 
 class FopenCreationExpr extends FileCreationExpr {
@@ -153,6 +226,6 @@ class FopensCreationExpr extends FileCreationExpr {
     // fopen_s has restrictive permissions unless you have "u" in the mode
     if this.getArgument(2).getValue().charAt(_) = "u"
     then result = fopenMode()
-    else result = s_irusr().bitOr(s_iwusr())
+    else result = UnixConstants::s_irusr().bitOr(UnixConstants::s_iwusr())
   }
 }

cpp/ql/src/Security/CWE/CWE-732/OpenCallMissingModeArgument.c

@@ -0,0 +1,9 @@
+int open_file_bad() {
+	// BAD - this uses arbitrary bytes from the stack as mode argument
+        return open(FILE, O_CREAT)
+}
+
+int open_file_good() {
+	// GOOD - the mode argument is supplied
+        return open(FILE, O_CREAT, S_IRUSR | S_IWUSR)
+}

cpp/ql/src/Security/CWE/CWE-732/OpenCallMissingModeArgument.qhelp

@@ -0,0 +1,31 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p>
+When opening a file with the <code>O_CREAT</code> or <code>O_TMPFILE</code> flag, the <code>mode</code> must
+be supplied. If the <code>mode</code> argument is omitted, some arbitrary bytes from the stack will be used
+as the file mode. This leaks some bits from the stack into the permissions of the file.
+</p>
+</overview>
+
+<recommendation>
+<p>
+The <code>mode</code> must be supplied when <code>O_CREAT</code> or <code>O_TMPFILE</code> is specified.
+</p>
+</recommendation>
+
+<example>
+<p>
+The first example opens a file with the <code>O_CREAT</code> flag without supplying the <code>mode</code>
+argument. In this case arbitrary bytes from the stack will be used as <code>mode</code> argument. The
+second example correctly supplies the <code>mode</code> argument and creates a file that is user readable
+and writable.
+</p>
+
+<sample src="OpenCallMissingModeArgument.c" />
+
+</example>
+</qhelp>

cpp/ql/src/Security/CWE/CWE-732/OpenCallMissingModeArgument.ql

@@ -0,0 +1,19 @@
+/**
+ * @name File opened with O_CREAT flag but without mode argument
+ * @description Opening a file with the O_CREAT flag but without mode argument reads arbitrary bytes from the stack.
+ * @kind problem
+ * @problem.severity error
+ * @security-severity 7.8
+ * @precision high
+ * @id cpp/open-call-with-mode-argument
+ * @tags security
+ *       external/cwe/cwe-732
+ */
+
+import cpp
+import FilePermissions
+
+from FileCreationWithOptionalModeExpr fc
+where not fc.hasModeArgument()
+select fc,
+  "A file is created here without providing a mode argument, which may leak bits from the stack."

cpp/ql/src/change-notes/2021-12-14-overruning-write-split.md

@@ -1,4 +0,0 @@
----
-category: newQuery
----
-* A new `cpp/very-likely-overruning-write` query has been added to the default query suite for C/C++. The query reports some results that were formerly flagged by `cpp/overruning-write`.

cpp/ql/src/change-notes/2021-12-30-ambiguously-signed-bit-field.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* Added exception for GLib's gboolean to cpp/ambiguously-signed-bit-field.
-  This change reduces the number of false positives in the query.

cpp/ql/src/change-notes/2022-01-05-promote-uncontrolled-arithmetic.md

@@ -1,4 +0,0 @@
----
-category: newQuery
----
-* The "Uncontrolled data in arithmetic expression" (cpp/uncontrolled-arithmetic) query has been enhanced to reduce false positive results and its @precision increased to high.

cpp/ql/src/change-notes/2022-01-18-improper-null-termination.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The "Potential improper null termination" (`cpp/improper-null-termination`) query now produces fewer false positive results around control flow branches and loops.

cpp/ql/src/change-notes/2022-01-19-cleartext-transmission.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The "Cleartext transmission of sensitive information" (`cpp/cleartext-transmission`) query has been improved in several ways to reduce false positive results.

cpp/ql/src/change-notes/2022-01-20-return-stack-allocated-memory.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* The `cpp/return-stack-allocated-memory` query has been improved to produce fewer false positives. The
-  query has also been converted to a `path-problem` query.

cpp/ql/src/change-notes/2022-01-24-cleartext-storage-file.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The `cpp/cleartext-storage-file` query has been upgraded with non-local taint flow and has been converted to a `path-problem` query.

cpp/ql/src/change-notes/2022-01-24-return-stack-allocated-memory.md

@@ -1,4 +0,0 @@
----
-category: newQuery
----
-* The `security` tag has been added to the `cpp/return-stack-allocated-memory` query. As a result, its results will now appear by default.

cpp/ql/src/change-notes/2022-01-26-unnamed-variable-declaration-hiding.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* Fix an issue with the `cpp/declaration-hides-variable` query where it would report variables that are unnamed in a database.

cpp/ql/src/change-notes/2022-02-22-cleartext-transmission.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The "Cleartext transmission of sensitive information" (`cpp/cleartext-transmission`) query has been further improved to reduce false positive results, and upgraded from `medium` to `high` precision.

cpp/ql/src/change-notes/released/0.0.8.md

@@ -0,0 +1,18 @@
+## 0.0.8
+
+### New Queries
+
+* The `security` tag has been added to the `cpp/return-stack-allocated-memory` query. As a result, its results will now appear by default.
+* The "Uncontrolled data in arithmetic expression" (cpp/uncontrolled-arithmetic) query has been enhanced to reduce false positive results and its @precision increased to high.
+* A new `cpp/very-likely-overruning-write` query has been added to the default query suite for C/C++. The query reports some results that were formerly flagged by `cpp/overruning-write`.
+
+### Minor Analysis Improvements
+
+* Fix an issue with the `cpp/declaration-hides-variable` query where it would report variables that are unnamed in a database.
+* The `cpp/cleartext-storage-file` query has been upgraded with non-local taint flow and has been converted to a `path-problem` query.
+* The `cpp/return-stack-allocated-memory` query has been improved to produce fewer false positives. The
+  query has also been converted to a `path-problem` query.
+* The "Cleartext transmission of sensitive information" (`cpp/cleartext-transmission`) query has been improved in several ways to reduce false positive results.
+* The "Potential improper null termination" (`cpp/improper-null-termination`) query now produces fewer false positive results around control flow branches and loops.
+* Added exception for GLib's gboolean to cpp/ambiguously-signed-bit-field.
+  This change reduces the number of false positives in the query.

cpp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.0.7
+lastReleaseVersion: 0.0.8

cpp/ql/src/experimental/Best Practices/UselessTest.ql

@@ -38,6 +38,8 @@ where
   ) and
   eq.getRightOperand() instanceof Literal and
   ne.getRightOperand() instanceof Literal and
+  eq.getLeftOperand().getFullyConverted().getUnspecifiedType() =
+    ne.getLeftOperand().getFullyConverted().getUnspecifiedType() and
   nearestParent(eq) = nearestParent(ne) and
   sameExpr(eq.getLeftOperand(), ne.getLeftOperand())
 select ne, "Useless Test"

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 0.0.8-dev
+version: 0.0.9-dev
 groups: 
   - cpp
   - queries

cpp/ql/test/TestUtilities/InlineExpectationsTest.qll

@@ -123,6 +123,15 @@ abstract class InlineExpectationsTest extends string {
    */
   abstract predicate hasActualResult(Location location, string element, string tag, string value);
 
+  /**
+   * Like `hasActualResult`, but returns results that do not require a matching annotation.
+   * A failure will still arise if there is an annotation that does not match any results, but not vice versa.
+   * Override this predicate to specify optional results.
+   */
+  predicate hasOptionalResult(Location location, string element, string tag, string value) {
+    none()
+  }
+
   final predicate hasFailureMessage(FailureLocatable element, string message) {
     exists(ActualResult actualResult |
       actualResult.getTest() = this and
@@ -134,7 +143,8 @@ abstract class InlineExpectationsTest extends string {
         )
         or
         not exists(ValidExpectation expectation | expectation.matchesActualResult(actualResult)) and
-        message = "Unexpected result: " + actualResult.getExpectationText()
+        message = "Unexpected result: " + actualResult.getExpectationText() and
+        not actualResult.isOptional()
       )
     )
     or
@@ -243,9 +253,13 @@ private string expectationPattern() {
 
 private newtype TFailureLocatable =
   TActualResult(
-    InlineExpectationsTest test, Location location, string element, string tag, string value
+    InlineExpectationsTest test, Location location, string element, string tag, string value,
+    boolean optional
   ) {
-    test.hasActualResult(location, element, tag, value)
+    test.hasActualResult(location, element, tag, value) and
+    optional = false
+    or
+    test.hasOptionalResult(location, element, tag, value) and optional = true
   } or
   TValidExpectation(ExpectationComment comment, string tag, string value, string knownFailure) {
     exists(TColumn column, string tags |
@@ -277,8 +291,9 @@ class ActualResult extends FailureLocatable, TActualResult {
   string element;
   string tag;
   string value;
+  boolean optional;
 
-  ActualResult() { this = TActualResult(test, location, element, tag, value) }
+  ActualResult() { this = TActualResult(test, location, element, tag, value, optional) }
 
   override string toString() { result = element }
 
@@ -289,6 +304,8 @@ class ActualResult extends FailureLocatable, TActualResult {
   override string getTag() { result = tag }
 
   override string getValue() { result = value }
+
+  predicate isOptional() { optional = true }
 }
 
 abstract private class Expectation extends FailureLocatable {

cpp/ql/test/library-tests/ir/ir/PrintAST.expected

@@ -11332,6 +11332,64 @@ ir.cpp:
 # 1447|                         Type = [Struct] POD_Derived
 # 1447|                         ValueCategory = prvalue
 # 1448|     getStmt(4): [ReturnStmt] return ...
+# 1450| [CopyAssignmentOperator] Inheritance_Test_B& Inheritance_Test_B::operator=(Inheritance_Test_B const&)
+# 1450|   <params>: 
+#-----|     getParameter(0): [Parameter] (unnamed parameter 0)
+#-----|         Type = [LValueReferenceType] const Inheritance_Test_B &
+# 1450| [Constructor] void Inheritance_Test_B::Inheritance_Test_B()
+# 1450|   <params>: 
+# 1451| [Destructor] void Inheritance_Test_B::~Inheritance_Test_B()
+# 1451|   <params>: 
+# 1451|   getEntryPoint(): [BlockStmt] { ... }
+# 1451|     getStmt(0): [ReturnStmt] return ...
+# 1451|   <destructions>: 
+# 1454| [CopyAssignmentOperator] Inheritance_Test_A& Inheritance_Test_A::operator=(Inheritance_Test_A const&)
+# 1454|   <params>: 
+#-----|     getParameter(0): [Parameter] (unnamed parameter 0)
+#-----|         Type = [LValueReferenceType] const Inheritance_Test_A &
+# 1454| [MoveAssignmentOperator] Inheritance_Test_A& Inheritance_Test_A::operator=(Inheritance_Test_A&&)
+# 1454|   <params>: 
+#-----|     getParameter(0): [Parameter] (unnamed parameter 0)
+#-----|         Type = [RValueReferenceType] Inheritance_Test_A &&
+# 1454| [CopyConstructor] void Inheritance_Test_A::Inheritance_Test_A(Inheritance_Test_A const&)
+# 1454|   <params>: 
+#-----|     getParameter(0): [Parameter] (unnamed parameter 0)
+#-----|         Type = [LValueReferenceType] const Inheritance_Test_A &
+# 1454| [MoveConstructor] void Inheritance_Test_A::Inheritance_Test_A(Inheritance_Test_A&&)
+# 1454|   <params>: 
+#-----|     getParameter(0): [Parameter] (unnamed parameter 0)
+#-----|         Type = [RValueReferenceType] Inheritance_Test_A &&
+# 1454| [Destructor] void Inheritance_Test_A::~Inheritance_Test_A()
+# 1454|   <params>: 
+# 1457| [Constructor] void Inheritance_Test_A::Inheritance_Test_A()
+# 1457|   <params>: 
+# 1457|   <initializations>: 
+# 1457|     getInitializer(0): (no string representation)
+# 1457|         Type = [Struct] Inheritance_Test_B
+# 1457|         ValueCategory = prvalue
+# 1457|     getInitializer(1): [ConstructorFieldInit] constructor init of field x
+# 1457|         Type = [IntType] int
+# 1457|         ValueCategory = prvalue
+# 1457|       getExpr(): [Literal] 42
+# 1457|           Type = [IntType] int
+# 1457|           Value = [Literal] 42
+# 1457|           ValueCategory = prvalue
+# 1457|   getEntryPoint(): [BlockStmt] { ... }
+# 1458|     getStmt(0): [ExprStmt] ExprStmt
+# 1458|       getExpr(): [AssignExpr] ... = ...
+# 1458|           Type = [IntType] int
+# 1458|           ValueCategory = lvalue
+# 1458|         getLValue(): [PointerFieldAccess] y
+# 1458|             Type = [IntType] int
+# 1458|             ValueCategory = lvalue
+# 1458|           getQualifier(): [ThisExpr] this
+# 1458|               Type = [PointerType] Inheritance_Test_A *
+# 1458|               ValueCategory = prvalue(load)
+# 1458|         getRValue(): [Literal] 3
+# 1458|             Type = [IntType] int
+# 1458|             Value = [Literal] 3
+# 1458|             ValueCategory = prvalue
+# 1459|     getStmt(1): [ReturnStmt] return ...
 perf-regression.cpp:
 #    4| [CopyAssignmentOperator] Big& Big::operator=(Big const&)
 #    4|   <params>: 

cpp/ql/test/library-tests/ir/ir/ir.cpp

@@ -1447,4 +1447,15 @@ void temporary_hierarchy() {
     float f = (returnValue<POD_Derived>()).f();
 }
 
+struct Inheritance_Test_B {
+  ~Inheritance_Test_B() {}
+};
+
+struct Inheritance_Test_A : public Inheritance_Test_B {
+  int x;
+  int y;
+  Inheritance_Test_A() : x(42) {
+    y = 3;
+  }
+};
 // semmle-extractor-options: -std=c++17 --clang

cpp/ql/test/library-tests/ir/ir/operand_locations.expected

@@ -6626,6 +6626,36 @@
 | ir.cpp:1447:44:1447:44 | ChiTotal | total:m1447_5 |
 | ir.cpp:1447:44:1447:44 | SideEffect | ~m1447_5 |
 | ir.cpp:1447:44:1447:44 | StoreValue | r1447_8 |
+| ir.cpp:1451:3:1451:21 | Address | &:r1451_5 |
+| ir.cpp:1451:3:1451:21 | Address | &:r1451_5 |
+| ir.cpp:1451:3:1451:21 | Address | &:r1451_7 |
+| ir.cpp:1451:3:1451:21 | Address | &:r1451_7 |
+| ir.cpp:1451:3:1451:21 | ChiPartial | partial:m1451_3 |
+| ir.cpp:1451:3:1451:21 | ChiTotal | total:m1451_2 |
+| ir.cpp:1451:3:1451:21 | Load | m1451_6 |
+| ir.cpp:1451:3:1451:21 | SideEffect | m1451_3 |
+| ir.cpp:1451:3:1451:21 | SideEffect | m1451_8 |
+| ir.cpp:1457:3:1457:20 | Address | &:r1457_5 |
+| ir.cpp:1457:3:1457:20 | Address | &:r1457_5 |
+| ir.cpp:1457:3:1457:20 | Address | &:r1457_7 |
+| ir.cpp:1457:3:1457:20 | Address | &:r1457_7 |
+| ir.cpp:1457:3:1457:20 | ChiPartial | partial:m1457_3 |
+| ir.cpp:1457:3:1457:20 | ChiTotal | total:m1457_2 |
+| ir.cpp:1457:3:1457:20 | Load | m1457_6 |
+| ir.cpp:1457:3:1457:20 | SideEffect | m1457_3 |
+| ir.cpp:1457:3:1457:20 | SideEffect | m1458_6 |
+| ir.cpp:1457:3:1457:20 | Unary | m1457_6 |
+| ir.cpp:1457:26:1457:30 | Address | &:r1457_9 |
+| ir.cpp:1457:26:1457:30 | ChiPartial | partial:m1457_11 |
+| ir.cpp:1457:26:1457:30 | ChiTotal | total:m1457_8 |
+| ir.cpp:1457:26:1457:30 | StoreValue | r1457_10 |
+| ir.cpp:1458:5:1458:5 | Address | &:r1458_2 |
+| ir.cpp:1458:5:1458:5 | Address | &:r1458_4 |
+| ir.cpp:1458:5:1458:5 | Load | m1457_6 |
+| ir.cpp:1458:5:1458:5 | Unary | r1458_3 |
+| ir.cpp:1458:5:1458:9 | ChiPartial | partial:m1458_5 |
+| ir.cpp:1458:5:1458:9 | ChiTotal | total:m1457_12 |
+| ir.cpp:1458:9:1458:9 | StoreValue | r1458_1 |
 | perf-regression.cpp:6:3:6:5 | Address | &:r6_5 |
 | perf-regression.cpp:6:3:6:5 | Address | &:r6_5 |
 | perf-regression.cpp:6:3:6:5 | Address | &:r6_7 |

cpp/ql/test/library-tests/ir/ir/raw_ir.expected

@@ -7858,6 +7858,44 @@ ir.cpp:
 # 1443|     v1443_5(void)               = AliasedUse                                        : ~m?
 # 1443|     v1443_6(void)               = ExitFunction                                      : 
 
+# 1451| void Inheritance_Test_B::~Inheritance_Test_B()
+# 1451|   Block 0
+# 1451|     v1451_1(void)                       = EnterFunction                : 
+# 1451|     mu1451_2(unknown)                   = AliasedDefinition            : 
+# 1451|     mu1451_3(unknown)                   = InitializeNonLocal           : 
+# 1451|     r1451_4(glval<unknown>)             = VariableAddress[#this]       : 
+# 1451|     mu1451_5(glval<Inheritance_Test_B>) = InitializeParameter[#this]   : &:r1451_4
+# 1451|     r1451_6(glval<Inheritance_Test_B>)  = Load[#this]                  : &:r1451_4, ~m?
+# 1451|     mu1451_7(Inheritance_Test_B)        = InitializeIndirection[#this] : &:r1451_6
+# 1451|     v1451_8(void)                       = NoOp                         : 
+# 1451|     v1451_9(void)                       = ReturnIndirection[#this]     : &:r1451_6, ~m?
+# 1451|     v1451_10(void)                      = ReturnVoid                   : 
+# 1451|     v1451_11(void)                      = AliasedUse                   : ~m?
+# 1451|     v1451_12(void)                      = ExitFunction                 : 
+
+# 1457| void Inheritance_Test_A::Inheritance_Test_A()
+# 1457|   Block 0
+# 1457|     v1457_1(void)                       = EnterFunction                : 
+# 1457|     mu1457_2(unknown)                   = AliasedDefinition            : 
+# 1457|     mu1457_3(unknown)                   = InitializeNonLocal           : 
+# 1457|     r1457_4(glval<unknown>)             = VariableAddress[#this]       : 
+# 1457|     mu1457_5(glval<Inheritance_Test_A>) = InitializeParameter[#this]   : &:r1457_4
+# 1457|     r1457_6(glval<Inheritance_Test_A>)  = Load[#this]                  : &:r1457_4, ~m?
+# 1457|     mu1457_7(Inheritance_Test_A)        = InitializeIndirection[#this] : &:r1457_6
+# 1457|     r1457_8(glval<int>)                 = FieldAddress[x]              : mu1457_5
+# 1457|     r1457_9(int)                        = Constant[42]                 : 
+# 1457|     mu1457_10(int)                      = Store[?]                     : &:r1457_8, r1457_9
+# 1458|     r1458_1(int)                        = Constant[3]                  : 
+# 1458|     r1458_2(glval<unknown>)             = VariableAddress[#this]       : 
+# 1458|     r1458_3(Inheritance_Test_A *)       = Load[#this]                  : &:r1458_2, ~m?
+# 1458|     r1458_4(glval<int>)                 = FieldAddress[y]              : r1458_3
+# 1458|     mu1458_5(int)                       = Store[?]                     : &:r1458_4, r1458_1
+# 1459|     v1459_1(void)                       = NoOp                         : 
+# 1457|     v1457_11(void)                      = ReturnIndirection[#this]     : &:r1457_6, ~m?
+# 1457|     v1457_12(void)                      = ReturnVoid                   : 
+# 1457|     v1457_13(void)                      = AliasedUse                   : ~m?
+# 1457|     v1457_14(void)                      = ExitFunction                 : 
+
 perf-regression.cpp:
 #    6| void Big::Big()
 #    6|   Block 0

cpp/ql/test/library-tests/special_members/generated_copy/assign.expected

@@ -11,7 +11,6 @@
 | difference::Base | can | does NOT | have implicit copy assignment |
 | difference::OnlyAssign | can | does | have implicit copy assignment |
 | difference::OnlyCtor | can NOT | does NOT | have implicit copy assignment |
-| instantiated_explicit_ctor::Wrapper<int> | can | does | have implicit copy assignment |
 | moves::MoveAssign | can NOT | does NOT | have implicit copy assignment |
 | moves::MoveCtor | can NOT | does NOT | have implicit copy assignment |
 | private_cc::C | can | does NOT | have implicit copy assignment |

cpp/ql/test/library-tests/special_members/generated_copy/copy.cpp

@@ -131,21 +131,3 @@ namespace difference {
   class OnlyAssign : Base {
   };
 }
-
-namespace instantiated_explicit_ctor {
-  template<class T>
-  class Wrapper {
-  public:
-    Wrapper(Wrapper<T> &other) {
-      m_t = other.m_t;
-    }
-    
-    Wrapper() {
-      m_t = 0;
-    }
-  private:
-    T m_t;
-  };
-
-  Wrapper<int> wrapped_int;
-}

cpp/ql/test/library-tests/special_members/generated_copy/ctor.expected

@@ -11,7 +11,6 @@
 | difference::Base | can | does NOT | have implicit copy constructor |
 | difference::OnlyAssign | can NOT | does NOT | have implicit copy constructor |
 | difference::OnlyCtor | can | does | have implicit copy constructor |
-| instantiated_explicit_ctor::Wrapper<int> | can | does NOT | have implicit copy constructor |
 | moves::MoveAssign | can NOT | does NOT | have implicit copy constructor |
 | moves::MoveCtor | can NOT | does NOT | have implicit copy constructor |
 | private_cc::C | can | does NOT | have implicit copy constructor |

cpp/ql/test/library-tests/special_members/generated_copy/functions.expected

@@ -86,9 +86,5 @@
 | copy.cpp:131:9:131:9 | OnlyAssign | deleted |  |
 | copy.cpp:131:9:131:9 | operator= |  |  |
 | copy.cpp:131:9:131:9 | operator= |  |  |
-| copy.cpp:137:9:137:9 | operator= |  |  |
-| copy.cpp:139:5:139:11 | Wrapper |  |  |
-| copy.cpp:143:5:143:5 | Wrapper |  |  |
-| copy.cpp:143:5:143:11 | Wrapper |  |  |
 | file://:0:0:0:0 | operator= |  |  |
 | file://:0:0:0:0 | operator= |  |  |

cpp/ql/test/query-tests/Best Practices/RuleOfTwo/RuleOfTwo.cpp

@@ -117,8 +117,12 @@ struct HasVPV {
   }
 };
 
-// NOT OK: the relevant copy constructor of ProtectedVolatile is
-// accessible, so our class will get a generated copy constructor.
+// FALSE NEGATIVE: the relevant copy constructor of ProtectedVolatile is
+// accessible, so our class will get a generated copy constructor. Our query
+// thinks the copy constructor is inaccessible because it picks up the other
+// copy constructor. To fix this, our library should be changed to distinguish
+// between copy constructors and resolve overloading properly instead of
+// assuming that there is at most one.
 struct HasPV {
   ProtectedVolatile pv;
   HasPV& operator=(const HasPV& that) {

cpp/ql/test/query-tests/Best Practices/RuleOfTwo/RuleOfTwo.expected

@@ -1,6 +1,6 @@
 | RuleOfTwo.cpp:4:3:4:17 | CopyButNoAssign | No matching copy assignment operator in class CopyButNoAssign. It is good practice to match a copy constructor with a copy assignment operator. |
 | RuleOfTwo.cpp:10:20:10:28 | operator= | No matching copy constructor in class AssignButNoCopy. It is good practice to match a copy assignment operator with a copy constructor. |
 | RuleOfTwo.cpp:81:18:81:26 | operator= | No matching copy constructor in class MyClassFriend. It is good practice to match a copy assignment operator with a copy constructor. |
-| RuleOfTwo.cpp:140:3:140:20 | IsAProtectedAssign | No matching copy assignment operator in class IsAProtectedAssign. It is good practice to match a copy constructor with a copy assignment operator. |
-| RuleOfTwo.cpp:163:19:163:27 | operator= | No matching copy constructor in class IsAProtectedCC. It is good practice to match a copy assignment operator with a copy constructor. |
-| RuleOfTwo.cpp:308:5:308:8 | R1_C | No matching copy assignment operator in class R1_C. It is good practice to match a copy constructor with a copy assignment operator. |
+| RuleOfTwo.cpp:144:3:144:20 | IsAProtectedAssign | No matching copy assignment operator in class IsAProtectedAssign. It is good practice to match a copy constructor with a copy assignment operator. |
+| RuleOfTwo.cpp:167:19:167:27 | operator= | No matching copy constructor in class IsAProtectedCC. It is good practice to match a copy assignment operator with a copy constructor. |
+| RuleOfTwo.cpp:312:5:312:8 | R1_C | No matching copy assignment operator in class R1_C. It is good practice to match a copy constructor with a copy assignment operator. |

cpp/ql/test/query-tests/Security/CWE/CWE-311/semmle/tests/CleartextTransmission.expected

@@ -91,6 +91,7 @@ edges
 | test3.cpp:398:18:398:25 | password | test3.cpp:400:16:400:23 | password |
 | test3.cpp:398:18:398:25 | password | test3.cpp:400:33:400:40 | password |
 | test3.cpp:421:21:421:28 | password | test3.cpp:421:3:421:17 | call to decrypt_inplace |
+| test3.cpp:429:7:429:14 | password | test3.cpp:431:8:431:15 | password |
 | test.cpp:41:23:41:43 | cleartext password! | test.cpp:48:21:48:27 | call to encrypt |
 | test.cpp:41:23:41:43 | cleartext password! | test.cpp:48:29:48:39 | thePassword |
 | test.cpp:66:23:66:43 | cleartext password! | test.cpp:76:21:76:27 | call to encrypt |
@@ -218,6 +219,8 @@ nodes
 | test3.cpp:421:3:421:17 | call to decrypt_inplace | semmle.label | call to decrypt_inplace |
 | test3.cpp:421:21:421:28 | password | semmle.label | password |
 | test3.cpp:421:21:421:28 | password | semmle.label | password |
+| test3.cpp:429:7:429:14 | password | semmle.label | password |
+| test3.cpp:431:8:431:15 | password | semmle.label | password |
 | test.cpp:41:23:41:43 | cleartext password! | semmle.label | cleartext password! |
 | test.cpp:48:21:48:27 | call to encrypt | semmle.label | call to encrypt |
 | test.cpp:48:29:48:39 | thePassword | semmle.label | thePassword |
@@ -250,3 +253,4 @@ subpaths
 | test3.cpp:388:3:388:6 | call to recv | test3.cpp:386:8:386:15 | password | test3.cpp:388:15:388:22 | password | This operation receives into 'password', which may put unencrypted sensitive data into $@ | test3.cpp:386:8:386:15 | password | password |
 | test3.cpp:414:3:414:6 | call to recv | test3.cpp:414:17:414:24 | password | test3.cpp:414:17:414:24 | password | This operation receives into 'password', which may put unencrypted sensitive data into $@ | test3.cpp:414:17:414:24 | password | password |
 | test3.cpp:420:3:420:6 | call to recv | test3.cpp:420:17:420:24 | password | test3.cpp:420:17:420:24 | password | This operation receives into 'password', which may put unencrypted sensitive data into $@ | test3.cpp:420:17:420:24 | password | password |
+| test3.cpp:431:2:431:6 | call to fgets | test3.cpp:429:7:429:14 | password | test3.cpp:431:8:431:15 | password | This operation receives into 'password', which may put unencrypted sensitive data into $@ | test3.cpp:429:7:429:14 | password | password |

cpp/ql/test/query-tests/Security/CWE/CWE-311/semmle/tests/test3.cpp

@@ -421,3 +421,56 @@ void test_member_password()
 		decrypt_inplace(p.password); // proof that `password` was in fact encrypted
 	}
 }
+
+extern FILE *stdin;
+
+void test_stdin_param(FILE *stream)
+{
+	char password[128];
+
+	fgets(password, 128, stream); // GOOD: from standard input (see call below) [FALSE POSITIVE]
+}
+
+void test_stdin()
+{
+	char password[128];
+	FILE *f = stdin;
+
+	fgets(password, 128, stdin); // GOOD: from standard input
+	fgets(password, 128, f); // GOOD: from standard input
+	test_stdin_param(stdin);
+}
+
+int open(const char *filename, int b);
+
+void test_tty()
+{
+	{
+		char password[256];
+		int f;
+
+		f = open("/dev/tty", val());
+		recv(f, password, 256, val()); // GOOD: from terminal
+	}
+
+	{
+		char password[256];
+		int f;
+
+		f = STDIN_FILENO;
+		recv(f, password, 256, val()); // GOOD: from stdin
+	}
+
+	{
+		char password[256];
+		int f;
+
+		f = open("/dev/tty", val());
+		if (f == -1)
+		{
+			f = STDIN_FILENO;
+		}
+
+		recv(f, password, 256, val()); // GOOD: from terminal or stdin
+	}
+}

cpp/ql/test/query-tests/Security/CWE/CWE-732/OpenCallMissingModeArgument.c

@@ -0,0 +1,29 @@
+typedef unsigned int mode_t;
+
+#define O_RDWR     0x0002
+#define O_CLOEXEC  0x0040
+#define O_NONBLOCK 0x0080
+#define O_CREAT    0x0200
+#define O_APPEND   0x0800
+#define O_TMPFILE  0x2000
+
+int open(const char *pathname, int flags, ...);
+
+int openat(int dirfd, const char *pathname, int flags, ...);
+
+const char *a_file = "/a_file";
+
+void test_open() {
+  open(a_file, O_NONBLOCK); // GOOD
+  open(a_file, O_RDWR | O_CLOEXEC); // GOOD
+  open(a_file, O_APPEND); // GOOD
+  open(a_file, O_CREAT); // BAD
+  open(a_file, O_CREAT, 0); // GOOD
+  open(a_file, O_TMPFILE); // BAD
+  open(a_file, O_TMPFILE, 0); // GOOD
+  openat(0, a_file, O_APPEND); // GOOD
+  openat(0, a_file, O_CREAT); // BAD
+  openat(0, a_file, O_CREAT, 0); // GOOD
+  openat(0, a_file, O_TMPFILE); // BAD
+  openat(0, a_file, O_TMPFILE, 0); // GOOD
+}

cpp/ql/test/query-tests/Security/CWE/CWE-732/OpenCallMissingModeArgument.expected

@@ -0,0 +1,4 @@
+| OpenCallMissingModeArgument.c:20:3:20:6 | call to open | A file is created here without providing a mode argument, which may leak bits from the stack. |
+| OpenCallMissingModeArgument.c:22:3:22:6 | call to open | A file is created here without providing a mode argument, which may leak bits from the stack. |
+| OpenCallMissingModeArgument.c:25:3:25:8 | call to openat | A file is created here without providing a mode argument, which may leak bits from the stack. |
+| OpenCallMissingModeArgument.c:27:3:27:8 | call to openat | A file is created here without providing a mode argument, which may leak bits from the stack. |

cpp/ql/test/query-tests/Security/CWE/CWE-732/OpenCallMissingModeArgument.qlref

@@ -0,0 +1 @@
+Security/CWE/CWE-732/OpenCallMissingModeArgument.ql

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/Semmle.Autobuild.CSharp.Tests.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
 	<PropertyGroup>
-		<TargetFramework>net5.0</TargetFramework>
+		<TargetFramework>net6.0</TargetFramework>
 		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
 		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
 		<Nullable>enable</Nullable>

csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
 	<PropertyGroup>
-		<TargetFramework>net5.0</TargetFramework>
+		<TargetFramework>net6.0</TargetFramework>
 		<AssemblyName>Semmle.Autobuild.CSharp</AssemblyName>
 		<RootNamespace>Semmle.Autobuild.CSharp</RootNamespace>
 		<ApplicationIcon/>

csharp/autobuilder/Semmle.Autobuild.Shared/Semmle.Autobuild.Shared.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
 	<PropertyGroup>
-		<TargetFramework>net5.0</TargetFramework>
+		<TargetFramework>net6.0</TargetFramework>
 		<AssemblyName>Semmle.Autobuild.Shared</AssemblyName>
 		<RootNamespace>Semmle.Autobuild.Shared</RootNamespace>
 		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>

csharp/downgrades/4b9c288ece4218779b6113b17af42795f4bbbee1/upgrade.properties

@@ -0,0 +1,3 @@
+description: Remove 'lambda_expr_return_type' relation.
+compatibility: backwards
+lambda_expr_return_type.rel: delete

csharp/downgrades/ab09ac8287516082b7a7367f8fda1862b1be47c5/upgrade.properties

@@ -0,0 +1,3 @@
+description: Remove 'kind' from 'attributes'.
+compatability: full
+attributes.rel: reorder attributes.rel (int id, int kind, int type_id, int target) id type_id target

csharp/extractor/Semmle.Extraction.CIL.Driver/Semmle.Extraction.CIL.Driver.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <AssemblyName>Semmle.Extraction.CIL.Driver</AssemblyName>
     <RootNamespace>Semmle.Extraction.CIL.Driver</RootNamespace>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>

csharp/extractor/Semmle.Extraction.CIL/Semmle.Extraction.CIL.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <AssemblyName>Semmle.Extraction.CIL</AssemblyName>
     <RootNamespace>Semmle.Extraction.CIL</RootNamespace>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>

csharp/extractor/Semmle.Extraction.CSharp.Driver/Semmle.Extraction.CSharp.Driver.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <AssemblyName>Semmle.Extraction.CSharp.Driver</AssemblyName>
     <RootNamespace>Semmle.Extraction.CSharp.Driver</RootNamespace>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>

csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyInfo.cs

@@ -146,7 +146,7 @@ namespace Semmle.BuildAnalyser
                  *  loading the same assembly from different locations.
                  */
                 using var pereader = new System.Reflection.PortableExecutable.PEReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read));
-                using var sha1 = new SHA1CryptoServiceProvider();
+                using var sha1 = SHA1.Create();
                 var metadata = pereader.GetMetadata();
                 unsafe
                 {

csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs

@@ -160,7 +160,7 @@ namespace Semmle.BuildAnalyser
         {
             var bytes = Encoding.Unicode.GetBytes(srcDir);
 
-            using var sha1 = new SHA1CryptoServiceProvider();
+            using var sha1 = SHA1.Create();
             var sha = sha1.ComputeHash(bytes);
             var sb = new StringBuilder();
             foreach (var b in sha.Take(8))

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 	<PropertyGroup>
 		<OutputType>Exe</OutputType>
-		<TargetFramework>net5.0</TargetFramework>
+		<TargetFramework>net6.0</TargetFramework>
 		<AssemblyName>Semmle.Extraction.CSharp.Standalone</AssemblyName>
 		<RootNamespace>Semmle.Extraction.CSharp.Standalone</RootNamespace>
 		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>

csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs

@@ -1,3 +1,5 @@
+using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using Microsoft.CodeAnalysis;
@@ -6,18 +8,28 @@ using Semmle.Extraction.Entities;
 
 namespace Semmle.Extraction.CSharp.Entities
 {
+    internal enum AttributeKind
+    {
+        Default = 0,
+        Return = 1,
+        Assembly = 2,
+        Module = 3,
+    }
+
     internal class Attribute : CachedEntity<AttributeData>, IExpressionParentEntity
     {
         bool IExpressionParentEntity.IsTopLevelParent => true;
 
         private readonly AttributeSyntax? attributeSyntax;
         private readonly IEntity entity;
+        private readonly AttributeKind kind;
 
-        private Attribute(Context cx, AttributeData attributeData, IEntity entity)
+        private Attribute(Context cx, AttributeData attributeData, IEntity entity, AttributeKind kind)
             : base(cx, attributeData)
         {
             this.attributeSyntax = attributeData.ApplicationSyntaxReference?.GetSyntax() as AttributeSyntax;
             this.entity = entity;
+            this.kind = kind;
         }
 
         public override void WriteId(EscapingTextWriter trapFile)
@@ -48,7 +60,7 @@ namespace Semmle.Extraction.CSharp.Entities
         public override void Populate(TextWriter trapFile)
         {
             var type = Type.Create(Context, Symbol.AttributeClass);
-            trapFile.attributes(this, type.TypeRef, entity);
+            trapFile.attributes(this, kind, type.TypeRef, entity);
             trapFile.attribute_location(this, Location);
 
             if (attributeSyntax is not null)
@@ -125,26 +137,36 @@ namespace Semmle.Extraction.CSharp.Entities
 
         public override bool NeedsPopulation => true;
 
-        public static void ExtractAttributes(Context cx, ISymbol symbol, IEntity entity)
+        private static void ExtractAttributes(Context cx, IEnumerable<AttributeData> attributes, IEntity entity, AttributeKind kind)
         {
-            foreach (var attribute in symbol.GetAttributes())
+            foreach (var attribute in attributes)
             {
-                Create(cx, attribute, entity);
+                Create(cx, attribute, entity, kind);
             }
         }
 
-        public static Attribute Create(Context cx, AttributeData attributeData, IEntity entity)
+        public static void ExtractAttributes(Context cx, ISymbol symbol, IEntity entity)
         {
-            var init = (attributeData, entity);
+            ExtractAttributes(cx, symbol.GetAttributes(), entity, AttributeKind.Default);
+            if (symbol is IMethodSymbol method)
+            {
+                ExtractAttributes(cx, method.GetReturnTypeAttributes(), entity, AttributeKind.Return);
+            }
+        }
+
+
+        public static Attribute Create(Context cx, AttributeData attributeData, IEntity entity, AttributeKind kind)
+        {
+            var init = (attributeData, entity, kind);
             return AttributeFactory.Instance.CreateEntity(cx, attributeData, init);
         }
 
-        private class AttributeFactory : CachedEntityFactory<(AttributeData attributeData, IEntity receiver), Attribute>
+        private class AttributeFactory : CachedEntityFactory<(AttributeData attributeData, IEntity receiver, AttributeKind kind), Attribute>
         {
             public static readonly AttributeFactory Instance = new AttributeFactory();
 
-            public override Attribute Create(Context cx, (AttributeData attributeData, IEntity receiver) init) =>
-                new Attribute(cx, init.attributeData, init.receiver);
+            public override Attribute Create(Context cx, (AttributeData attributeData, IEntity receiver, AttributeKind kind) init) =>
+                new Attribute(cx, init.attributeData, init.receiver, init.kind);
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Lambda.cs

@@ -22,12 +22,13 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             Parameter.Create(Context, symbol, this);
         }
 
-        private Lambda(ExpressionNodeInfo info, CSharpSyntaxNode body, IEnumerable<ParameterSyntax> @params)
+        private Lambda(ExpressionNodeInfo info, CSharpSyntaxNode body, IEnumerable<ParameterSyntax> @params, TypeSyntax? @return)
             : base(info)
         {
             if (Context.GetModel(info.Node).GetSymbolInfo(info.Node).Symbol is IMethodSymbol symbol)
             {
                 Modifier.ExtractModifiers(Context, info.Context.TrapWriter.Writer, this, symbol);
+                Attribute.ExtractAttributes(Context, symbol, this);
             }
             else
             {
@@ -40,6 +41,13 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 foreach (var param in @params)
                     VisitParameter(param);
 
+                if (@return is not null)
+                {
+                    var symbol = Context.GetType(@return);
+                    var type = Entities.Type.Create(Context, symbol);
+                    var trapFile = Context.TrapWriter.Writer;
+                    trapFile.lambda_expr_return_type(this, type.TypeRef);
+                }
                 if (body is ExpressionSyntax exprBody)
                     Create(Context, exprBody, this, 0);
                 else if (body is BlockSyntax blockBody)
@@ -50,17 +58,17 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         }
 
         private Lambda(ExpressionNodeInfo info, ParenthesizedLambdaExpressionSyntax node)
-            : this(info.SetKind(ExprKind.LAMBDA), node.Body, node.ParameterList.Parameters) { }
+            : this(info.SetKind(ExprKind.LAMBDA), node.Body, node.ParameterList.Parameters, node.ReturnType) { }
 
         public static Lambda Create(ExpressionNodeInfo info, ParenthesizedLambdaExpressionSyntax node) => new Lambda(info, node);
 
         private Lambda(ExpressionNodeInfo info, SimpleLambdaExpressionSyntax node)
-            : this(info.SetKind(ExprKind.LAMBDA), node.Body, Enumerators.Singleton(node.Parameter)) { }
+            : this(info.SetKind(ExprKind.LAMBDA), node.Body, Enumerators.Singleton(node.Parameter), null) { }
 
         public static Lambda Create(ExpressionNodeInfo info, SimpleLambdaExpressionSyntax node) => new Lambda(info, node);
 
         private Lambda(ExpressionNodeInfo info, AnonymousMethodExpressionSyntax node) :
-            this(info.SetKind(ExprKind.ANONYMOUS_METHOD), node.Body, node.ParameterList is null ? Enumerable.Empty<ParameterSyntax>() : node.ParameterList.Parameters)
+            this(info.SetKind(ExprKind.ANONYMOUS_METHOD), node.Body, node.ParameterList is null ? Enumerable.Empty<ParameterSyntax>() : node.ParameterList.Parameters, null)
         { }
 
         public static Lambda Create(ExpressionNodeInfo info, AnonymousMethodExpressionSyntax node) => new Lambda(info, node);

csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs

@@ -178,6 +178,9 @@ namespace Semmle.Extraction.CSharp.Entities
         /// <returns>The converted name.</returns>
         private static string OperatorSymbol(Context cx, IMethodSymbol method)
         {
+            if (method.ExplicitInterfaceImplementations.Any())
+                return OperatorSymbol(cx, method.ExplicitInterfaceImplementations.First());
+
             var methodName = method.Name;
             if (!OperatorSymbol(methodName, out var result))
                 cx.ModelError(method, $"Unhandled operator name in OperatorSymbol(): '{methodName}'");

csharp/extractor/Semmle.Extraction.CSharp/Populators/TypeContainerVisitor.cs

@@ -86,11 +86,17 @@ namespace Semmle.Extraction.CSharp.Populators
                 return;
 
             var outputAssembly = Assembly.CreateOutputAssembly(Cx);
+            var kind = node.Target?.Identifier.Kind() switch
+            {
+                SyntaxKind.AssemblyKeyword => Entities.AttributeKind.Assembly,
+                SyntaxKind.ModuleKeyword => Entities.AttributeKind.Module,
+                _ => throw new InternalError(node, "Unhandled global target")
+            };
             foreach (var attribute in node.Attributes)
             {
                 if (attributeLookup.Value(attribute) is AttributeData attributeData)
                 {
-                    var ae = Entities.Attribute.Create(Cx, attributeData, outputAssembly);
+                    var ae = Entities.Attribute.Create(Cx, attributeData, outputAssembly, kind);
                     Cx.BindComments(ae, attribute.GetLocation());
                 }
             }

csharp/extractor/Semmle.Extraction.CSharp/Semmle.Extraction.CSharp.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
 	<PropertyGroup>
-		<TargetFramework>net5.0</TargetFramework>
+		<TargetFramework>net6.0</TargetFramework>
 		<AssemblyName>Semmle.Extraction.CSharp</AssemblyName>
 		<RootNamespace>Semmle.Extraction.CSharp</RootNamespace>
 		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>

csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs

@@ -32,8 +32,8 @@ namespace Semmle.Extraction.CSharp
         internal static void array_element_type(this TextWriter trapFile, ArrayType array, int dimension, int rank, Type elementType) =>
             trapFile.WriteTuple("array_element_type", array, dimension, rank, elementType);
 
-        internal static void attributes(this TextWriter trapFile, Attribute attribute, Type attributeType, IEntity entity) =>
-            trapFile.WriteTuple("attributes", attribute, attributeType, entity);
+        internal static void attributes(this TextWriter trapFile, Attribute attribute, AttributeKind kind, Type attributeType, IEntity entity) =>
+            trapFile.WriteTuple("attributes", attribute, kind, attributeType, entity);
 
         internal static void attribute_location(this TextWriter trapFile, Attribute attribute, Location location) =>
             trapFile.WriteTuple("attribute_location", attribute, location);
@@ -215,6 +215,9 @@ namespace Semmle.Extraction.CSharp
         internal static void indexers(this TextWriter trapFile, Indexer propKey, string name, Type declaringType, Type memberType, Indexer unboundProperty) =>
             trapFile.WriteTuple("indexers", propKey, name, declaringType, memberType, unboundProperty);
 
+        internal static void lambda_expr_return_type(this TextWriter trapFile, Lambda expr, Type returnType) =>
+            trapFile.WriteTuple("lambda_expr_return_type", expr, returnType);
+
         internal static void local_function_stmts(this TextWriter trapFile, Entities.Statements.LocalFunction fnStmt, LocalFunction fn) =>
             trapFile.WriteTuple("local_function_stmts", fnStmt, fn);
 

csharp/extractor/Semmle.Extraction.Tests/Semmle.Extraction.Tests.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
 	<PropertyGroup>
-		<TargetFramework>net5.0</TargetFramework>
+		<TargetFramework>net6.0</TargetFramework>
 		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
 		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
 		<Nullable>enable</Nullable>

csharp/extractor/Semmle.Extraction/Semmle.Extraction.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
 	<PropertyGroup>
-		<TargetFramework>net5.0</TargetFramework>
+		<TargetFramework>net6.0</TargetFramework>
 		<AssemblyName>Semmle.Extraction</AssemblyName>
 		<RootNamespace>Semmle.Extraction</RootNamespace>
 		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>

csharp/extractor/Semmle.Util.Tests/Semmle.Util.Tests.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
 	<PropertyGroup>
-		<TargetFramework>net5.0</TargetFramework>
+		<TargetFramework>net6.0</TargetFramework>
 		<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
 		<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
 		<Nullable>enable</Nullable>

csharp/extractor/Semmle.Util/FileUtils.cs

@@ -84,7 +84,7 @@ namespace Semmle.Util
         public static string ComputeFileHash(string filePath)
         {
             using var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
-            using var shaAlg = new SHA256Managed();
+            using var shaAlg = SHA256.Create();
             var sha = shaAlg.ComputeHash(fileStream);
             var hex = new StringBuilder(sha.Length * 2);
             foreach (var b in sha)

csharp/extractor/Semmle.Util/Semmle.Util.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <AssemblyName>Semmle.Util</AssemblyName>
     <RootNamespace>Semmle.Util</RootNamespace>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>

csharp/ql/campaigns/Solorigate/lib/CHANGELOG.md

@@ -1 +1,5 @@
+## 1.0.2
+
 ## 1.0.0
+
+## 0.0.6

csharp/ql/campaigns/Solorigate/lib/change-notes/released/1.0.2.md

@@ -0,0 +1 @@
+## 1.0.2

csharp/ql/campaigns/Solorigate/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.0.1
+lastReleaseVersion: 1.0.2

csharp/ql/campaigns/Solorigate/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-all
-version: 1.0.2-dev
+version: 1.0.3-dev
 groups:
     - csharp
     - solorigate

csharp/ql/campaigns/Solorigate/src/CHANGELOG.md

@@ -1 +1,5 @@
+## 1.0.2
+
 ## 1.0.0
+
+## 0.0.6

csharp/ql/campaigns/Solorigate/src/change-notes/released/1.0.2.md

@@ -0,0 +1 @@
+## 1.0.2