C++: another shot at implicit/deleted copy

Fixes some cases in instantiations of templates with manually written
copy constructors or copy assignment operators where
hasImplicitCopyConstructor would incorrectly hold

.codeqlmanifest.json

@@ -6,6 +6,7 @@
         "*/ql/examples/qlpack.yml",
         "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
         "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
+        "javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml",
         "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
         "csharp/ql/campaigns/Solorigate/lib/qlpack.yml",
         "csharp/ql/campaigns/Solorigate/src/qlpack.yml",

.github/workflows/mad_modelDiff.yml

@@ -0,0 +1,103 @@
+name: Models as Data - Diff
+
+on:
+  workflow_dispatch:
+    inputs:
+      projects:
+        description: "The projects to generate models for"
+        required: true
+        default: '["netty/netty"]'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "java/ql/src/utils/model-generator/**/*.*"
+      - ".github/workflows/mad_modelDiff.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  model-diff:
+    name: Model Difference
+    runs-on: ubuntu-latest
+    if: github.repository == 'github/codeql'
+    strategy:
+      matrix:
+        slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}}
+    steps:
+      - name: Clone github/codeql from PR
+        uses: actions/checkout@v2
+        if: github.event.pull_request
+        with:
+          path: codeql-pr
+      - name: Clone github/codeql from main
+        uses: actions/checkout@v2
+        with:
+          path: codeql-main
+          ref: main
+      - uses: ./codeql-main/.github/actions/fetch-codeql
+      - name: Download database
+        env:
+          SLUG: ${{ matrix.slug }}
+        run: |
+          set -x
+          mkdir lib-dbs
+          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
+          projectId=`curl -s https://lgtm.com/api/v1.0/projects/g/${SLUG} | jq .id`
+          curl -L "https://lgtm.com/api/v1.0/snapshots/$projectId/java" -o "$SHORTNAME.zip"
+          unzip -q -d "${SHORTNAME}-db" "${SHORTNAME}.zip"
+          mkdir "lib-dbs/$SHORTNAME/"
+          mv "${SHORTNAME}-db/"$(ls -1 "${SHORTNAME}"-db)/* "lib-dbs/${SHORTNAME}/"
+      - name: Generate Models (PR and main)
+        run: |
+          set -x
+          mkdir tmp-models
+          MODELS=`pwd`/tmp-models
+          DATABASES=`pwd`/lib-dbs
+
+          analyzeDatabaseWithCheckout() {
+            QL_VARIANT=$1
+            DATABASE=$2
+            cd codeql-$QL_VARIANT
+            SHORTNAME=`basename $DATABASE`
+            python java/ql/src/utils/model-generator/GenerateFlowModel.py $DATABASE $MODELS/${SHORTNAME}.qll
+            mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll
+            cd ..
+          }
+
+          for d in $DATABASES/*/ ; do
+            ls -1 "$d"
+
+            analyzeDatabaseWithCheckout "main" $d
+            if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]
+            then
+              analyzeDatabaseWithCheckout "pr" $d
+            fi
+          done
+      - name: Install diff2html
+        if: github.event.pull_request
+        run: |
+          npm install -g diff2html-cli
+      - name: Generate Model Diff
+        if: github.event.pull_request
+        run: |
+          set -x
+          MODELS=`pwd`/tmp-models
+          ls -1 tmp-models/
+          for m in $MODELS/*_main.qll ; do
+            t="${m/main/"pr"}"
+            basename=`basename $m`
+            name="diff_${basename/_main.qll/""}"
+            (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
+          done
+      - uses: actions/upload-artifact@v2
+        with:
+          name: models
+          path: tmp-models/*.qll
+          retention-days: 20
+      - uses: actions/upload-artifact@v2
+        with:
+          name: diffs
+          path: tmp-models/*.html
+          retention-days: 20

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -17,7 +17,7 @@ jobs:
       CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
     strategy:
       matrix:
-        repo: 
+        repo:
           - github/codeql
           - github/codeql-go
     runs-on: ubuntu-latest
@@ -35,7 +35,7 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             ql/target
-          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Build Extractor
         run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./create-extractor-pack.sh
         env:

.github/workflows/ql-for-ql-tests.yml

@@ -29,24 +29,24 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             ql/target
-          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Build extractor
         run: |
           cd ql;
           codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
           env "PATH=$PATH:$codeqlpath" ./create-extractor-pack.sh
       - name: Run QL tests
-        run: | 
+        run: |
           "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries ql/ql/test
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Check QL formatting
-        run: | 
+        run: |
           find ql/ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 "${CODEQL}" query format --check-only
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Check QL compilation
-        run: | 
+        run: |
           "${CODEQL}" query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}/ql/extractor-pack" "ql/ql/src" "ql/ql/examples"
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}

.github/workflows/ruby-build.yml

@@ -50,7 +50,7 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             ruby/target
-          key: ${{ runner.os }}-rust-cargo-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-ruby-rust-cargo-${{ hashFiles('ruby/**/Cargo.lock') }}
       - name: Check formatting
         run: cargo fmt --all -- --check
       - name: Build

.github/workflows/ruby-qltest.yml

@@ -24,27 +24,45 @@ defaults:
     working-directory: ruby
 
 jobs:
-  qltest:
+  qlformat:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/fetch-codeql
-      - uses: ./ruby/actions/create-extractor-pack
-      - name: Run QL tests
-        run: |
-          codeql test run --threads=0 --ram 5000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
       - name: Check QL formatting
         run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
+  qlcompile:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/fetch-codeql
       - name: Check QL compilation
         run: |
           codeql query compile --check-only --threads=0 --ram 5000 --warnings=error "ql/src" "ql/examples"
         env:
           GITHUB_TOKEN: ${{ github.token }}
+  qlupgrade:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/fetch-codeql
       - name: Check DB upgrade scripts
         run: |
           echo >empty.trap
           codeql dataset import -S ql/lib/upgrades/initial/ruby.dbscheme testdb empty.trap
           codeql dataset upgrade testdb --additional-packs ql/lib
           diff -q testdb/ruby.dbscheme ql/lib/ruby.dbscheme
+  qltest:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        slice: ["1/2", "2/2"]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/fetch-codeql
+      - uses: ./ruby/actions/create-extractor-pack
+      - name: Run QL tests
+        run: |
+          codeql test run --threads=0 --ram 5000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test
+        env:
+          GITHUB_TOKEN: ${{ github.token }}

cpp/ql/examples/qlpack.yml

@@ -1,4 +1,6 @@
 name: codeql/cpp-examples
-version: 0.0.2
+groups: 
+  - cpp
+  - examples
 dependencies:
   codeql/cpp-all: "*"

cpp/ql/lib/change-notes/2022-03-03-template-implicit-copy.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The `Class::hasImplicitCopyConstructor` and `Class::hasImplicitCopyAssignmentOperator` methods now handle template instantiations more accurately. This should improve results for the `cpp/rule-of-two` query.

cpp/ql/lib/semmle/code/cpp/Class.qll

@@ -285,7 +285,17 @@ class Class extends UserType {
   predicate hasImplicitCopyConstructor() {
     not this.implicitCopyConstructorDeleted() and
     forall(CopyConstructor cc | cc = this.getAMemberFunction() |
-      cc.isCompilerGenerated() and not cc.isDeleted()
+      cc.isCompilerGenerated() and not cc.isDeleted() and not cc.isDefaulted()
+    ) and
+    (
+      not this instanceof ClassTemplateInstantiation
+      or
+      this.(ClassTemplateInstantiation).getTemplate().hasImplicitCopyConstructor()
+    ) and
+    (
+      not this instanceof PartialClassTemplateSpecialization
+      or
+      this.(PartialClassTemplateSpecialization).getPrimaryTemplate().hasImplicitCopyConstructor()
     )
   }
 
@@ -300,7 +310,19 @@ class Class extends UserType {
   predicate hasImplicitCopyAssignmentOperator() {
     not this.implicitCopyAssignmentOperatorDeleted() and
     forall(CopyAssignmentOperator ca | ca = this.getAMemberFunction() |
-      ca.isCompilerGenerated() and not ca.isDeleted()
+      ca.isCompilerGenerated() and not ca.isDeleted() and not ca.isDefaulted()
+    ) and
+    (
+      not this instanceof ClassTemplateInstantiation
+      or
+      this.(ClassTemplateInstantiation).getTemplate().hasImplicitCopyAssignmentOperator()
+    ) and
+    (
+      not this instanceof PartialClassTemplateSpecialization
+      or
+      this.(PartialClassTemplateSpecialization)
+          .getPrimaryTemplate()
+          .hasImplicitCopyAssignmentOperator()
     )
   }
 
@@ -311,6 +333,12 @@ class Class extends UserType {
    * http://en.cppreference.com/w/cpp/language/copy_constructor#Deleted_implicitly-declared_copy_constructor
    */
   predicate implicitCopyConstructorDeleted() {
+    forex(CopyConstructor cc | cc = this.getAConstructor() |
+      cc.isDeleted()
+      or
+      not cc.isCompilerGenerated()
+    )
+    or
     // - T has non-static data members that cannot be copied (have deleted,
     //   inaccessible, or ambiguous copy constructors);
     exists(Type t | t = this.getAFieldSubobjectType().getUnspecifiedType() |
@@ -318,34 +346,6 @@ class Class extends UserType {
       // constructors are considered equal.
       this.cannotAccessCopyConstructorOnAny(t)
     )
-    or
-    // - T has direct or virtual base class that cannot be copied (has deleted,
-    //   inaccessible, or ambiguous copy constructors);
-    exists(Class c | c = this.getADirectOrVirtualBase() |
-      // Note: Overload resolution is not implemented -- all copy
-      // constructors are considered equal.
-      this.cannotAccessCopyConstructorOnThis(c)
-    )
-    or
-    // - T has direct or virtual base class with a deleted or inaccessible
-    //   destructor;
-    exists(Class base | base = this.getADirectOrVirtualBase() |
-      this.cannotAccessDestructor(base, this)
-    )
-    or
-    // - T has a user-defined move constructor or move assignment operator;
-    exists(MoveConstructor mc | mc = this.getAMemberFunction() | not mc.isCompilerGenerated())
-    or
-    exists(MoveAssignmentOperator ma | ma = this.getAMemberFunction() |
-      not ma.isCompilerGenerated()
-    )
-    or
-    // - T is a union and has a variant member with non-trivial copy
-    //   constructor (since C++11)
-    none() // Not implemented
-    or
-    // - T has a data member of rvalue reference type.
-    exists(Type t | t = this.getAFieldSubobjectType() | t instanceof RValueReferenceType)
   }
 
   /**
@@ -355,34 +355,12 @@ class Class extends UserType {
    * http://en.cppreference.com/w/cpp/language/copy_assignment#Deleted_implicitly-declared_copy_assignment_operator
    */
   predicate implicitCopyAssignmentOperatorDeleted() {
-    // - T has a user-declared move constructor;
-    exists(MoveConstructor mc | mc = this.getAMemberFunction() | not mc.isCompilerGenerated())
-    or
-    // - T has a user-declared move assignment operator.
-    exists(MoveAssignmentOperator ma | ma = this.getAMemberFunction() |
-      not ma.isCompilerGenerated()
+    forex(CopyAssignmentOperator ca | ca = this.getAMemberFunction() |
+      ca.isDeleted()
+      or
+      not ca.isCompilerGenerated()
     )
     or
-    // - T has a non-static data member of non-class type (or array thereof)
-    //   that is const;
-    exists(Type t | t = this.getAFieldSubobjectType() |
-      // The rule for this case refers only to non-class types only, but our
-      // implementation extends it to cover class types too. Class types are
-      // supposed to be covered by the rule below on data members that
-      // cannot be copy-assigned. Copy-assigning a const class-typed member
-      // would call an overload of type
-      // `const C& operator=(const C&) const;`. Such an overload is unlikely
-      // to exist because it contradicts the intention of "const": it allows
-      // assigning to a const object. But since we have not implemented the
-      // ability to distinguish between overloads, we cannot distinguish that
-      // overload from the ordinary `C& operator=(const C&);`. Instead, we
-      // require class types to be non-const in this clause.
-      /* not t instanceof Class and */ t.isConst()
-    )
-    or
-    // - T has a non-static data member of a reference type;
-    exists(Type t | t = this.getAFieldSubobjectType() | t instanceof ReferenceType)
-    or
     // - T has a non-static data member or a direct or virtual base class that
     //   cannot be copy-assigned (overload resolution for the copy assignment
     //   fails, or selects a deleted or inaccessible function);
@@ -391,15 +369,6 @@ class Class extends UserType {
       // operators are considered equal.
       this.cannotAccessCopyAssignmentOperatorOnAny(t)
     )
-    or
-    exists(Class c | c = this.getADirectOrVirtualBase() |
-      // Note: Overload resolution is not implemented -- all copy assignment
-      // operators are considered equal.
-      this.cannotAccessCopyAssignmentOperatorOnThis(c)
-    )
-    // - T is a union-like class, and has a variant member whose corresponding
-    //   assignment operator is non-trivial.
-    // Not implemented
   }
 
   /** Gets the destructor of this class, struct or union, if any. */

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll

@@ -1290,7 +1290,7 @@ class DataFlowCallOption extends TDataFlowCallOption {
   }
 }
 
-/** Content tagged with the type of a containing object. */
+/** A `Content` tagged with the type of a containing object. */
 class TypedContent extends MkTypedContent {
   private Content c;
   private DataFlowType t;

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll

@@ -1290,7 +1290,7 @@ class DataFlowCallOption extends TDataFlowCallOption {
   }
 }
 
-/** Content tagged with the type of a containing object. */
+/** A `Content` tagged with the type of a containing object. */
 class TypedContent extends MkTypedContent {
   private Content c;
   private DataFlowType t;

cpp/ql/lib/semmle/code/cpp/models/implementations/Gets.qll

@@ -11,15 +11,14 @@ import semmle.code.cpp.models.interfaces.SideEffect
 import semmle.code.cpp.models.interfaces.FlowSource
 
 /**
- * The standard functions `gets` and `fgets`.
+ * The standard functions `fgets` and `fgetws`.
  */
-private class GetsFunction extends DataFlowFunction, TaintFunction, ArrayFunction, AliasFunction,
+private class FgetsFunction extends DataFlowFunction, TaintFunction, ArrayFunction, AliasFunction,
   SideEffectFunction, RemoteFlowSourceFunction {
-  GetsFunction() {
-    // gets(str)
+  FgetsFunction() {
     // fgets(str, num, stream)
     // fgetws(wstr, num, stream)
-    this.hasGlobalOrStdOrBslName(["gets", "fgets", "fgetws"])
+    this.hasGlobalOrStdOrBslName(["fgets", "fgetws"])
   }
 
   override predicate hasDataFlow(FunctionInput input, FunctionOutput output) {
@@ -51,20 +50,61 @@ private class GetsFunction extends DataFlowFunction, TaintFunction, ArrayFunctio
   override predicate hasRemoteFlowSource(FunctionOutput output, string description) {
     output.isParameterDeref(0) and
     description = "String read by " + this.getName()
+    or
+    output.isReturnValue() and
+    description = "String read by " + this.getName()
   }
 
   override predicate hasArrayWithVariableSize(int bufParam, int countParam) {
-    not this.hasName("gets") and
     bufParam = 0 and
     countParam = 1
   }
 
-  override predicate hasArrayWithUnknownSize(int bufParam) {
-    this.hasName("gets") and
-    bufParam = 0
-  }
-
   override predicate hasArrayOutput(int bufParam) { bufParam = 0 }
 
-  override predicate hasSocketInput(FunctionInput input) { input.isParameter(2) }
+  override predicate hasSocketInput(FunctionInput input) { input.isParameterDeref(2) }
+}
+
+/**
+ * The standard functions `gets`.
+ */
+private class GetsFunction extends DataFlowFunction, ArrayFunction, AliasFunction,
+  SideEffectFunction, LocalFlowSourceFunction {
+  GetsFunction() {
+    // gets(str)
+    this.hasGlobalOrStdOrBslName("gets")
+  }
+
+  override predicate hasDataFlow(FunctionInput input, FunctionOutput output) {
+    input.isParameter(0) and
+    output.isReturnValue()
+  }
+
+  override predicate parameterNeverEscapes(int index) { none() }
+
+  override predicate parameterEscapesOnlyViaReturn(int index) { index = 0 }
+
+  override predicate parameterIsAlwaysReturned(int index) { index = 0 }
+
+  override predicate hasOnlySpecificReadSideEffects() { any() }
+
+  override predicate hasOnlySpecificWriteSideEffects() { any() }
+
+  override predicate hasSpecificWriteSideEffect(ParameterIndex i, boolean buffer, boolean mustWrite) {
+    i = 0 and
+    buffer = true and
+    mustWrite = true
+  }
+
+  override predicate hasLocalFlowSource(FunctionOutput output, string description) {
+    output.isParameterDeref(0) and
+    description = "String read by " + this.getName()
+    or
+    output.isReturnValue() and
+    description = "String read by " + this.getName()
+  }
+
+  override predicate hasArrayWithUnknownSize(int bufParam) { bufParam = 0 }
+
+  override predicate hasArrayOutput(int bufParam) { bufParam = 0 }
 }

cpp/ql/lib/tutorial.qll

@@ -6,122 +6,22 @@
  */
 class Person extends string {
   Person() {
-    this = "Ronil" or
-    this = "Dina" or
-    this = "Ravi" or
-    this = "Bruce" or
-    this = "Jo" or
-    this = "Aida" or
-    this = "Esme" or
-    this = "Charlie" or
-    this = "Fred" or
-    this = "Meera" or
-    this = "Maya" or
-    this = "Chad" or
-    this = "Tiana" or
-    this = "Laura" or
-    this = "George" or
-    this = "Will" or
-    this = "Mary" or
-    this = "Almira" or
-    this = "Susannah" or
-    this = "Rhoda" or
-    this = "Cynthia" or
-    this = "Eunice" or
-    this = "Olive" or
-    this = "Virginia" or
-    this = "Angeline" or
-    this = "Helen" or
-    this = "Cornelia" or
-    this = "Harriet" or
-    this = "Mahala" or
-    this = "Abby" or
-    this = "Margaret" or
-    this = "Deb" or
-    this = "Minerva" or
-    this = "Severus" or
-    this = "Lavina" or
-    this = "Adeline" or
-    this = "Cath" or
-    this = "Elisa" or
-    this = "Lucretia" or
-    this = "Anne" or
-    this = "Eleanor" or
-    this = "Joanna" or
-    this = "Adam" or
-    this = "Agnes" or
-    this = "Rosanna" or
-    this = "Clara" or
-    this = "Melissa" or
-    this = "Amy" or
-    this = "Isabel" or
-    this = "Jemima" or
-    this = "Cordelia" or
-    this = "Melinda" or
-    this = "Delila" or
-    this = "Jeremiah" or
-    this = "Elijah" or
-    this = "Hester" or
-    this = "Walter" or
-    this = "Oliver" or
-    this = "Hugh" or
-    this = "Aaron" or
-    this = "Reuben" or
-    this = "Eli" or
-    this = "Amos" or
-    this = "Augustus" or
-    this = "Theodore" or
-    this = "Ira" or
-    this = "Timothy" or
-    this = "Cyrus" or
-    this = "Horace" or
-    this = "Simon" or
-    this = "Asa" or
-    this = "Frank" or
-    this = "Nelson" or
-    this = "Leonard" or
-    this = "Harrison" or
-    this = "Anthony" or
-    this = "Louis" or
-    this = "Milton" or
-    this = "Noah" or
-    this = "Cornelius" or
-    this = "Abdul" or
-    this = "Warren" or
-    this = "Harvey" or
-    this = "Dennis" or
-    this = "Wesley" or
-    this = "Sylvester" or
-    this = "Gilbert" or
-    this = "Sullivan" or
-    this = "Edmund" or
-    this = "Wilson" or
-    this = "Perry" or
-    this = "Matthew" or
-    this = "Simba" or
-    this = "Nala" or
-    this = "Rafiki" or
-    this = "Shenzi" or
-    this = "Ernest" or
-    this = "Gertrude" or
-    this = "Oscar" or
-    this = "Lilian" or
-    this = "Raymond" or
-    this = "Elgar" or
-    this = "Elmer" or
-    this = "Herbert" or
-    this = "Maude" or
-    this = "Mae" or
-    this = "Otto" or
-    this = "Edwin" or
-    this = "Ophelia" or
-    this = "Parsley" or
-    this = "Sage" or
-    this = "Rosemary" or
-    this = "Thyme" or
-    this = "Garfunkel" or
-    this = "King Basil" or
-    this = "Stephen"
+    this =
+      [
+        "Ronil", "Dina", "Ravi", "Bruce", "Jo", "Aida", "Esme", "Charlie", "Fred", "Meera", "Maya",
+        "Chad", "Tiana", "Laura", "George", "Will", "Mary", "Almira", "Susannah", "Rhoda",
+        "Cynthia", "Eunice", "Olive", "Virginia", "Angeline", "Helen", "Cornelia", "Harriet",
+        "Mahala", "Abby", "Margaret", "Deb", "Minerva", "Severus", "Lavina", "Adeline", "Cath",
+        "Elisa", "Lucretia", "Anne", "Eleanor", "Joanna", "Adam", "Agnes", "Rosanna", "Clara",
+        "Melissa", "Amy", "Isabel", "Jemima", "Cordelia", "Melinda", "Delila", "Jeremiah", "Elijah",
+        "Hester", "Walter", "Oliver", "Hugh", "Aaron", "Reuben", "Eli", "Amos", "Augustus",
+        "Theodore", "Ira", "Timothy", "Cyrus", "Horace", "Simon", "Asa", "Frank", "Nelson",
+        "Leonard", "Harrison", "Anthony", "Louis", "Milton", "Noah", "Cornelius", "Abdul", "Warren",
+        "Harvey", "Dennis", "Wesley", "Sylvester", "Gilbert", "Sullivan", "Edmund", "Wilson",
+        "Perry", "Matthew", "Simba", "Nala", "Rafiki", "Shenzi", "Ernest", "Gertrude", "Oscar",
+        "Lilian", "Raymond", "Elgar", "Elmer", "Herbert", "Maude", "Mae", "Otto", "Edwin",
+        "Ophelia", "Parsley", "Sage", "Rosemary", "Thyme", "Garfunkel", "King Basil", "Stephen"
+      ]
   }
 
   /** Gets the hair color of the person. If the person is bald, there is no result. */
@@ -936,25 +836,12 @@ class Person extends string {
 
   /** Holds if the person is deceased. */
   predicate isDeceased() {
-    this = "Ernest" or
-    this = "Gertrude" or
-    this = "Oscar" or
-    this = "Lilian" or
-    this = "Edwin" or
-    this = "Raymond" or
-    this = "Elgar" or
-    this = "Elmer" or
-    this = "Herbert" or
-    this = "Maude" or
-    this = "Mae" or
-    this = "Otto" or
-    this = "Ophelia" or
-    this = "Parsley" or
-    this = "Sage" or
-    this = "Rosemary" or
-    this = "Thyme" or
-    this = "Garfunkel" or
-    this = "King Basil"
+    this =
+      [
+        "Ernest", "Gertrude", "Oscar", "Lilian", "Edwin", "Raymond", "Elgar", "Elmer", "Herbert",
+        "Maude", "Mae", "Otto", "Ophelia", "Parsley", "Sage", "Rosemary", "Thyme", "Garfunkel",
+        "King Basil"
+      ]
   }
 
   /** Gets a parent of the person (alive or deceased). */
@@ -1195,12 +1082,7 @@ class Person extends string {
   }
 
   /** Holds if the person is allowed in the region. Initially, all villagers are allowed in every region. */
-  predicate isAllowedIn(string region) {
-    region = "north" or
-    region = "south" or
-    region = "east" or
-    region = "west"
-  }
+  predicate isAllowedIn(string region) { region = ["north", "south", "east", "west"] }
 }
 
 /** Returns a parent of the person. */

cpp/ql/src/Security/CWE/CWE-311/CleartextFileWrite.ql

@@ -65,6 +65,7 @@ where
   midNode.getNode().asExpr() = mid and
   mid = w.getASource() and
   dest = w.getDest() and
+  not dest.(VariableAccess).getTarget().getName() = ["stdin", "stdout", "stderr"] and // exclude calls with standard streams
   not isFileName(globalValueNumber(source)) and // file names are not passwords
   not exists(string convChar | convChar = w.getSourceConvChar(mid) | not convChar = ["s", "S"]) // ignore things written with other conversion characters
 select w, sourceNode, midNode,

cpp/ql/src/Security/CWE/CWE-311/CleartextTransmission.ql

@@ -27,6 +27,7 @@ class SensitiveNode extends DataFlow::Node {
     this.asExpr() = any(SensitiveVariable sv).getInitializer().getExpr() or
     this.asExpr().(VariableAccess).getTarget() =
       any(SensitiveVariable sv).(GlobalOrNamespaceVariable) or
+    this.asExpr().(VariableAccess).getTarget() = any(SensitiveVariable v | v instanceof Field) or
     this.asUninitialized() instanceof SensitiveVariable or
     this.asParameter() instanceof SensitiveVariable or
     this.asExpr().(FunctionCall).getTarget() instanceof SensitiveFunction
@@ -58,7 +59,10 @@ class Send extends SendRecv instanceof RemoteFlowSinkFunction {
     call.getTarget() = this and
     exists(FunctionInput input, int arg |
       super.hasSocketInput(input) and
-      input.isParameter(arg) and
+      (
+        input.isParameter(arg) or
+        input.isParameterDeref(arg)
+      ) and
       result = call.getArgument(arg)
     )
   }
@@ -81,7 +85,10 @@ class Recv extends SendRecv instanceof RemoteFlowSourceFunction {
     call.getTarget() = this and
     exists(FunctionInput input, int arg |
       super.hasSocketInput(input) and
-      input.isParameter(arg) and
+      (
+        input.isParameter(arg) or
+        input.isParameterDeref(arg)
+      ) and
       result = call.getArgument(arg)
     )
   }

cpp/ql/src/change-notes/2022-01-22-cleartext-transmission.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The "Cleartext transmission of sensitive information" (`cpp/cleartext-transmission`) query now finds more results, where a password is stored in a struct field or class member variable.

cpp/ql/src/change-notes/2022-01-28-cleartext-storage-file.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The `cpp/cleartext-storage-file` query has been improved, removing false positives where data is written to a standard output stream.

cpp/ql/src/experimental/Best Practices/UselessTest.cpp

@@ -0,0 +1,9 @@
+void test(){
+    int a = 8;
+    int b = 9;
+
+    //Useless NonEquals
+    if(a==8 && a != 7) {}
+
+    while(a==8 && a!=7){}
+}

cpp/ql/src/experimental/Best Practices/UselessTest.qhelp

@@ -0,0 +1,18 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+
+<overview>
+<p>Comparison operations like <code>a==8 &amp;&amp; a!=7</code> contain a useless part : the non-equal part. This rule finds tests of this kind within an <code>if</code> or a <code>while</code> statement</p>
+</overview>
+
+<recommendation>
+<p>Remove the useless comparisons</p>
+</recommendation>
+
+<example>
+<sample src="UselessTest.cpp" />
+</example>
+
+</qhelp>

cpp/ql/src/experimental/Best Practices/UselessTest.ql

@@ -0,0 +1,43 @@
+/**
+ * @name Useless Test
+ * @description A boolean condition that is guaranteed to never be evaluated should be deleted.
+ * @kind problem
+ * @problem.severity warning
+ * @id cpp/uselesstest
+ * @tags reliability
+ *       readability
+ */
+
+import cpp
+import semmle.code.cpp.valuenumbering.GlobalValueNumbering
+
+predicate sameExpr(Expr e1, Expr e2) { globalValueNumber(e1).getAnExpr() = e2 }
+
+Element nearestParent(Expr e) {
+  if
+    e.getParent().(Expr).getConversion*() instanceof ParenthesisExpr or
+    e.getParent() instanceof IfStmt or
+    e.getParent() instanceof WhileStmt
+  then result = e.getParent()
+  else result = nearestParent(e.getParent())
+}
+
+from LogicalAndExpr b, EQExpr eq, NEExpr ne
+where
+  (
+    b.getAChild*() = eq and
+    b.getAChild*() = ne and
+    eq.getParent() instanceof LogicalAndExpr and
+    ne.getParent() instanceof LogicalAndExpr
+  ) and
+  (
+    eq.getLeftOperand() instanceof VariableAccess and ne.getLeftOperand() instanceof VariableAccess
+    or
+    eq.getLeftOperand() instanceof PointerDereferenceExpr and
+    ne.getLeftOperand() instanceof PointerDereferenceExpr
+  ) and
+  eq.getRightOperand() instanceof Literal and
+  ne.getRightOperand() instanceof Literal and
+  nearestParent(eq) = nearestParent(ne) and
+  sameExpr(eq.getLeftOperand(), ne.getLeftOperand())
+select ne, "Useless Test"

cpp/ql/src/qlpack.yml

@@ -1,6 +1,8 @@
 name: codeql/cpp-queries
 version: 0.0.8-dev
-groups: cpp
+groups: 
+  - cpp
+  - queries
 dependencies:
     codeql/cpp-all: "*"
     codeql/suite-helpers: "*"

cpp/ql/test/TestUtilities/InlineExpectationsTest.qll

@@ -93,7 +93,7 @@
 private import InlineExpectationsTestPrivate
 
 /**
- * Base class for tests with inline expectations. The test extends this class to provide the actual
+ * The base class for tests with inline expectations. The test extends this class to provide the actual
  * results of the query, which are then compared with the expected results in comments to produce a
  * list of failure messages that point out where the actual results differ from the expected
  * results.

cpp/ql/test/library-tests/special_members/generated_copy/assign.expected

@@ -11,6 +11,7 @@
 | difference::Base | can | does NOT | have implicit copy assignment |
 | difference::OnlyAssign | can | does | have implicit copy assignment |
 | difference::OnlyCtor | can NOT | does NOT | have implicit copy assignment |
+| instantiated_explicit_ctor::Wrapper<int> | can | does | have implicit copy assignment |
 | moves::MoveAssign | can NOT | does NOT | have implicit copy assignment |
 | moves::MoveCtor | can NOT | does NOT | have implicit copy assignment |
 | private_cc::C | can | does NOT | have implicit copy assignment |

cpp/ql/test/library-tests/special_members/generated_copy/copy.cpp

@@ -131,3 +131,21 @@ namespace difference {
   class OnlyAssign : Base {
   };
 }
+
+namespace instantiated_explicit_ctor {
+  template<class T>
+  class Wrapper {
+  public:
+    Wrapper(Wrapper<T> &other) {
+      m_t = other.m_t;
+    }
+    
+    Wrapper() {
+      m_t = 0;
+    }
+  private:
+    T m_t;
+  };
+
+  Wrapper<int> wrapped_int;
+}

cpp/ql/test/library-tests/special_members/generated_copy/ctor.expected

@@ -11,6 +11,7 @@
 | difference::Base | can | does NOT | have implicit copy constructor |
 | difference::OnlyAssign | can NOT | does NOT | have implicit copy constructor |
 | difference::OnlyCtor | can | does | have implicit copy constructor |
+| instantiated_explicit_ctor::Wrapper<int> | can | does NOT | have implicit copy constructor |
 | moves::MoveAssign | can NOT | does NOT | have implicit copy constructor |
 | moves::MoveCtor | can NOT | does NOT | have implicit copy constructor |
 | private_cc::C | can | does NOT | have implicit copy constructor |

cpp/ql/test/library-tests/special_members/generated_copy/functions.expected

@@ -86,5 +86,9 @@
 | copy.cpp:131:9:131:9 | OnlyAssign | deleted |  |
 | copy.cpp:131:9:131:9 | operator= |  |  |
 | copy.cpp:131:9:131:9 | operator= |  |  |
+| copy.cpp:137:9:137:9 | operator= |  |  |
+| copy.cpp:139:5:139:11 | Wrapper |  |  |
+| copy.cpp:143:5:143:5 | Wrapper |  |  |
+| copy.cpp:143:5:143:11 | Wrapper |  |  |
 | file://:0:0:0:0 | operator= |  |  |
 | file://:0:0:0:0 | operator= |  |  |

cpp/ql/test/query-tests/Best Practices/RuleOfTwo/RuleOfTwo.cpp

@@ -117,12 +117,8 @@ struct HasVPV {
   }
 };
 
-// FALSE NEGATIVE: the relevant copy constructor of ProtectedVolatile is
-// accessible, so our class will get a generated copy constructor. Our query
-// thinks the copy constructor is inaccessible because it picks up the other
-// copy constructor. To fix this, our library should be changed to distinguish
-// between copy constructors and resolve overloading properly instead of
-// assuming that there is at most one.
+// NOT OK: the relevant copy constructor of ProtectedVolatile is
+// accessible, so our class will get a generated copy constructor.
 struct HasPV {
   ProtectedVolatile pv;
   HasPV& operator=(const HasPV& that) {

cpp/ql/test/query-tests/Best Practices/RuleOfTwo/RuleOfTwo.expected

@@ -1,6 +1,6 @@
 | RuleOfTwo.cpp:4:3:4:17 | CopyButNoAssign | No matching copy assignment operator in class CopyButNoAssign. It is good practice to match a copy constructor with a copy assignment operator. |
 | RuleOfTwo.cpp:10:20:10:28 | operator= | No matching copy constructor in class AssignButNoCopy. It is good practice to match a copy assignment operator with a copy constructor. |
 | RuleOfTwo.cpp:81:18:81:26 | operator= | No matching copy constructor in class MyClassFriend. It is good practice to match a copy assignment operator with a copy constructor. |
-| RuleOfTwo.cpp:144:3:144:20 | IsAProtectedAssign | No matching copy assignment operator in class IsAProtectedAssign. It is good practice to match a copy constructor with a copy assignment operator. |
-| RuleOfTwo.cpp:167:19:167:27 | operator= | No matching copy constructor in class IsAProtectedCC. It is good practice to match a copy assignment operator with a copy constructor. |
-| RuleOfTwo.cpp:312:5:312:8 | R1_C | No matching copy assignment operator in class R1_C. It is good practice to match a copy constructor with a copy assignment operator. |
+| RuleOfTwo.cpp:140:3:140:20 | IsAProtectedAssign | No matching copy assignment operator in class IsAProtectedAssign. It is good practice to match a copy constructor with a copy assignment operator. |
+| RuleOfTwo.cpp:163:19:163:27 | operator= | No matching copy constructor in class IsAProtectedCC. It is good practice to match a copy assignment operator with a copy constructor. |
+| RuleOfTwo.cpp:308:5:308:8 | R1_C | No matching copy assignment operator in class R1_C. It is good practice to match a copy constructor with a copy assignment operator. |

cpp/ql/test/query-tests/Security/CWE/CWE-311/semmle/tests/CleartextBufferWrite.expected

@@ -1,8 +1,10 @@
 edges
 | test.cpp:54:17:54:20 | argv | test.cpp:58:25:58:29 | input |
 nodes
+| test2.cpp:110:3:110:6 | call to gets | semmle.label | call to gets |
 | test.cpp:54:17:54:20 | argv | semmle.label | argv |
 | test.cpp:58:25:58:29 | input | semmle.label | input |
 subpaths
 #select
+| test2.cpp:110:3:110:6 | call to gets | test2.cpp:110:3:110:6 | call to gets | test2.cpp:110:3:110:6 | call to gets | This write into buffer 'password' may contain unencrypted data from $@ | test2.cpp:110:3:110:6 | call to gets | user input (String read by gets) |
 | test.cpp:58:3:58:9 | call to sprintf | test.cpp:54:17:54:20 | argv | test.cpp:58:25:58:29 | input | This write into buffer 'passwd' may contain unencrypted data from $@ | test.cpp:54:17:54:20 | argv | user input (a command-line argument) |

cpp/ql/test/query-tests/Security/CWE/CWE-311/semmle/tests/CleartextTransmission.expected

@@ -1,4 +1,5 @@
 edges
+| test2.cpp:63:24:63:31 | password | test2.cpp:63:16:63:20 | call to crypt |
 | test3.cpp:17:28:17:36 | password1 | test3.cpp:22:15:22:23 | password1 |
 | test3.cpp:17:51:17:59 | password2 | test3.cpp:26:15:26:23 | password2 |
 | test3.cpp:45:8:45:15 | password | test3.cpp:47:15:47:22 | password |
@@ -89,11 +90,15 @@ edges
 | test3.cpp:398:18:398:25 | password | test3.cpp:400:15:400:23 | & ... |
 | test3.cpp:398:18:398:25 | password | test3.cpp:400:16:400:23 | password |
 | test3.cpp:398:18:398:25 | password | test3.cpp:400:33:400:40 | password |
+| test3.cpp:421:21:421:28 | password | test3.cpp:421:3:421:17 | call to decrypt_inplace |
 | test.cpp:41:23:41:43 | cleartext password! | test.cpp:48:21:48:27 | call to encrypt |
 | test.cpp:41:23:41:43 | cleartext password! | test.cpp:48:29:48:39 | thePassword |
 | test.cpp:66:23:66:43 | cleartext password! | test.cpp:76:21:76:27 | call to encrypt |
 | test.cpp:66:23:66:43 | cleartext password! | test.cpp:76:29:76:39 | thePassword |
 nodes
+| test2.cpp:63:16:63:20 | call to crypt | semmle.label | call to crypt |
+| test2.cpp:63:24:63:31 | password | semmle.label | password |
+| test2.cpp:63:24:63:31 | password | semmle.label | password |
 | test3.cpp:17:28:17:36 | password1 | semmle.label | password1 |
 | test3.cpp:17:51:17:59 | password2 | semmle.label | password2 |
 | test3.cpp:22:15:22:23 | password1 | semmle.label | password1 |
@@ -208,6 +213,11 @@ nodes
 | test3.cpp:400:15:400:23 | & ... | semmle.label | & ... |
 | test3.cpp:400:16:400:23 | password | semmle.label | password |
 | test3.cpp:400:33:400:40 | password | semmle.label | password |
+| test3.cpp:414:17:414:24 | password | semmle.label | password |
+| test3.cpp:420:17:420:24 | password | semmle.label | password |
+| test3.cpp:421:3:421:17 | call to decrypt_inplace | semmle.label | call to decrypt_inplace |
+| test3.cpp:421:21:421:28 | password | semmle.label | password |
+| test3.cpp:421:21:421:28 | password | semmle.label | password |
 | test.cpp:41:23:41:43 | cleartext password! | semmle.label | cleartext password! |
 | test.cpp:48:21:48:27 | call to encrypt | semmle.label | call to encrypt |
 | test.cpp:48:29:48:39 | thePassword | semmle.label | thePassword |
@@ -238,3 +248,5 @@ subpaths
 | test3.cpp:300:2:300:5 | call to send | test3.cpp:308:58:308:66 | password2 | test3.cpp:300:14:300:17 | data | This operation transmits 'data', which may contain unencrypted sensitive data from $@ | test3.cpp:308:58:308:66 | password2 | password2 |
 | test3.cpp:341:4:341:7 | call to recv | test3.cpp:339:9:339:16 | password | test3.cpp:341:16:341:23 | password | This operation receives into 'password', which may put unencrypted sensitive data into $@ | test3.cpp:339:9:339:16 | password | password |
 | test3.cpp:388:3:388:6 | call to recv | test3.cpp:386:8:386:15 | password | test3.cpp:388:15:388:22 | password | This operation receives into 'password', which may put unencrypted sensitive data into $@ | test3.cpp:386:8:386:15 | password | password |
+| test3.cpp:414:3:414:6 | call to recv | test3.cpp:414:17:414:24 | password | test3.cpp:414:17:414:24 | password | This operation receives into 'password', which may put unencrypted sensitive data into $@ | test3.cpp:414:17:414:24 | password | password |
+| test3.cpp:420:3:420:6 | call to recv | test3.cpp:420:17:420:24 | password | test3.cpp:420:17:420:24 | password | This operation receives into 'password', which may put unencrypted sensitive data into $@ | test3.cpp:420:17:420:24 | password | password |

cpp/ql/test/query-tests/Security/CWE/CWE-311/semmle/tests/test2.cpp

@@ -99,3 +99,14 @@ void tests(FILE *log, myStruct &s)
 		fprintf(log, "log: %s", buffer); // BAD
 	}
 }
+
+char *gets(char *s);
+
+void test_gets()
+{
+	{
+		char password[1024];
+
+		gets(password); // BAD
+	}
+}

cpp/ql/test/query-tests/Security/CWE/CWE-311/semmle/tests/test3.cpp

@@ -411,13 +411,13 @@ void test_member_password()
 	{
 		packet p;
 
-		recv(val(), p.password, 256, val()); // BAD: not encrypted [NOT DETECTED]
+		recv(val(), p.password, 256, val()); // BAD: not encrypted
 	}
 
 	{
 		packet p;
 
-		recv(val(), p.password, 256, val()); // GOOD: password is encrypted
+		recv(val(), p.password, 256, val()); // GOOD: password is encrypted [FALSE POSITIVE]
 		decrypt_inplace(p.password); // proof that `password` was in fact encrypted
 	}
 }

csharp/downgrades/ba2201248071b2bf0bb52909b35014091d2e18a6/upgrade.properties

@@ -0,0 +1,2 @@
+description: Support for compiler-generated event accessors.
+compatibility: backwards

csharp/extractor/Semmle.Extraction.CSharp/Entities/Accessor.cs

@@ -6,11 +6,15 @@ namespace Semmle.Extraction.CSharp.Entities
 {
     internal class Accessor : Method
     {
-        protected Accessor(Context cx, IMethodSymbol init)
-            : base(cx, init) { }
+        private readonly IPropertySymbol property;
+        protected Accessor(Context cx, IMethodSymbol init, IPropertySymbol property)
+            : base(cx, init)
+        {
+            this.property = property;
+        }
 
         /// <summary>
-        /// Gets the property symbol associated accessor `symbol`, or `null`
+        /// Gets the property symbol associated with accessor `symbol`, or `null`
         /// if there is no associated symbol.
         /// </summary>
         public static IPropertySymbol? GetPropertySymbol(IMethodSymbol symbol)
@@ -26,39 +30,26 @@ namespace Semmle.Extraction.CSharp.Entities
             return props.SingleOrDefault();
         }
 
-        /// <summary>
-        /// Gets the property symbol associated with this accessor.
-        /// </summary>
-        private IPropertySymbol? PropertySymbol => GetPropertySymbol(Symbol);
-
-        public new Accessor OriginalDefinition => Create(Context, Symbol.OriginalDefinition);
-
         public override void Populate(TextWriter trapFile)
         {
             PopulateMethod(trapFile);
             PopulateModifiers(trapFile);
             ContainingType!.PopulateGenerics();
 
-            var prop = PropertySymbol;
-            if (prop is null)
-            {
-                var type = Symbol.AssociatedSymbol?.GetType().ToString() ?? "null";
-                Context.ModelError(Symbol, $"Unhandled accessor associated symbol of type {type}");
-                return;
-            }
-
-            var parent = Property.Create(Context, prop);
+            var parent = Property.Create(Context, property);
             int kind;
             Accessor unboundAccessor;
-            if (SymbolEqualityComparer.Default.Equals(Symbol, prop.GetMethod))
+            if (SymbolEqualityComparer.Default.Equals(Symbol, property.GetMethod))
             {
                 kind = 1;
-                unboundAccessor = Create(Context, prop.OriginalDefinition.GetMethod!);
+                var orig = property.OriginalDefinition;
+                unboundAccessor = Create(Context, orig.GetMethod!, orig);
             }
-            else if (SymbolEqualityComparer.Default.Equals(Symbol, prop.SetMethod))
+            else if (SymbolEqualityComparer.Default.Equals(Symbol, property.SetMethod))
             {
                 kind = 2;
-                unboundAccessor = Create(Context, prop.OriginalDefinition.SetMethod!);
+                var orig = property.OriginalDefinition;
+                unboundAccessor = Create(Context, orig.SetMethod!, orig);
             }
             else
             {
@@ -84,14 +75,14 @@ namespace Semmle.Extraction.CSharp.Entities
             }
         }
 
-        public static new Accessor Create(Context cx, IMethodSymbol symbol) =>
-            AccessorFactory.Instance.CreateEntityFromSymbol(cx, symbol);
+        public static Accessor Create(Context cx, IMethodSymbol symbol, IPropertySymbol prop) =>
+            AccessorFactory.Instance.CreateEntity(cx, symbol, (symbol, prop));
 
-        private class AccessorFactory : CachedEntityFactory<IMethodSymbol, Accessor>
+        private class AccessorFactory : CachedEntityFactory<(IMethodSymbol, IPropertySymbol), Accessor>
         {
             public static AccessorFactory Instance { get; } = new AccessorFactory();
 
-            public override Accessor Create(Context cx, IMethodSymbol init) => new Accessor(cx, init);
+            public override Accessor Create(Context cx, (IMethodSymbol, IPropertySymbol) init) => new(cx, init.Item1, init.Item2);
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp/Entities/EventAccessor.cs

@@ -3,45 +3,46 @@ using System.IO;
 
 namespace Semmle.Extraction.CSharp.Entities
 {
-    internal class EventAccessor : Accessor
+    internal class EventAccessor : Method
     {
-        private EventAccessor(Context cx, IMethodSymbol init)
-            : base(cx, init) { }
+        private readonly IEventSymbol @event;
+
+        private EventAccessor(Context cx, IMethodSymbol init, IEventSymbol @event)
+            : base(cx, init)
+        {
+            this.@event = @event;
+        }
 
         /// <summary>
-        /// Gets the event symbol associated with this accessor.
+        /// Gets the event symbol associated with accessor `symbol`, or `null`
+        /// if there is no associated symbol.
         /// </summary>
-        private IEventSymbol? EventSymbol => Symbol.AssociatedSymbol as IEventSymbol;
+        public static IEventSymbol? GetEventSymbol(IMethodSymbol symbol) =>
+            symbol.AssociatedSymbol as IEventSymbol;
 
         public override void Populate(TextWriter trapFile)
         {
             PopulateMethod(trapFile);
             ContainingType!.PopulateGenerics();
 
-            var @event = EventSymbol;
-            if (@event is null)
-            {
-                var type = Symbol.AssociatedSymbol?.GetType().ToString() ?? "null";
-                Context.ModelError(Symbol, $"Unhandled event accessor associated symbol of type {type}");
-                return;
-            }
-
             var parent = Event.Create(Context, @event);
             int kind;
             EventAccessor unboundAccessor;
             if (SymbolEqualityComparer.Default.Equals(Symbol, @event.AddMethod))
             {
                 kind = 1;
-                unboundAccessor = Create(Context, @event.OriginalDefinition.AddMethod!);
+                var orig = @event.OriginalDefinition;
+                unboundAccessor = Create(Context, orig.AddMethod!, orig);
             }
             else if (SymbolEqualityComparer.Default.Equals(Symbol, @event.RemoveMethod))
             {
                 kind = 2;
-                unboundAccessor = Create(Context, @event.OriginalDefinition.RemoveMethod!);
+                var orig = @event.OriginalDefinition;
+                unboundAccessor = Create(Context, orig.RemoveMethod!, orig);
             }
             else
             {
-                Context.ModelError(Symbol, "Undhandled event accessor kind");
+                Context.ModelError(Symbol, $"Undhandled event accessor kind {Symbol.ToDisplayString()}");
                 return;
             }
 
@@ -51,16 +52,21 @@ namespace Semmle.Extraction.CSharp.Entities
                 trapFile.event_accessor_location(this, l);
 
             Overrides(trapFile);
+
+            if (Symbol.FromSource() && Block is null)
+            {
+                trapFile.compiler_generated(this);
+            }
         }
 
-        public static new EventAccessor Create(Context cx, IMethodSymbol symbol) =>
-            EventAccessorFactory.Instance.CreateEntityFromSymbol(cx, symbol);
+        public static EventAccessor Create(Context cx, IMethodSymbol symbol, IEventSymbol @event) =>
+            EventAccessorFactory.Instance.CreateEntity(cx, symbol, (symbol, @event));
 
-        private class EventAccessorFactory : CachedEntityFactory<IMethodSymbol, EventAccessor>
+        private class EventAccessorFactory : CachedEntityFactory<(IMethodSymbol, IEventSymbol), EventAccessor>
         {
             public static EventAccessorFactory Instance { get; } = new EventAccessorFactory();
 
-            public override EventAccessor Create(Context cx, IMethodSymbol init) => new EventAccessor(cx, init);
+            public override EventAccessor Create(Context cx, (IMethodSymbol, IEventSymbol) init) => new EventAccessor(cx, init.Item1, init.Item2);
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Patterns/PropertyPattern.cs

@@ -1,3 +1,5 @@
+using System;
+using System.Collections.Generic;
 using Microsoft.CodeAnalysis.CSharp.Syntax;
 using Semmle.Extraction.Kinds;
 using Semmle.Extraction.Entities;
@@ -10,17 +12,69 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
             base(new ExpressionInfo(cx, null, cx.CreateLocation(pp.GetLocation()), ExprKind.PROPERTY_PATTERN, parent, child, false, null))
         {
             child = 0;
-            var trapFile = cx.TrapWriter.Writer;
             foreach (var sub in pp.Subpatterns)
             {
-                var p = Expressions.Pattern.Create(cx, sub.Pattern, this, child++);
-                if (sub.NameColon is null)
+                if (sub.ExpressionColon is null)
                 {
-                    Context.ModelError(sub, "Expected to find 'Name:' in pattern.");
+                    Context.ModelError(sub, "Expected to find 'Expression:' in pattern.");
                     continue;
                 }
-                trapFile.exprorstmt_name(p, sub.NameColon.Name.ToString());
+                MakeExpressions(cx, this, sub, child++);
             }
         }
+
+        private record AccessStep(string Identifier, Microsoft.CodeAnalysis.Location Location);
+
+        private class AccessStepPack
+        {
+            public readonly List<AccessStep> Prefix = new List<AccessStep>();
+            public AccessStep Last { get; private set; }
+
+            public AccessStepPack Add(string identifier, Microsoft.CodeAnalysis.Location location)
+            {
+                Prefix.Add(Last);
+                Last = new AccessStep(identifier, location);
+                return this;
+            }
+
+            public AccessStepPack(string identifier, Microsoft.CodeAnalysis.Location location) =>
+                Last = new AccessStep(identifier, location);
+        }
+
+        private static AccessStepPack GetAccessStepPack(ExpressionSyntax syntax) =>
+            syntax switch
+            {
+                MemberAccessExpressionSyntax memberAccess => GetAccessStepPack(memberAccess.Expression).Add(memberAccess.Name.Identifier.ValueText, memberAccess.Name.Identifier.GetLocation()),
+                IdentifierNameSyntax identifier => new AccessStepPack(identifier.Identifier.Text, identifier.GetLocation()),
+                _ => throw new InternalError(syntax, "Unexpected expression syntax in property patterns."),
+            };
+
+        private static AccessStepPack GetAccessStepPack(BaseExpressionColonSyntax syntax) =>
+            syntax switch
+            {
+                NameColonSyntax ncs => new AccessStepPack(ncs.Name.ToString(), ncs.Name.GetLocation()),
+                ExpressionColonSyntax ecs => GetAccessStepPack(ecs.Expression),
+                _ => throw new InternalError(syntax, "Unsupported expression colon in property pattern."),
+            };
+
+        private static Expression CreateSyntheticExp(Context cx, Microsoft.CodeAnalysis.Location location, IExpressionParentEntity parent, int child) =>
+            new Expression(new ExpressionInfo(cx, null, cx.CreateLocation(location), ExprKind.PROPERTY_PATTERN, parent, child, false, null));
+
+        private static void MakeExpressions(Context cx, IExpressionParentEntity parent, SubpatternSyntax syntax, int child)
+        {
+            var trapFile = cx.TrapWriter.Writer;
+            var pack = GetAccessStepPack(syntax.ExpressionColon!);
+
+            foreach (var step in pack.Prefix)
+            {
+                var exp = CreateSyntheticExp(cx, step.Location, parent, child);
+                trapFile.exprorstmt_name(exp, step.Identifier);
+                parent = exp;
+                child = 0;
+            }
+
+            var p = Expressions.Pattern.Create(cx, syntax.Pattern, parent, child);
+            trapFile.exprorstmt_name(p, pack.Last.Identifier);
+        }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs

@@ -262,10 +262,10 @@ namespace Semmle.Extraction.CSharp.Entities
                     return Destructor.Create(cx, methodDecl);
                 case MethodKind.PropertyGet:
                 case MethodKind.PropertySet:
-                    return Accessor.GetPropertySymbol(methodDecl) is null ? OrdinaryMethod.Create(cx, methodDecl) : (Method)Accessor.Create(cx, methodDecl);
+                    return Accessor.GetPropertySymbol(methodDecl) is IPropertySymbol prop ? Accessor.Create(cx, methodDecl, prop) : OrdinaryMethod.Create(cx, methodDecl);
                 case MethodKind.EventAdd:
                 case MethodKind.EventRemove:
-                    return EventAccessor.Create(cx, methodDecl);
+                    return EventAccessor.GetEventSymbol(methodDecl) is IEventSymbol @event ? EventAccessor.Create(cx, methodDecl, @event) : OrdinaryMethod.Create(cx, methodDecl);
                 case MethodKind.UserDefinedOperator:
                 case MethodKind.BuiltinOperator:
                     return UserOperator.Create(cx, methodDecl);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs

@@ -1,7 +1,5 @@
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp.Syntax;
-using Semmle.Extraction.Entities;
-using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 
@@ -21,34 +19,10 @@ namespace Semmle.Extraction.CSharp.Entities
 
         public override void Populate(TextWriter trapFile)
         {
-            var constraints = new TypeParameterConstraints(Context);
-            trapFile.type_parameter_constraints(constraints, this);
-
-            if (Symbol.HasReferenceTypeConstraint)
-                trapFile.general_type_parameter_constraints(constraints, 1);
-
-            if (Symbol.HasValueTypeConstraint)
-                trapFile.general_type_parameter_constraints(constraints, 2);
-
-            if (Symbol.HasConstructorConstraint)
-                trapFile.general_type_parameter_constraints(constraints, 3);
-
-            if (Symbol.HasUnmanagedTypeConstraint)
-                trapFile.general_type_parameter_constraints(constraints, 4);
-
-            if (Symbol.ReferenceTypeConstraintNullableAnnotation == NullableAnnotation.Annotated)
-                trapFile.general_type_parameter_constraints(constraints, 5);
-
-            foreach (var abase in Symbol.GetAnnotatedTypeConstraints())
-            {
-                var t = Create(Context, abase.Symbol);
-                trapFile.specific_type_parameter_constraints(constraints, t.TypeRef);
-                if (!abase.HasObliviousNullability())
-                    trapFile.specific_type_parameter_nullability(constraints, t.TypeRef, NullabilityEntity.Create(Context, Nullability.Create(abase)));
-            }
-
             trapFile.types(this, Kinds.TypeKind.TYPE_PARAMETER, Symbol.Name);
 
+            TypeParameterConstraints.Create(Context, this);
+
             var parentNs = Namespace.Create(Context, Symbol.TypeParameterKind == TypeParameterKind.Method ? Context.Compilation.GlobalNamespace : Symbol.ContainingNamespace);
             trapFile.parent_namespace(this, parentNs);
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameterConstraints.cs

@@ -1,14 +1,65 @@
+using Microsoft.CodeAnalysis;
 using System.IO;
 
 namespace Semmle.Extraction.CSharp.Entities
 {
-    internal class TypeParameterConstraints : FreshEntity
+    internal class TypeParameterConstraints : CachedEntity<ITypeParameterSymbol>
     {
-        public TypeParameterConstraints(Context cx)
-            : base(cx) { }
+        private readonly TypeParameter parent;
 
-        protected override void Populate(TextWriter trapFile)
+        public TypeParameterConstraints(Context cx, TypeParameter parent)
+            : base(cx, parent.Symbol)
         {
+            this.parent = parent;
+        }
+
+        public override void WriteId(EscapingTextWriter trapFile)
+        {
+            trapFile.WriteSubId(parent);
+            trapFile.Write(";typeparameterconstraints");
+        }
+
+        public override bool NeedsPopulation => true;
+
+        public override void Populate(TextWriter trapFile)
+        {
+            trapFile.type_parameter_constraints(this, parent);
+
+            if (Symbol.HasReferenceTypeConstraint)
+                trapFile.general_type_parameter_constraints(this, 1);
+
+            if (Symbol.HasValueTypeConstraint)
+                trapFile.general_type_parameter_constraints(this, 2);
+
+            if (Symbol.HasConstructorConstraint)
+                trapFile.general_type_parameter_constraints(this, 3);
+
+            if (Symbol.HasUnmanagedTypeConstraint)
+                trapFile.general_type_parameter_constraints(this, 4);
+
+            if (Symbol.ReferenceTypeConstraintNullableAnnotation == NullableAnnotation.Annotated)
+                trapFile.general_type_parameter_constraints(this, 5);
+
+            foreach (var abase in Symbol.GetAnnotatedTypeConstraints())
+            {
+                var t = Type.Create(Context, abase.Symbol);
+                trapFile.specific_type_parameter_constraints(this, t.TypeRef);
+                if (!abase.HasObliviousNullability())
+                    trapFile.specific_type_parameter_nullability(this, t.TypeRef, NullabilityEntity.Create(Context, Nullability.Create(abase)));
+            }
+        }
+
+        public override Location? ReportingLocation => null;
+
+        public static TypeParameterConstraints Create(Context cx, TypeParameter p) =>
+            TypeParameterConstraintsFactory.Instance.CreateEntity(cx, (typeof(TypeParameterConstraints), p), p);
+
+        private class TypeParameterConstraintsFactory : CachedEntityFactory<TypeParameter, TypeParameterConstraints>
+        {
+            public static TypeParameterConstraintsFactory Instance { get; } = new TypeParameterConstraintsFactory();
+
+            public override TypeParameterConstraints Create(Context cx, TypeParameter init) => new(cx, init);
         }
     }
 }
+

csharp/ql/examples/qlpack.yml

@@ -1,4 +1,6 @@
 name: codeql/csharp-examples
-version: 0.0.2
+groups: 
+  - csharp
+  - examples
 dependencies:
   codeql/csharp-all: "*"

csharp/ql/lib/change-notes/2022-01-25-extended-property-patterns.md

@@ -0,0 +1,4 @@
+---
+category: majorAnalysis
+---
+* Added support for C# 10 [Extended property patterns](https://docs.microsoft.com/en-us/dotnet/csharp/whats-new/csharp-10#extended-property-patterns).

csharp/ql/lib/semmle/code/csharp/controlflow/internal/ControlFlowGraphImplShared.qll

@@ -214,7 +214,7 @@ abstract class SplitKind extends SplitKindBase {
   abstract string toString();
 }
 
-/** Provides the interface for implementing an entity to split on. */
+/** An interface for implementing an entity to split on. */
 abstract class SplitImpl extends Split {
   /** Gets the kind of this split. */
   abstract SplitKind getKind();
@@ -894,16 +894,31 @@ module TestOutput {
             p
             order by
               l.getFile().getBaseName(), l.getFile().getAbsolutePath(), l.getStartLine(),
-              l.getStartColumn()
+              l.getStartColumn(), l.getEndLine(), l.getEndColumn(), p.toString()
           )
       ).toString()
   }
 
   query predicate edges(RelevantNode pred, RelevantNode succ, string attr, string val) {
+    attr = "semmle.label" and
     exists(SuccessorType t | succ = getASuccessor(pred, t) |
-      attr = "semmle.label" and
       if successorTypeIsSimple(t) then val = "" else val = t.toString()
     )
+    or
+    attr = "semmle.order" and
+    val =
+      any(int i |
+        succ =
+          rank[i](RelevantNode s, SuccessorType t, Location l |
+            s = getASuccessor(pred, t) and
+            l = s.getLocation()
+          |
+            s
+            order by
+              l.getFile().getBaseName(), l.getFile().getAbsolutePath(), l.getStartLine(),
+              l.getStartColumn(), l.getEndLine(), l.getEndColumn(), t.toString()
+          )
+      ).toString()
   }
 }
 

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll

@@ -1290,7 +1290,7 @@ class DataFlowCallOption extends TDataFlowCallOption {
   }
 }
 
-/** Content tagged with the type of a containing object. */
+/** A `Content` tagged with the type of a containing object. */
 class TypedContent extends MkTypedContent {
   private Content c;
   private DataFlowType t;

csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll

@@ -90,7 +90,9 @@ module Public {
     predicate contains(SummaryComponent c) { c = this.drop(_).head() }
 
     /** Gets the bottom element of this stack. */
-    SummaryComponent bottom() { result = this.drop(this.length() - 1).head() }
+    SummaryComponent bottom() {
+      this = TSingletonSummaryComponentStack(result) or result = this.tail().bottom()
+    }
 
     /** Gets a textual representation of this stack. */
     string toString() {

csharp/ql/lib/semmlecode.csharp.dbscheme

@@ -648,7 +648,7 @@ has_modifiers(
   int id: @modifiable_direct ref,
   int mod_id: @modifier ref);
 
-compiler_generated(unique int id: @modifiable_direct ref);
+compiler_generated(unique int id: @modifiable ref);
 
 /** MEMBERS **/
 

csharp/ql/lib/tutorial.qll

@@ -6,122 +6,22 @@
  */
 class Person extends string {
   Person() {
-    this = "Ronil" or
-    this = "Dina" or
-    this = "Ravi" or
-    this = "Bruce" or
-    this = "Jo" or
-    this = "Aida" or
-    this = "Esme" or
-    this = "Charlie" or
-    this = "Fred" or
-    this = "Meera" or
-    this = "Maya" or
-    this = "Chad" or
-    this = "Tiana" or
-    this = "Laura" or
-    this = "George" or
-    this = "Will" or
-    this = "Mary" or
-    this = "Almira" or
-    this = "Susannah" or
-    this = "Rhoda" or
-    this = "Cynthia" or
-    this = "Eunice" or
-    this = "Olive" or
-    this = "Virginia" or
-    this = "Angeline" or
-    this = "Helen" or
-    this = "Cornelia" or
-    this = "Harriet" or
-    this = "Mahala" or
-    this = "Abby" or
-    this = "Margaret" or
-    this = "Deb" or
-    this = "Minerva" or
-    this = "Severus" or
-    this = "Lavina" or
-    this = "Adeline" or
-    this = "Cath" or
-    this = "Elisa" or
-    this = "Lucretia" or
-    this = "Anne" or
-    this = "Eleanor" or
-    this = "Joanna" or
-    this = "Adam" or
-    this = "Agnes" or
-    this = "Rosanna" or
-    this = "Clara" or
-    this = "Melissa" or
-    this = "Amy" or
-    this = "Isabel" or
-    this = "Jemima" or
-    this = "Cordelia" or
-    this = "Melinda" or
-    this = "Delila" or
-    this = "Jeremiah" or
-    this = "Elijah" or
-    this = "Hester" or
-    this = "Walter" or
-    this = "Oliver" or
-    this = "Hugh" or
-    this = "Aaron" or
-    this = "Reuben" or
-    this = "Eli" or
-    this = "Amos" or
-    this = "Augustus" or
-    this = "Theodore" or
-    this = "Ira" or
-    this = "Timothy" or
-    this = "Cyrus" or
-    this = "Horace" or
-    this = "Simon" or
-    this = "Asa" or
-    this = "Frank" or
-    this = "Nelson" or
-    this = "Leonard" or
-    this = "Harrison" or
-    this = "Anthony" or
-    this = "Louis" or
-    this = "Milton" or
-    this = "Noah" or
-    this = "Cornelius" or
-    this = "Abdul" or
-    this = "Warren" or
-    this = "Harvey" or
-    this = "Dennis" or
-    this = "Wesley" or
-    this = "Sylvester" or
-    this = "Gilbert" or
-    this = "Sullivan" or
-    this = "Edmund" or
-    this = "Wilson" or
-    this = "Perry" or
-    this = "Matthew" or
-    this = "Simba" or
-    this = "Nala" or
-    this = "Rafiki" or
-    this = "Shenzi" or
-    this = "Ernest" or
-    this = "Gertrude" or
-    this = "Oscar" or
-    this = "Lilian" or
-    this = "Raymond" or
-    this = "Elgar" or
-    this = "Elmer" or
-    this = "Herbert" or
-    this = "Maude" or
-    this = "Mae" or
-    this = "Otto" or
-    this = "Edwin" or
-    this = "Ophelia" or
-    this = "Parsley" or
-    this = "Sage" or
-    this = "Rosemary" or
-    this = "Thyme" or
-    this = "Garfunkel" or
-    this = "King Basil" or
-    this = "Stephen"
+    this =
+      [
+        "Ronil", "Dina", "Ravi", "Bruce", "Jo", "Aida", "Esme", "Charlie", "Fred", "Meera", "Maya",
+        "Chad", "Tiana", "Laura", "George", "Will", "Mary", "Almira", "Susannah", "Rhoda",
+        "Cynthia", "Eunice", "Olive", "Virginia", "Angeline", "Helen", "Cornelia", "Harriet",
+        "Mahala", "Abby", "Margaret", "Deb", "Minerva", "Severus", "Lavina", "Adeline", "Cath",
+        "Elisa", "Lucretia", "Anne", "Eleanor", "Joanna", "Adam", "Agnes", "Rosanna", "Clara",
+        "Melissa", "Amy", "Isabel", "Jemima", "Cordelia", "Melinda", "Delila", "Jeremiah", "Elijah",
+        "Hester", "Walter", "Oliver", "Hugh", "Aaron", "Reuben", "Eli", "Amos", "Augustus",
+        "Theodore", "Ira", "Timothy", "Cyrus", "Horace", "Simon", "Asa", "Frank", "Nelson",
+        "Leonard", "Harrison", "Anthony", "Louis", "Milton", "Noah", "Cornelius", "Abdul", "Warren",
+        "Harvey", "Dennis", "Wesley", "Sylvester", "Gilbert", "Sullivan", "Edmund", "Wilson",
+        "Perry", "Matthew", "Simba", "Nala", "Rafiki", "Shenzi", "Ernest", "Gertrude", "Oscar",
+        "Lilian", "Raymond", "Elgar", "Elmer", "Herbert", "Maude", "Mae", "Otto", "Edwin",
+        "Ophelia", "Parsley", "Sage", "Rosemary", "Thyme", "Garfunkel", "King Basil", "Stephen"
+      ]
   }
 
   /** Gets the hair color of the person. If the person is bald, there is no result. */
@@ -936,25 +836,12 @@ class Person extends string {
 
   /** Holds if the person is deceased. */
   predicate isDeceased() {
-    this = "Ernest" or
-    this = "Gertrude" or
-    this = "Oscar" or
-    this = "Lilian" or
-    this = "Edwin" or
-    this = "Raymond" or
-    this = "Elgar" or
-    this = "Elmer" or
-    this = "Herbert" or
-    this = "Maude" or
-    this = "Mae" or
-    this = "Otto" or
-    this = "Ophelia" or
-    this = "Parsley" or
-    this = "Sage" or
-    this = "Rosemary" or
-    this = "Thyme" or
-    this = "Garfunkel" or
-    this = "King Basil"
+    this =
+      [
+        "Ernest", "Gertrude", "Oscar", "Lilian", "Edwin", "Raymond", "Elgar", "Elmer", "Herbert",
+        "Maude", "Mae", "Otto", "Ophelia", "Parsley", "Sage", "Rosemary", "Thyme", "Garfunkel",
+        "King Basil"
+      ]
   }
 
   /** Gets a parent of the person (alive or deceased). */
@@ -1195,12 +1082,7 @@ class Person extends string {
   }
 
   /** Holds if the person is allowed in the region. Initially, all villagers are allowed in every region. */
-  predicate isAllowedIn(string region) {
-    region = "north" or
-    region = "south" or
-    region = "east" or
-    region = "west"
-  }
+  predicate isAllowedIn(string region) { region = ["north", "south", "east", "west"] }
 }
 
 /** Returns a parent of the person. */

csharp/ql/lib/upgrades/58a680081d22ec56c376539b7cca9521dc38fdbb/upgrade.properties

@@ -0,0 +1,2 @@
+description: Support for compiler-generated event accessors.
+compatibility: backwards

csharp/ql/src/Metrics/internal/ExtractorDiagnostics.ql

@@ -0,0 +1,24 @@
+/**
+ * @name Extractor diagnostics
+ * @description This query is for internal use only and may change without notice.
+ * @kind table
+ * @id csharp/extractor-diagnostics
+ */
+
+import csharp
+
+bindingset[i]
+private float getCompilationTimeSum(int i) {
+  result = sum(float f | compilation_time(_, _, i, f) | f)
+}
+
+select getCompilationTimeSum(0) as sum_frontend_cpu_seconds,
+  getCompilationTimeSum(1) as sum_frontend_elapsed_seconds,
+  getCompilationTimeSum(4) as sum_frontend_user_seconds,
+  getCompilationTimeSum(2) as sum_extractor_cpu_seconds,
+  getCompilationTimeSum(3) as sum_extractor_elapsed_seconds,
+  getCompilationTimeSum(5) as sum_extractor_user_seconds,
+  sum(float f | compilation_finished(_, f, _) | f) as sum_total_cpu_seconds,
+  sum(float f | compilation_finished(_, _, f) | f) as sum_total_elapsed_seconds,
+  getCompilationTimeSum(6) as sum_peak_working_set_mb,
+  max(float f | compilation_time(_, _, 6, f) | f) as max_peak_working_set_mb

csharp/ql/src/qlpack.yml

@@ -1,6 +1,8 @@
 name: codeql/csharp-queries
 version: 0.0.8-dev
-groups: csharp
+groups: 
+  - csharp
+  - queries
 suites: codeql-suites
 extractor: csharp
 defaultSuiteFile: codeql-suites/csharp-code-scanning.qls

csharp/ql/test/TestUtilities/InlineExpectationsTest.qll

@@ -93,7 +93,7 @@
 private import InlineExpectationsTestPrivate
 
 /**
- * Base class for tests with inline expectations. The test extends this class to provide the actual
+ * The base class for tests with inline expectations. The test extends this class to provide the actual
  * results of the query, which are then compared with the expected results in comments to produce a
  * list of failure messages that point out where the actual results differ from the expected
  * results.

csharp/ql/test/library-tests/csharp10/PropertyPatterns.cs

@@ -0,0 +1,56 @@
+using System;
+
+public record Point(int X, int Y);
+public record Line(Point P1, Point P2);
+public record Wrap(Line L);
+
+public class PropertyPatterns
+{
+    private static Line l = new Line(new Point(1, 2), new Point(3, 6));
+    private static Wrap w = new Wrap(l);
+
+    public void M1()
+    {
+        if (l is { P1: { X: 1 } })
+        {
+        }
+
+        if (l is { P1.X: 2 })
+        {
+        }
+    }
+
+    public void M2()
+    {
+        if (w is { L: { P2: { Y: 3 } } })
+        {
+        }
+
+        if (w is { L.P2.Y: 4 })
+        {
+        }
+    }
+
+    public void M3()
+    {
+        if (w is { L: { P2.Y: 5 } })
+        {
+        }
+
+        if (w is { L.P2: { Y: 6 } })
+        {
+        }
+    }
+
+    public void M4()
+    {
+        if (l is { P1: { X: 7 }, P1: { Y: 8 } })
+        {
+        }
+
+        if (l is { P1.X: 9, P1.Y: 10 })
+        {
+        }
+    }
+}
+

csharp/ql/test/library-tests/csharp10/propertyPatterns.expected

@@ -0,0 +1,73 @@
+propertyPatterns
+| PropertyPatterns.cs:14:18:14:33 | { ... } |
+| PropertyPatterns.cs:14:24:14:31 | { ... } |
+| PropertyPatterns.cs:18:18:18:28 | { ... } |
+| PropertyPatterns.cs:18:20:18:21 | { ... } |
+| PropertyPatterns.cs:25:18:25:40 | { ... } |
+| PropertyPatterns.cs:25:23:25:38 | { ... } |
+| PropertyPatterns.cs:25:29:25:36 | { ... } |
+| PropertyPatterns.cs:29:18:29:30 | { ... } |
+| PropertyPatterns.cs:29:20:29:20 | { ... } |
+| PropertyPatterns.cs:29:22:29:23 | { ... } |
+| PropertyPatterns.cs:36:18:36:35 | { ... } |
+| PropertyPatterns.cs:36:23:36:33 | { ... } |
+| PropertyPatterns.cs:36:25:36:26 | { ... } |
+| PropertyPatterns.cs:40:18:40:35 | { ... } |
+| PropertyPatterns.cs:40:20:40:20 | { ... } |
+| PropertyPatterns.cs:40:26:40:33 | { ... } |
+| PropertyPatterns.cs:47:18:47:47 | { ... } |
+| PropertyPatterns.cs:47:24:47:31 | { ... } |
+| PropertyPatterns.cs:47:38:47:45 | { ... } |
+| PropertyPatterns.cs:51:18:51:38 | { ... } |
+| PropertyPatterns.cs:51:20:51:21 | { ... } |
+| PropertyPatterns.cs:51:29:51:30 | { ... } |
+propertyPatternChild
+| PropertyPatterns.cs:14:18:14:33 | { ... } | 0 | PropertyPatterns.cs:14:24:14:31 | { ... } |
+| PropertyPatterns.cs:14:24:14:31 | { ... } | 0 | PropertyPatterns.cs:14:29:14:29 | 1 |
+| PropertyPatterns.cs:18:18:18:28 | { ... } | 0 | PropertyPatterns.cs:18:20:18:21 | { ... } |
+| PropertyPatterns.cs:18:20:18:21 | { ... } | 0 | PropertyPatterns.cs:18:26:18:26 | 2 |
+| PropertyPatterns.cs:25:18:25:40 | { ... } | 0 | PropertyPatterns.cs:25:23:25:38 | { ... } |
+| PropertyPatterns.cs:25:23:25:38 | { ... } | 0 | PropertyPatterns.cs:25:29:25:36 | { ... } |
+| PropertyPatterns.cs:25:29:25:36 | { ... } | 0 | PropertyPatterns.cs:25:34:25:34 | 3 |
+| PropertyPatterns.cs:29:18:29:30 | { ... } | 0 | PropertyPatterns.cs:29:20:29:20 | { ... } |
+| PropertyPatterns.cs:29:20:29:20 | { ... } | 0 | PropertyPatterns.cs:29:22:29:23 | { ... } |
+| PropertyPatterns.cs:29:22:29:23 | { ... } | 0 | PropertyPatterns.cs:29:28:29:28 | 4 |
+| PropertyPatterns.cs:36:18:36:35 | { ... } | 0 | PropertyPatterns.cs:36:23:36:33 | { ... } |
+| PropertyPatterns.cs:36:23:36:33 | { ... } | 0 | PropertyPatterns.cs:36:25:36:26 | { ... } |
+| PropertyPatterns.cs:36:25:36:26 | { ... } | 0 | PropertyPatterns.cs:36:31:36:31 | 5 |
+| PropertyPatterns.cs:40:18:40:35 | { ... } | 0 | PropertyPatterns.cs:40:20:40:20 | { ... } |
+| PropertyPatterns.cs:40:20:40:20 | { ... } | 0 | PropertyPatterns.cs:40:26:40:33 | { ... } |
+| PropertyPatterns.cs:40:26:40:33 | { ... } | 0 | PropertyPatterns.cs:40:31:40:31 | 6 |
+| PropertyPatterns.cs:47:18:47:47 | { ... } | 0 | PropertyPatterns.cs:47:24:47:31 | { ... } |
+| PropertyPatterns.cs:47:18:47:47 | { ... } | 1 | PropertyPatterns.cs:47:38:47:45 | { ... } |
+| PropertyPatterns.cs:47:24:47:31 | { ... } | 0 | PropertyPatterns.cs:47:29:47:29 | 7 |
+| PropertyPatterns.cs:47:38:47:45 | { ... } | 0 | PropertyPatterns.cs:47:43:47:43 | 8 |
+| PropertyPatterns.cs:51:18:51:38 | { ... } | 0 | PropertyPatterns.cs:51:20:51:21 | { ... } |
+| PropertyPatterns.cs:51:18:51:38 | { ... } | 1 | PropertyPatterns.cs:51:29:51:30 | { ... } |
+| PropertyPatterns.cs:51:20:51:21 | { ... } | 0 | PropertyPatterns.cs:51:26:51:26 | 9 |
+| PropertyPatterns.cs:51:29:51:30 | { ... } | 0 | PropertyPatterns.cs:51:35:51:36 | 10 |
+propertyPatternLabels
+| PropertyPatterns.cs:14:24:14:31 | { ... } | P1 |
+| PropertyPatterns.cs:14:29:14:29 | 1 | X |
+| PropertyPatterns.cs:18:20:18:21 | { ... } | P1 |
+| PropertyPatterns.cs:18:26:18:26 | 2 | X |
+| PropertyPatterns.cs:25:23:25:38 | { ... } | L |
+| PropertyPatterns.cs:25:29:25:36 | { ... } | P2 |
+| PropertyPatterns.cs:25:34:25:34 | 3 | Y |
+| PropertyPatterns.cs:29:20:29:20 | { ... } | L |
+| PropertyPatterns.cs:29:22:29:23 | { ... } | P2 |
+| PropertyPatterns.cs:29:28:29:28 | 4 | Y |
+| PropertyPatterns.cs:36:23:36:33 | { ... } | L |
+| PropertyPatterns.cs:36:25:36:26 | { ... } | P2 |
+| PropertyPatterns.cs:36:31:36:31 | 5 | Y |
+| PropertyPatterns.cs:40:20:40:20 | { ... } | L |
+| PropertyPatterns.cs:40:26:40:33 | { ... } | P2 |
+| PropertyPatterns.cs:40:31:40:31 | 6 | Y |
+| PropertyPatterns.cs:47:24:47:31 | { ... } | P1 |
+| PropertyPatterns.cs:47:29:47:29 | 7 | X |
+| PropertyPatterns.cs:47:38:47:45 | { ... } | P1 |
+| PropertyPatterns.cs:47:43:47:43 | 8 | Y |
+| PropertyPatterns.cs:51:20:51:21 | { ... } | P1 |
+| PropertyPatterns.cs:51:26:51:26 | 9 | X |
+| PropertyPatterns.cs:51:29:51:30 | { ... } | P1 |
+| PropertyPatterns.cs:51:35:51:36 | 10 | Y |

csharp/ql/test/library-tests/csharp10/propertyPatterns.ql

@@ -0,0 +1,11 @@
+import csharp
+
+query predicate propertyPatterns(PropertyPatternExpr exp) { any() }
+
+query predicate propertyPatternChild(PropertyPatternExpr pp, int n, PatternExpr child) {
+  child = pp.getPattern(n)
+}
+
+query predicate propertyPatternLabels(LabeledPatternExpr exp, string label) {
+  label = exp.getLabel()
+}

csharp/ql/test/library-tests/csharp10/recordTypes.expected

@@ -1,4 +1,7 @@
 recordTypes
+| PropertyPatterns.cs:3:1:3:34 | Point |
+| PropertyPatterns.cs:4:1:4:39 | Line |
+| PropertyPatterns.cs:5:1:5:27 | Wrap |
 | RecordTypes.cs:3:1:6:2 | MyEntry |
 | RecordTypes.cs:8:1:8:53 | MyClassRecord |
 | RecordTypes.cs:10:1:10:70 | MyReadonlyRecordStruct |
@@ -9,5 +12,8 @@ recordStructs
 | RecordTypes.cs:12:1:12:51 | MyRecordStruct1 |
 | WithExpression.cs:9:1:9:47 | MyRecordStruct2 |
 recordClass
+| PropertyPatterns.cs:3:1:3:34 | Point |
+| PropertyPatterns.cs:4:1:4:39 | Line |
+| PropertyPatterns.cs:5:1:5:27 | Wrap |
 | RecordTypes.cs:3:1:6:2 | MyEntry |
 | RecordTypes.cs:8:1:8:53 | MyClassRecord |

docs/codeql/codeql-language-guides/codeql-library-for-javascript.rst

@@ -82,7 +82,7 @@ For example, the following query computes, for each folder, the number of JavaSc
 Locations
 ^^^^^^^^^
 
-Most entities in a CodeQL database have an associated source location. Locations are identified by four pieces of information: a file, a start line, a start column, an end line, and an end column. Line and column counts are 1-based (so the first character of a file is at line 1, column 1), and the end position is inclusive.
+Most entities in a CodeQL database have an associated source location. Locations are identified by five pieces of information: a file, a start line, a start column, an end line, and an end column. Line and column counts are 1-based (so the first character of a file is at line 1, column 1), and the end position is inclusive.
 
 All entities associated with a source location belong to the class `Locatable <https://codeql.github.com/codeql-standard-libraries/javascript/semmle/javascript/Locations.qll/type.Locations$Locatable.html>`__. The location itself is modeled by the class `Location <https://codeql.github.com/codeql-standard-libraries/javascript/semmle/javascript/Locations.qll/type.Locations$Location.html>`__ and can be accessed through the member predicate ``Locatable.getLocation()``. The `Location <https://codeql.github.com/codeql-standard-libraries/javascript/semmle/javascript/Locations.qll/type.Locations$Location.html>`__ class provides the following member predicates:
 

java/documentation/library-coverage/coverage.csv

@@ -33,7 +33,7 @@ java.lang,8,,58,,,,,,,,,,8,,,,,,,,,,,,,,,46,12
 java.net,10,3,7,,,,,,,,,,,,,10,,,,,,,,,,,3,7,
 java.nio,15,,6,,13,,,,,,,,,,,,,,,,,2,,,,,,6,
 java.sql,7,,,,,,,,,,,,,,,,,,7,,,,,,,,,,
-java.util,34,,430,,,,,,,,,,34,,,,,,,,,,,,,,,16,414
+java.util,34,,438,,,,,,,,,,34,,,,,,,,,,,,,,,24,414
 javax.faces.context,2,7,,,,,,,,,,,,,,,,,,,,,,,2,,7,,
 javax.json,,,123,,,,,,,,,,,,,,,,,,,,,,,,,100,23
 javax.management.remote,2,,,,,,,,,,2,,,,,,,,,,,,,,,,,,

java/documentation/library-coverage/coverage.rst

@@ -15,9 +15,9 @@ Java framework & library support
    `Apache HttpComponents <https://hc.apache.org/>`_,"``org.apache.hc.core5.*``, ``org.apache.http``",5,136,28,,,3,,,,25
    `Google Guava <https://guava.dev/>`_,``com.google.common.*``,,728,35,,6,,,,,
    `JSON-java <https://github.com/stleary/JSON-java>`_,``org.json``,,236,,,,,,,,
-   Java Standard Library,``java.*``,3,533,111,28,,,7,,,10
+   Java Standard Library,``java.*``,3,541,111,28,,,7,,,10
    Java extensions,"``javax.*``, ``jakarta.*``",54,552,32,,,4,,1,1,2
    `Spring <https://spring.io/>`_,``org.springframework.*``,29,472,96,,,,19,14,,29
    Others,"``androidx.slice``, ``cn.hutool.core.codec``, ``com.esotericsoftware.kryo.io``, ``com.esotericsoftware.kryo5.io``, ``com.fasterxml.jackson.core``, ``com.fasterxml.jackson.databind``, ``com.opensymphony.xwork2.ognl``, ``com.unboundid.ldap.sdk``, ``flexjson``, ``groovy.lang``, ``groovy.util``, ``jodd.json``, ``net.sf.saxon.s9api``, ``ognl``, ``org.apache.commons.codec``, ``org.apache.commons.jexl2``, ``org.apache.commons.jexl3``, ``org.apache.commons.logging``, ``org.apache.commons.ognl``, ``org.apache.directory.ldap.client.api``, ``org.apache.ibatis.jdbc``, ``org.apache.log4j``, ``org.apache.logging.log4j``, ``org.apache.shiro.codec``, ``org.apache.shiro.jndi``, ``org.codehaus.groovy.control``, ``org.dom4j``, ``org.hibernate``, ``org.jboss.logging``, ``org.jooq``, ``org.mvel2``, ``org.scijava.log``, ``org.slf4j``, ``org.xml.sax``, ``org.xmlpull.v1``, ``play.mvc``, ``ratpack.core.form``, ``ratpack.core.handling``, ``ratpack.core.http``, ``ratpack.exec``, ``ratpack.form``, ``ratpack.func``, ``ratpack.handling``, ``ratpack.http``, ``ratpack.util``",44,283,921,,,,14,18,,
-   Totals,,182,6212,1424,106,6,10,107,33,1,81
+   Totals,,182,6220,1424,106,6,10,107,33,1,81
 

java/ql/examples/qlpack.yml

@@ -1,4 +1,6 @@
 name: codeql/java-examples
-version: 0.0.2
+groups: 
+  - java
+  - examples
 dependencies:
-    codeql/java-all: "*"
+  codeql/java-all: "*"

java/ql/lib/semmle/code/java/dataflow/ExternalFlow.qll

@@ -78,10 +78,12 @@ private import FlowSummary
 private module Frameworks {
   private import internal.ContainerFlow
   private import semmle.code.java.frameworks.android.Android
+  private import semmle.code.java.frameworks.android.ContentProviders
   private import semmle.code.java.frameworks.android.Intent
   private import semmle.code.java.frameworks.android.Notifications
   private import semmle.code.java.frameworks.android.Slice
   private import semmle.code.java.frameworks.android.SQLite
+  private import semmle.code.java.frameworks.android.Widget
   private import semmle.code.java.frameworks.android.XssSinks
   private import semmle.code.java.frameworks.ApacheHttp
   private import semmle.code.java.frameworks.apache.Collections

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll

@@ -1290,7 +1290,7 @@ class DataFlowCallOption extends TDataFlowCallOption {
   }
 }
 
-/** Content tagged with the type of a containing object. */
+/** A `Content` tagged with the type of a containing object. */
 class TypedContent extends MkTypedContent {
   private Content c;
   private DataFlowType t;

java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll

@@ -90,7 +90,9 @@ module Public {
     predicate contains(SummaryComponent c) { c = this.drop(_).head() }
 
     /** Gets the bottom element of this stack. */
-    SummaryComponent bottom() { result = this.drop(this.length() - 1).head() }
+    SummaryComponent bottom() {
+      this = TSingletonSummaryComponentStack(result) or result = this.tail().bottom()
+    }
 
     /** Gets a textual representation of this stack. */
     string toString() {

java/ql/lib/semmle/code/java/frameworks/Thrift.qll

@@ -25,6 +25,7 @@ class ThriftIface extends Interface {
     this.getFile() instanceof ThriftGeneratedFile
   }
 
+  /** Gets an implementation of a method of this interface. */
   Method getAnImplementingMethod() {
     result.getDeclaringType().(Class).getASupertype+() = this and
     result.overrides+(this.getAMethod()) and

java/ql/lib/semmle/code/java/frameworks/android/Android.qll

@@ -177,42 +177,6 @@ private class UriModel extends SummaryModelCsv {
   }
 }
 
-private class ContentProviderSourceModels extends SourceModelCsv {
-  override predicate row(string row) {
-    row =
-      [
-        // ContentInterface models are here for backwards compatibility (it was removed in API 28)
-        "android.content;ContentInterface;true;call;(String,String,String,Bundle);;Parameter[0..3];contentprovider",
-        "android.content;ContentProvider;true;call;(String,String,String,Bundle);;Parameter[0..3];contentprovider",
-        "android.content;ContentProvider;true;call;(String,String,Bundle);;Parameter[0..2];contentprovider",
-        "android.content;ContentProvider;true;delete;(Uri,String,String[]);;Parameter[0..2];contentprovider",
-        "android.content;ContentInterface;true;delete;(Uri,Bundle);;Parameter[0..1];contentprovider",
-        "android.content;ContentProvider;true;delete;(Uri,Bundle);;Parameter[0..1];contentprovider",
-        "android.content;ContentInterface;true;getType;(Uri);;Parameter[0];contentprovider",
-        "android.content;ContentProvider;true;getType;(Uri);;Parameter[0];contentprovider",
-        "android.content;ContentInterface;true;insert;(Uri,ContentValues,Bundle);;Parameter[0];contentprovider",
-        "android.content;ContentProvider;true;insert;(Uri,ContentValues,Bundle);;Parameter[0..2];contentprovider",
-        "android.content;ContentProvider;true;insert;(Uri,ContentValues);;Parameter[0..1];contentprovider",
-        "android.content;ContentInterface;true;openAssetFile;(Uri,String,CancellationSignal);;Parameter[0];contentprovider",
-        "android.content;ContentProvider;true;openAssetFile;(Uri,String,CancellationSignal);;Parameter[0];contentprovider",
-        "android.content;ContentProvider;true;openAssetFile;(Uri,String);;Parameter[0];contentprovider",
-        "android.content;ContentInterface;true;openTypedAssetFile;(Uri,String,Bundle,CancellationSignal);;Parameter[0..2];contentprovider",
-        "android.content;ContentProvider;true;openTypedAssetFile;(Uri,String,Bundle,CancellationSignal);;Parameter[0..2];contentprovider",
-        "android.content;ContentProvider;true;openTypedAssetFile;(Uri,String,Bundle);;Parameter[0..2];contentprovider",
-        "android.content;ContentInterface;true;openFile;(Uri,String,CancellationSignal);;Parameter[0];contentprovider",
-        "android.content;ContentProvider;true;openFile;(Uri,String,CancellationSignal);;Parameter[0];contentprovider",
-        "android.content;ContentProvider;true;openFile;(Uri,String);;Parameter[0];contentprovider",
-        "android.content;ContentInterface;true;query;(Uri,String[],Bundle,CancellationSignal);;Parameter[0..2];contentprovider",
-        "android.content;ContentProvider;true;query;(Uri,String[],Bundle,CancellationSignal);;Parameter[0..2];contentprovider",
-        "android.content;ContentProvider;true;query;(Uri,String[],String,String[],String);;Parameter[0..4];contentprovider",
-        "android.content;ContentProvider;true;query;(Uri,String[],String,String[],String,CancellationSignal);;Parameter[0..4];contentprovider",
-        "android.content;ContentInterface;true;update;(Uri,ContentValues,Bundle);;Parameter[0..2];contentprovider",
-        "android.content;ContentProvider;true;update;(Uri,ContentValues,Bundle);;Parameter[0..2];contentprovider",
-        "android.content;ContentProvider;true;update;(Uri,ContentValues,String,String[]);;Parameter[0..3];contentprovider"
-      ]
-  }
-}
-
 /** Interface for classes whose instances can be written to and restored from a Parcel. */
 class TypeParcelable extends Interface {
   TypeParcelable() { this.hasQualifiedName("android.os", "Parcelable") }

java/ql/lib/semmle/code/java/frameworks/android/ContentProviders.qll

@@ -0,0 +1,59 @@
+/**
+ * Provides classes and predicates for working with Content Providers.
+ */
+
+import java
+import semmle.code.java.dataflow.ExternalFlow
+
+/** The class `android.content.ContentValues`. */
+class ContentValues extends Class {
+  ContentValues() { this.hasQualifiedName("android.content", "ContentValues") }
+}
+
+private class ContentProviderSourceModels extends SourceModelCsv {
+  override predicate row(string row) {
+    row =
+      [
+        // ContentInterface models are here for backwards compatibility (it was removed in API 28)
+        "android.content;ContentInterface;true;call;(String,String,String,Bundle);;Parameter[0..3];contentprovider",
+        "android.content;ContentProvider;true;call;(String,String,String,Bundle);;Parameter[0..3];contentprovider",
+        "android.content;ContentProvider;true;call;(String,String,Bundle);;Parameter[0..2];contentprovider",
+        "android.content;ContentProvider;true;delete;(Uri,String,String[]);;Parameter[0..2];contentprovider",
+        "android.content;ContentInterface;true;delete;(Uri,Bundle);;Parameter[0..1];contentprovider",
+        "android.content;ContentProvider;true;delete;(Uri,Bundle);;Parameter[0..1];contentprovider",
+        "android.content;ContentInterface;true;getType;(Uri);;Parameter[0];contentprovider",
+        "android.content;ContentProvider;true;getType;(Uri);;Parameter[0];contentprovider",
+        "android.content;ContentInterface;true;insert;(Uri,ContentValues,Bundle);;Parameter[0];contentprovider",
+        "android.content;ContentProvider;true;insert;(Uri,ContentValues,Bundle);;Parameter[0..2];contentprovider",
+        "android.content;ContentProvider;true;insert;(Uri,ContentValues);;Parameter[0..1];contentprovider",
+        "android.content;ContentInterface;true;openAssetFile;(Uri,String,CancellationSignal);;Parameter[0];contentprovider",
+        "android.content;ContentProvider;true;openAssetFile;(Uri,String,CancellationSignal);;Parameter[0];contentprovider",
+        "android.content;ContentProvider;true;openAssetFile;(Uri,String);;Parameter[0];contentprovider",
+        "android.content;ContentInterface;true;openTypedAssetFile;(Uri,String,Bundle,CancellationSignal);;Parameter[0..2];contentprovider",
+        "android.content;ContentProvider;true;openTypedAssetFile;(Uri,String,Bundle,CancellationSignal);;Parameter[0..2];contentprovider",
+        "android.content;ContentProvider;true;openTypedAssetFile;(Uri,String,Bundle);;Parameter[0..2];contentprovider",
+        "android.content;ContentInterface;true;openFile;(Uri,String,CancellationSignal);;Parameter[0];contentprovider",
+        "android.content;ContentProvider;true;openFile;(Uri,String,CancellationSignal);;Parameter[0];contentprovider",
+        "android.content;ContentProvider;true;openFile;(Uri,String);;Parameter[0];contentprovider",
+        "android.content;ContentInterface;true;query;(Uri,String[],Bundle,CancellationSignal);;Parameter[0..2];contentprovider",
+        "android.content;ContentProvider;true;query;(Uri,String[],Bundle,CancellationSignal);;Parameter[0..2];contentprovider",
+        "android.content;ContentProvider;true;query;(Uri,String[],String,String[],String);;Parameter[0..4];contentprovider",
+        "android.content;ContentProvider;true;query;(Uri,String[],String,String[],String,CancellationSignal);;Parameter[0..4];contentprovider",
+        "android.content;ContentInterface;true;update;(Uri,ContentValues,Bundle);;Parameter[0..2];contentprovider",
+        "android.content;ContentProvider;true;update;(Uri,ContentValues,Bundle);;Parameter[0..2];contentprovider",
+        "android.content;ContentProvider;true;update;(Uri,ContentValues,String,String[]);;Parameter[0..3];contentprovider"
+      ]
+  }
+}
+
+private class SummaryModels extends SummaryModelCsv {
+  override predicate row(string row) {
+    row =
+      [
+        "android.content;ContentValues;false;put;;;Argument[0];MapKey of Argument[-1];value",
+        "android.content;ContentValues;false;put;;;Argument[1];MapValue of Argument[-1];value",
+        "android.content;ContentValues;false;putAll;;;MapKey of Argument[0];MapKey of Argument[-1];value",
+        "android.content;ContentValues;false;putAll;;;MapValue of Argument[0];MapValue of Argument[-1];value"
+      ]
+  }
+}

java/ql/lib/semmle/code/java/frameworks/android/SQLite.qll

@@ -1,3 +1,5 @@
+/** Provides classes and predicates for working with SQLite databases. */
+
 import java
 import Android
 import semmle.code.java.dataflow.FlowSteps
@@ -24,6 +26,20 @@ class TypeDatabaseUtils extends Class {
   TypeDatabaseUtils() { hasQualifiedName("android.database", "DatabaseUtils") }
 }
 
+/**
+ * The class `android.database.sqlite.SQLiteOpenHelper`.
+ */
+class TypeSQLiteOpenHelper extends Class {
+  TypeSQLiteOpenHelper() { this.hasQualifiedName("android.database.sqlite", "SQLiteOpenHelper") }
+}
+
+/**
+ * The class `android.database.sqlite.SQLiteStatement`.
+ */
+class TypeSQLiteStatement extends Class {
+  TypeSQLiteStatement() { this.hasQualifiedName("android.database.sqlite", "SQLiteStatement") }
+}
+
 private class SQLiteSinkCsv extends SinkModelCsv {
   override predicate row(string row) {
     row =

java/ql/lib/semmle/code/java/frameworks/android/Widget.qll

@@ -0,0 +1,23 @@
+/** Provides classes and predicates for working with Android widgets. */
+
+import java
+import semmle.code.java.dataflow.ExternalFlow
+import semmle.code.java.dataflow.FlowSources
+
+private class AndroidWidgetSourceModels extends SourceModelCsv {
+  override predicate row(string row) {
+    row = "android.widget;EditText;true;getText;;;ReturnValue;android-widget"
+  }
+}
+
+private class DefaultAndroidWidgetSources extends RemoteFlowSource {
+  DefaultAndroidWidgetSources() { sourceNode(this, "android-widget") }
+
+  override string getSourceType() { result = "Android widget source" }
+}
+
+private class AndroidWidgetSummaryModels extends SummaryModelCsv {
+  override predicate row(string row) {
+    row = "android.widget;EditText;true;getText;;;Argument[-1];ReturnValue;taint"
+  }
+}

java/ql/lib/semmle/code/java/frameworks/struts/StrutsAnnotations.qll

@@ -15,6 +15,7 @@ class StrutsAnnotation extends Annotation {
 class StrutsActionAnnotation extends StrutsAnnotation {
   StrutsActionAnnotation() { this.getType().hasName("Action") }
 
+  /** Gets a callable annotated with this annotation. */
   Callable getActionCallable() {
     result = this.getAnnotatedElement()
     or

java/ql/lib/semmle/code/java/security/CleartextStorageAndroidDatabaseQuery.qll

@@ -0,0 +1,131 @@
+/** Provides classes and predicates to reason about cleartext storage in Android databases. */
+
+import java
+import semmle.code.java.dataflow.DataFlow
+import semmle.code.java.frameworks.android.ContentProviders
+import semmle.code.java.frameworks.android.Intent
+import semmle.code.java.frameworks.android.SQLite
+import semmle.code.java.security.CleartextStorageQuery
+
+private class LocalDatabaseCleartextStorageSink extends CleartextStorageSink {
+  LocalDatabaseCleartextStorageSink() { localDatabaseInput(_, this.asExpr()) }
+}
+
+private class LocalDatabaseCleartextStorageStep extends CleartextStorageAdditionalTaintStep {
+  override predicate step(DataFlow::Node n1, DataFlow::Node n2) {
+    // EditText.getText() return type is parsed as `Object`, so we need to
+    // add a taint step for `Object.toString` to model `editText.getText().toString()`
+    exists(MethodAccess ma, Method m |
+      ma.getMethod() = m and
+      m.getDeclaringType() instanceof TypeObject and
+      m.hasName("toString")
+    |
+      n1.asExpr() = ma.getQualifier() and n2.asExpr() = ma
+    )
+  }
+}
+
+/** The creation of an object that can be used to store data in a local database. */
+class LocalDatabaseOpenMethodAccess extends Storable, Call {
+  LocalDatabaseOpenMethodAccess() {
+    exists(Method m | this.(MethodAccess).getMethod() = m |
+      m.getDeclaringType().getASupertype*() instanceof TypeSQLiteOpenHelper and
+      m.hasName("getWritableDatabase")
+      or
+      m.getDeclaringType() instanceof TypeSQLiteDatabase and
+      m.hasName(["create", "openDatabase", "openOrCreateDatabase", "compileStatement"])
+      or
+      m.getDeclaringType().getASupertype*() instanceof TypeContext and
+      m.hasName("openOrCreateDatabase")
+    )
+    or
+    this.(ClassInstanceExpr).getConstructedType() instanceof ContentValues
+  }
+
+  override Expr getAnInput() {
+    exists(LocalDatabaseFlowConfig config, DataFlow::Node database |
+      localDatabaseInput(database, result) and
+      config.hasFlow(DataFlow::exprNode(this), database)
+    )
+  }
+
+  override Expr getAStore() {
+    exists(LocalDatabaseFlowConfig config, DataFlow::Node database |
+      localDatabaseStore(database, result) and
+      config.hasFlow(DataFlow::exprNode(this), database)
+    )
+  }
+}
+
+/** A method that is both a database input and a database store. */
+private class LocalDatabaseInputStoreMethod extends Method {
+  LocalDatabaseInputStoreMethod() {
+    this.getDeclaringType() instanceof TypeSQLiteDatabase and
+    this.getName().matches("exec%SQL")
+  }
+}
+
+/**
+ * Holds if `input` is a value being prepared for being stored into the SQLite dataabse `database`.
+ * This can be done using prepared statements, using the class `ContentValues`, or directly
+ * appending `input` to a SQL query.
+ */
+private predicate localDatabaseInput(DataFlow::Node database, Argument input) {
+  exists(Method m | input.getCall().getCallee() = m |
+    m instanceof LocalDatabaseInputStoreMethod and
+    database.asExpr() = input.getCall().getQualifier()
+    or
+    m.getDeclaringType() instanceof TypeSQLiteDatabase and
+    m.hasName("compileStatement") and
+    database.asExpr() = input.getCall()
+    or
+    m.getDeclaringType() instanceof ContentValues and
+    m.hasName("put") and
+    input.getPosition() = 1 and
+    database.asExpr() = input.getCall().getQualifier()
+  )
+}
+
+/**
+ * Holds if `store` is a method call for storing a previously appended input value,
+ * either through the use of prepared statements, via the `ContentValues` class, or
+ * directly executing a raw SQL query.
+ */
+private predicate localDatabaseStore(DataFlow::Node database, MethodAccess store) {
+  exists(Method m | store.getMethod() = m |
+    m instanceof LocalDatabaseInputStoreMethod and
+    database.asExpr() = store.getQualifier()
+    or
+    m.getDeclaringType() instanceof TypeSQLiteDatabase and
+    m.getName().matches(["insert%", "replace%", "update%"]) and
+    database.asExpr() = store.getAnArgument() and
+    database.getType() instanceof ContentValues
+    or
+    m.getDeclaringType() instanceof TypeSQLiteStatement and
+    m.hasName(["executeInsert", "executeUpdateDelete"]) and
+    database.asExpr() = store.getQualifier()
+  )
+}
+
+private class LocalDatabaseFlowConfig extends DataFlow::Configuration {
+  LocalDatabaseFlowConfig() { this = "LocalDatabaseFlowConfig" }
+
+  override predicate isSource(DataFlow::Node source) {
+    source.asExpr() instanceof LocalDatabaseOpenMethodAccess
+  }
+
+  override predicate isSink(DataFlow::Node sink) {
+    localDatabaseInput(sink, _) or
+    localDatabaseStore(sink, _)
+  }
+
+  override predicate isAdditionalFlowStep(DataFlow::Node n1, DataFlow::Node n2) {
+    // Adds a step for tracking databases through field flow, that is, a database is opened and
+    // assigned to a field, and then an input or store method is called on that field elsewhere.
+    exists(Field f |
+      f.getType() instanceof TypeSQLiteDatabase and
+      f.getAnAssignedValue() = n1.asExpr() and
+      f = n2.asExpr().(FieldRead).getField()
+    )
+  }
+}

java/ql/lib/semmle/code/java/security/SensitiveActions.qll

@@ -14,7 +14,11 @@
 import java
 
 private string suspicious() {
-  result = ["%password%", "%passwd%", "%account%", "%accnt%", "%trusted%"]
+  result =
+    [
+      "%password%", "%passwd%", "pwd", "%account%", "%accnt%", "%trusted%", "%refresh%token%",
+      "%secret%token"
+    ]
 }
 
 private string nonSuspicious() {

java/ql/lib/tutorial.qll

@@ -6,122 +6,22 @@
  */
 class Person extends string {
   Person() {
-    this = "Ronil" or
-    this = "Dina" or
-    this = "Ravi" or
-    this = "Bruce" or
-    this = "Jo" or
-    this = "Aida" or
-    this = "Esme" or
-    this = "Charlie" or
-    this = "Fred" or
-    this = "Meera" or
-    this = "Maya" or
-    this = "Chad" or
-    this = "Tiana" or
-    this = "Laura" or
-    this = "George" or
-    this = "Will" or
-    this = "Mary" or
-    this = "Almira" or
-    this = "Susannah" or
-    this = "Rhoda" or
-    this = "Cynthia" or
-    this = "Eunice" or
-    this = "Olive" or
-    this = "Virginia" or
-    this = "Angeline" or
-    this = "Helen" or
-    this = "Cornelia" or
-    this = "Harriet" or
-    this = "Mahala" or
-    this = "Abby" or
-    this = "Margaret" or
-    this = "Deb" or
-    this = "Minerva" or
-    this = "Severus" or
-    this = "Lavina" or
-    this = "Adeline" or
-    this = "Cath" or
-    this = "Elisa" or
-    this = "Lucretia" or
-    this = "Anne" or
-    this = "Eleanor" or
-    this = "Joanna" or
-    this = "Adam" or
-    this = "Agnes" or
-    this = "Rosanna" or
-    this = "Clara" or
-    this = "Melissa" or
-    this = "Amy" or
-    this = "Isabel" or
-    this = "Jemima" or
-    this = "Cordelia" or
-    this = "Melinda" or
-    this = "Delila" or
-    this = "Jeremiah" or
-    this = "Elijah" or
-    this = "Hester" or
-    this = "Walter" or
-    this = "Oliver" or
-    this = "Hugh" or
-    this = "Aaron" or
-    this = "Reuben" or
-    this = "Eli" or
-    this = "Amos" or
-    this = "Augustus" or
-    this = "Theodore" or
-    this = "Ira" or
-    this = "Timothy" or
-    this = "Cyrus" or
-    this = "Horace" or
-    this = "Simon" or
-    this = "Asa" or
-    this = "Frank" or
-    this = "Nelson" or
-    this = "Leonard" or
-    this = "Harrison" or
-    this = "Anthony" or
-    this = "Louis" or
-    this = "Milton" or
-    this = "Noah" or
-    this = "Cornelius" or
-    this = "Abdul" or
-    this = "Warren" or
-    this = "Harvey" or
-    this = "Dennis" or
-    this = "Wesley" or
-    this = "Sylvester" or
-    this = "Gilbert" or
-    this = "Sullivan" or
-    this = "Edmund" or
-    this = "Wilson" or
-    this = "Perry" or
-    this = "Matthew" or
-    this = "Simba" or
-    this = "Nala" or
-    this = "Rafiki" or
-    this = "Shenzi" or
-    this = "Ernest" or
-    this = "Gertrude" or
-    this = "Oscar" or
-    this = "Lilian" or
-    this = "Raymond" or
-    this = "Elgar" or
-    this = "Elmer" or
-    this = "Herbert" or
-    this = "Maude" or
-    this = "Mae" or
-    this = "Otto" or
-    this = "Edwin" or
-    this = "Ophelia" or
-    this = "Parsley" or
-    this = "Sage" or
-    this = "Rosemary" or
-    this = "Thyme" or
-    this = "Garfunkel" or
-    this = "King Basil" or
-    this = "Stephen"
+    this =
+      [
+        "Ronil", "Dina", "Ravi", "Bruce", "Jo", "Aida", "Esme", "Charlie", "Fred", "Meera", "Maya",
+        "Chad", "Tiana", "Laura", "George", "Will", "Mary", "Almira", "Susannah", "Rhoda",
+        "Cynthia", "Eunice", "Olive", "Virginia", "Angeline", "Helen", "Cornelia", "Harriet",
+        "Mahala", "Abby", "Margaret", "Deb", "Minerva", "Severus", "Lavina", "Adeline", "Cath",
+        "Elisa", "Lucretia", "Anne", "Eleanor", "Joanna", "Adam", "Agnes", "Rosanna", "Clara",
+        "Melissa", "Amy", "Isabel", "Jemima", "Cordelia", "Melinda", "Delila", "Jeremiah", "Elijah",
+        "Hester", "Walter", "Oliver", "Hugh", "Aaron", "Reuben", "Eli", "Amos", "Augustus",
+        "Theodore", "Ira", "Timothy", "Cyrus", "Horace", "Simon", "Asa", "Frank", "Nelson",
+        "Leonard", "Harrison", "Anthony", "Louis", "Milton", "Noah", "Cornelius", "Abdul", "Warren",
+        "Harvey", "Dennis", "Wesley", "Sylvester", "Gilbert", "Sullivan", "Edmund", "Wilson",
+        "Perry", "Matthew", "Simba", "Nala", "Rafiki", "Shenzi", "Ernest", "Gertrude", "Oscar",
+        "Lilian", "Raymond", "Elgar", "Elmer", "Herbert", "Maude", "Mae", "Otto", "Edwin",
+        "Ophelia", "Parsley", "Sage", "Rosemary", "Thyme", "Garfunkel", "King Basil", "Stephen"
+      ]
   }
 
   /** Gets the hair color of the person. If the person is bald, there is no result. */
@@ -936,25 +836,12 @@ class Person extends string {
 
   /** Holds if the person is deceased. */
   predicate isDeceased() {
-    this = "Ernest" or
-    this = "Gertrude" or
-    this = "Oscar" or
-    this = "Lilian" or
-    this = "Edwin" or
-    this = "Raymond" or
-    this = "Elgar" or
-    this = "Elmer" or
-    this = "Herbert" or
-    this = "Maude" or
-    this = "Mae" or
-    this = "Otto" or
-    this = "Ophelia" or
-    this = "Parsley" or
-    this = "Sage" or
-    this = "Rosemary" or
-    this = "Thyme" or
-    this = "Garfunkel" or
-    this = "King Basil"
+    this =
+      [
+        "Ernest", "Gertrude", "Oscar", "Lilian", "Edwin", "Raymond", "Elgar", "Elmer", "Herbert",
+        "Maude", "Mae", "Otto", "Ophelia", "Parsley", "Sage", "Rosemary", "Thyme", "Garfunkel",
+        "King Basil"
+      ]
   }
 
   /** Gets a parent of the person (alive or deceased). */
@@ -1195,12 +1082,7 @@ class Person extends string {
   }
 
   /** Holds if the person is allowed in the region. Initially, all villagers are allowed in every region. */
-  predicate isAllowedIn(string region) {
-    region = "north" or
-    region = "south" or
-    region = "east" or
-    region = "west"
-  }
+  predicate isAllowedIn(string region) { region = ["north", "south", "east", "west"] }
 }
 
 /** Returns a parent of the person. */

java/ql/src/Security/CWE/CWE-312/CleartextStorageAndroidDatabase.java

@@ -0,0 +1,27 @@
+public void sqliteStorageUnsafe(Context ctx, String name, String password) {
+	// BAD - sensitive information saved in cleartext.
+	SQLiteDatabase db = ctx.openOrCreateDatabase("test", Context.MODE_PRIVATE, null);
+	db.execSQL("INSERT INTO users VALUES (?, ?)", new String[] {name, password});
+}
+
+public void sqliteStorageSafe(Context ctx, String name, String password) {
+	// GOOD - sensitive information encrypted with a custom method.
+	SQLiteDatabase db = ctx.openOrCreateDatabase("test", Context.MODE_PRIVATE, null);
+	db.execSQL("INSERT INTO users VALUES (?, ?)", new String[] {name, encrypt(password)});
+}
+
+public void sqlCipherStorageSafe(String name, String password, String databasePassword) {
+	// GOOD - sensitive information saved using SQLCipher.
+	net.sqlcipher.database.SQLiteDatabase db = 
+		net.sqlcipher.database.SQLiteDatabase.openOrCreateDatabase("test", databasePassword, null);
+	db.execSQL("INSERT INTO users VALUES (?, ?)", new String[] {name, password});
+}
+
+private static String encrypt(String cleartext) {
+    // Use an encryption or strong hashing algorithm in the real world.
+    // The example below just returns a SHA-256 hash.
+    MessageDigest digest = MessageDigest.getInstance("SHA-256");
+    byte[] hash = digest.digest(cleartext.getBytes(StandardCharsets.UTF_8));
+    String encoded = Base64.getEncoder().encodeToString(hash);
+    return encoded;
+}

java/ql/src/Security/CWE/CWE-312/CleartextStorageAndroidDatabase.qhelp

@@ -0,0 +1,36 @@
+<!DOCTYPE qhelp PUBLIC "-//Semmle//qhelp//EN" "qhelp.dtd">
+<qhelp>
+  <overview>
+    <p>
+      SQLite is a lightweight database engine commonly used in Android devices to store data. By itself, SQLite does not offer any encryption mechanism by default and stores all data in cleartext, which introduces a risk if sensitive data like credentials, authentication tokens or personal identifiable information (PII) are directly stored in a SQLite database. The information could be accessed by any process or user in rooted devices, or can be disclosed through chained vulnerabilities, like unexpected access to the private storage through exposed components. 
+    </p>
+  </overview>
+
+  <recommendation>
+    <p>
+      Use <code>SQLCipher</code> or similar libraries to add encryption capabilities to SQLite. Alternatively, encrypt sensitive data using cryptographically secure algorithms before storing it in the database.
+    </p>
+  </recommendation>
+
+  <example>
+    <p>
+      In the first example, sensitive user information is stored in cleartext.
+    </p>
+
+    <p>
+      In the second and third examples, the code encrypts sensitive information before saving it to the database.
+    </p>
+    <sample src="CleartextStorageAndroidDatabase.java" />
+  </example>
+
+  <references>
+    <li>
+      Android Developers:
+      <a href="https://developer.android.com/topic/security/data">Work with data more securely</a>
+    </li>
+    <li>
+      SQLCipher:
+      <a href="https://www.zetetic.net/sqlcipher/sqlcipher-for-android/">Android Application Integration</a>
+    </li>
+  </references>
+</qhelp>

java/ql/src/Security/CWE/CWE-312/CleartextStorageAndroidDatabase.ql

@@ -0,0 +1,23 @@
+/**
+ * @name Cleartext storage of sensitive information using a local database on Android
+ * @description Cleartext Storage of Sensitive Information using
+ *              a local database on Android allows access for users with root
+ *              privileges or unexpected exposure from chained vulnerabilities.
+ * @kind problem
+ * @problem.severity warning
+ * @precision medium
+ * @id java/android/cleartext-storage-database
+ * @tags security
+ *       external/cwe/cwe-312
+ */
+
+import java
+import semmle.code.java.security.CleartextStorageAndroidDatabaseQuery
+
+from SensitiveSource data, LocalDatabaseOpenMethodAccess s, Expr input, Expr store
+where
+  input = s.getAnInput() and
+  store = s.getAStore() and
+  data.flowsTo(input)
+select store, "SQLite database $@ containing $@ is stored $@. Data was added $@.", s, s.toString(),
+  data, "sensitive data", store, "here", input, "here"

java/ql/src/change-notes/2021-08-17-cleartext-storage-database-query.md

@@ -0,0 +1,4 @@
+---
+category: newQuery
+---
+* A new query "Cleartext storage of sensitive information using a local database on Android" (`java/android/cleartext-storage-database`) has been added. This query finds instances of sensitive data being stored in local databases without encryption, which may expose it to attackers or malicious applications.

java/ql/src/qlpack.yml

@@ -1,9 +1,11 @@
 name: codeql/java-queries
 version: 0.0.8-dev
-groups: java
+groups: 
+  - java
+  - queries
 suites: codeql-suites
 extractor: java
 defaultSuiteFile: codeql-suites/java-code-scanning.qls
 dependencies:
-    codeql/java-all: "*"
-    codeql/suite-helpers: "*"
+  codeql/java-all: "*"
+  codeql/suite-helpers: "*"

java/ql/test/TestUtilities/InlineExpectationsTest.qll

@@ -93,7 +93,7 @@
 private import InlineExpectationsTestPrivate
 
 /**
- * Base class for tests with inline expectations. The test extends this class to provide the actual
+ * The base class for tests with inline expectations. The test extends this class to provide the actual
  * results of the query, which are then compared with the expected results in comments to produce a
  * list of failure messages that point out where the actual results differ from the expected
  * results.

java/ql/test/query-tests/security/CWE-312/CleartextStorageAndroidDatabaseTest.java

@@ -0,0 +1,144 @@
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.util.Base64;
+import android.app.Activity;
+import android.content.ContentValues;
+import android.content.Context;
+import android.database.sqlite.SQLiteDatabase;
+import android.database.sqlite.SQLiteStatement;
+
+public class CleartextStorageAndroidDatabaseTest extends Activity {
+
+    public void testCleartextStorageAndroiDatabaseSafe1(Context ctx, String name, String password) {
+        SQLiteDatabase db = ctx.openOrCreateDatabase("test", Context.MODE_PRIVATE, null);
+        db.execSQL("CREATE TABLE IF NOT EXISTS users(user VARCHAR, password VARCHAR);"); // Safe
+    }
+
+    public void testCleartextStorageAndroiDatabaseSafe2(Context ctx, String name, String password) {
+        SQLiteDatabase db = ctx.openOrCreateDatabase("test", Context.MODE_PRIVATE, null);
+        db.execSQL("DROP TABLE passwords;"); // Safe - no sensitive value being stored
+    }
+
+    public void testCleartextStorageAndroiDatabase0(Context ctx, String name, String password) {
+        SQLiteDatabase db = ctx.openOrCreateDatabase("test", Context.MODE_PRIVATE, null);
+        String query = "INSERT INTO users VALUES ('" + name + "', '" + password + "');";
+        db.execSQL(query); // $ hasCleartextStorageAndroidDatabase
+    }
+
+    public void testCleartextStorageAndroiDatabase1(Context ctx, String name, String password) {
+        SQLiteDatabase db = SQLiteDatabase.openDatabase("", null, 0);
+        String query = "INSERT INTO users VALUES ('" + name + "', '" + password + "');";
+        db.execSQL(query); // $ hasCleartextStorageAndroidDatabase
+    }
+
+    public void testCleartextStorageAndroiDatabase2(String name, String password) {
+        SQLiteDatabase db = SQLiteDatabase.openOrCreateDatabase("", null);
+        String query = "INSERT INTO users VALUES (?, ?)";
+        db.execSQL(query, new String[] {name, password}); // $ hasCleartextStorageAndroidDatabase
+    }
+
+    //@formatter:off
+    public void testCleartextStorageAndroiDatabase3(String name, String password) {
+        SQLiteDatabase db = SQLiteDatabase.create(null);
+        String query = "INSERT INTO users VALUES (?, ?)";
+        db.execPerConnectionSQL(query, new String[] {name, password}); // $ hasCleartextStorageAndroidDatabase
+    }
+    //@formatter:on
+
+    public void testCleartextStorageAndroiDatabaseSafe3(String name, String password) {
+        SQLiteDatabase db = SQLiteDatabase.openDatabase("", null, 0);
+        ContentValues cv = new ContentValues();
+        cv.put("username", name);
+        cv.put("password", password); // Safe - ContentValues aren't added to any database
+    }
+
+    public void testCleartextStorageAndroiDatabase4(String name, String password) {
+        SQLiteDatabase db = SQLiteDatabase.openDatabase("", null, 0);
+        ContentValues cv = new ContentValues();
+        cv.put("username", name);
+        cv.put("password", password); // $ hasCleartextStorageAndroidDatabase
+        db.insert("table", null, cv);
+    }
+
+    public void testCleartextStorageAndroiDatabase5(String name, String password) {
+        SQLiteDatabase db = SQLiteDatabase.openDatabase("", null, 0);
+        ContentValues cv = new ContentValues();
+        cv.put("username", name);
+        cv.put("password", password); // $ hasCleartextStorageAndroidDatabase
+        db.insertOrThrow("table", null, cv);
+    }
+
+    public void testCleartextStorageAndroiDatabase6(String name, String password) {
+        SQLiteDatabase db = SQLiteDatabase.openDatabase("", null, 0);
+        ContentValues cv = new ContentValues();
+        cv.put("username", name);
+        cv.put("password", password); // $ hasCleartextStorageAndroidDatabase
+        db.insertWithOnConflict("table", null, cv, 0);
+    }
+
+    public void testCleartextStorageAndroiDatabase7(String name, String password) {
+        SQLiteDatabase db = SQLiteDatabase.openDatabase("", null, 0);
+        ContentValues cv = new ContentValues();
+        cv.put("username", name);
+        cv.put("password", password); // $ hasCleartextStorageAndroidDatabase
+        db.replace("table", null, cv);
+    }
+
+    public void testCleartextStorageAndroiDatabase8(String name, String password) {
+        SQLiteDatabase db = SQLiteDatabase.openDatabase("", null, 0);
+        ContentValues cv = new ContentValues();
+        cv.put("username", name);
+        cv.put("password", password); // $ hasCleartextStorageAndroidDatabase
+        db.replaceOrThrow("table", null, cv);
+    }
+
+    public void testCleartextStorageAndroiDatabase9(String name, String password) {
+        SQLiteDatabase db = SQLiteDatabase.openDatabase("", null, 0);
+        ContentValues cv = new ContentValues();
+        cv.put("username", name);
+        cv.put("password", password); // $ hasCleartextStorageAndroidDatabase
+        db.update("table", cv, "", new String[] {});
+    }
+
+    public void testCleartextStorageAndroiDatabase10(String name, String password) {
+        SQLiteDatabase db = SQLiteDatabase.openDatabase("", null, 0);
+        ContentValues cv = new ContentValues();
+        cv.put("username", name);
+        cv.put("password", password); // $ hasCleartextStorageAndroidDatabase
+        db.updateWithOnConflict("table", cv, "", new String[] {}, 0);
+    }
+
+    public void testCleartextStorageAndroiDatabaseSafe4(SQLiteDatabase db, String name,
+            String password) {
+        String query = "INSERT INTO users VALUES ('" + name + "', '" + password + "');";
+        SQLiteStatement stmt = db.compileStatement(query); // Safe - statement isn't executed
+    }
+
+    public void testCleartextStorageAndroiDatabase11(SQLiteDatabase db, String name,
+            String password) {
+        String query = "INSERT INTO users VALUES ('" + name + "', '" + password + "');";
+        SQLiteStatement stmt = db.compileStatement(query); // $ hasCleartextStorageAndroidDatabase
+        stmt.executeUpdateDelete();
+    }
+
+    public void testCleartextStorageAndroiDatabase12(SQLiteDatabase db, String name,
+            String password) {
+        String query = "INSERT INTO users VALUES ('" + name + "', '" + password + "');";
+        SQLiteStatement stmt = db.compileStatement(query); // $ hasCleartextStorageAndroidDatabase
+        stmt.executeInsert();
+    }
+
+    public void testCleartextStorageAndroiDatabaseSafe5(String name, String password)
+            throws Exception {
+        SQLiteDatabase db = SQLiteDatabase.create(null);
+        String query = "INSERT INTO users VALUES (?, ?)";
+        db.execSQL(query, new String[] {name, encrypt(password)}); // Safe
+    }
+
+    private static String encrypt(String cleartext) throws Exception {
+        MessageDigest digest = MessageDigest.getInstance("SHA-256");
+        byte[] hash = digest.digest(cleartext.getBytes(StandardCharsets.UTF_8));
+        String encoded = Base64.getEncoder().encodeToString(hash);
+        return encoded;
+    }
+}

java/ql/test/query-tests/security/CWE-312/CleartextStorageAndroidDatabaseTest.ql

@@ -0,0 +1,22 @@
+import java
+import semmle.code.java.security.CleartextStorageAndroidDatabaseQuery
+import TestUtilities.InlineExpectationsTest
+
+class CleartextStorageAndroidDatabaseTest extends InlineExpectationsTest {
+  CleartextStorageAndroidDatabaseTest() { this = "CleartextStorageAndroidDatabaseTest" }
+
+  override string getARelevantTag() { result = "hasCleartextStorageAndroidDatabase" }
+
+  override predicate hasActualResult(Location location, string element, string tag, string value) {
+    tag = "hasCleartextStorageAndroidDatabase" and
+    exists(SensitiveSource data, LocalDatabaseOpenMethodAccess s, Expr input, Expr store |
+      input = s.getAnInput() and
+      store = s.getAStore() and
+      data.flowsTo(input)
+    |
+      input.getLocation() = location and
+      element = input.toString() and
+      value = ""
+    )
+  }
+}

java/ql/test/stubs/google-android-9.0.0/android/annotation/IntRange.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package android.annotation;
+
+public @interface IntRange {
+    long from() default Long.MIN_VALUE;
+
+    long to() default Long.MAX_VALUE;
+
+}

java/ql/test/stubs/google-android-9.0.0/android/database/DatabaseUtils.java

@@ -0,0 +1,46 @@
+package android.database;
+
+import android.content.Context;
+import android.database.sqlite.SQLiteDatabase;
+import android.os.ParcelFileDescriptor;
+
+public class DatabaseUtils {
+
+	public static ParcelFileDescriptor blobFileDescriptorForQuery(SQLiteDatabase db, String query,
+			String[] selectionArgs) {
+		return null;
+	}
+
+	public static long longForQuery(SQLiteDatabase db, String query, String[] selectionArgs) {
+		return 0;
+
+	}
+
+	public static String stringForQuery(SQLiteDatabase db, String query, String[] selectionArgs) {
+		return null;
+
+	}
+
+	public static void createDbFromSqlStatements(Context context, String dbName, int dbVersion, String sqlStatements) {
+
+	}
+
+	public static int queryNumEntries(SQLiteDatabase db, String table, String selection) {
+		return 0;
+
+	}
+
+	public static int queryNumEntries(SQLiteDatabase db, String table, String selection, String[] selectionArgs) {
+		return 0;
+
+	}
+
+	public static String[] appendSelectionArgs(String[] originalValues, String[] newValues) {
+		return null;
+	}
+
+	public static String concatenateWhere(String a, String b) {
+		return null;
+	}
+
+}

java/ql/test/stubs/google-android-9.0.0/android/database/SQLException.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2006 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.database;
+
+public class SQLException extends RuntimeException {
+    public SQLException() {
+    }
+
+    public SQLException(String error) {
+    }
+
+    public SQLException(String error, Throwable cause) {
+    }
+
+}

java/ql/test/stubs/google-android-9.0.0/android/database/sqlite/SQLiteException.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2006 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.database.sqlite;
+import android.database.SQLException;
+
+public class SQLiteException extends SQLException {
+    public SQLiteException() {
+    }
+
+    public SQLiteException(String error) {
+    }
+
+    public SQLiteException(String error, Throwable cause) {
+    }
+
+}

java/ql/test/stubs/google-android-9.0.0/android/database/sqlite/SQLiteQueryBuilder.java

@@ -0,0 +1,57 @@
+package android.database.sqlite;
+
+import java.util.Map;
+import java.util.Set;
+
+import android.content.ContentValues;
+import android.os.CancellationSignal;
+
+public abstract class SQLiteQueryBuilder {
+	public abstract void delete(SQLiteDatabase db, String selection, String[] selectionArgs);
+
+	public abstract void insert(SQLiteDatabase db, ContentValues values);
+
+	public abstract void query(SQLiteDatabase db, String[] projectionIn, String selection, String[] selectionArgs,
+			String groupBy, String having, String sortOrder);
+
+	public abstract void query(SQLiteDatabase db, String[] projectionIn, String selection, String[] selectionArgs,
+			String groupBy, String having, String sortOrder, String limit);
+
+	public abstract void query(SQLiteDatabase db, String[] projectionIn, String selection, String[] selectionArgs,
+			String groupBy, String having, String sortOrder, String limit, CancellationSignal cancellationSignal);
+
+	public abstract void update(SQLiteDatabase db, ContentValues values, String selection, String[] selectionArgs);
+
+	public static String buildQueryString(boolean distinct, String tables, String[] columns, String where,
+			String groupBy, String having, String orderBy, String limit) {
+		return null;
+	}
+
+	public abstract String buildQuery(String[] projectionIn, String selection, String groupBy, String having, String sortOrder,
+			String limit);
+
+	public abstract String buildQuery(String[] projectionIn, String selection, String[] selectionArgs, String groupBy,
+			String having, String sortOrder, String limit);
+
+	public abstract String buildUnionQuery(String[] subQueries, String sortOrder, String limit);
+
+	public abstract String buildUnionSubQuery(String typeDiscriminatorColumn, String[] unionColumns,
+			Set<String> columnsPresentInTable, int computedColumnsOffset, String typeDiscriminatorValue,
+			String selection, String[] selectionArgs, String groupBy, String having);
+
+	public abstract String buildUnionSubQuery(String typeDiscriminatorColumn, String[] unionColumns,
+			Set<String> columnsPresentInTable, int computedColumnsOffset, String typeDiscriminatorValue,
+			String selection, String groupBy, String having);
+
+	public static void appendColumns(StringBuilder s, String[] columns) {
+	}
+
+	public abstract void setProjectionMap(Map<String, String> columnMap);
+
+	public abstract void setTables(String inTables);
+
+	public abstract void appendWhere(CharSequence inWhere);
+
+	public abstract void appendWhereStandalone(CharSequence inWhere);
+
+}

java/ql/test/stubs/google-android-9.0.0/android/webkit/WebResourceResponse.java

@@ -16,7 +16,6 @@
 package android.webkit;
 
 import java.io.InputStream;
-import java.io.StringBufferInputStream;
 import java.util.Map;
 
 /**

java/ql/test/stubs/google-android-9.0.0/android/webkit/WebSettings.java

@@ -15,11 +15,8 @@
  */
 package android.webkit;
 
+import java.net.CookieManager;
 import android.content.Context;
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
 
 /**
  * Manages settings state for a WebView. When a WebView is first created, it

java/ql/test/stubs/google-android-9.0.0/android/webkit/WebView.java

@@ -18,6 +18,7 @@ import android.content.Context;
 import android.view.View;
 
 public class WebView extends View {
+
     public WebView(Context context) {
         super(context);
     }

java/ql/test/stubs/google-android-9.0.0/android/webkit/WebViewClient.java

@@ -15,9 +15,6 @@
  */
 package android.webkit;
 
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-
 public class WebViewClient {
     /**
      * Give the host application a chance to take over the control when a new url is

javascript/ql/examples/qlpack.yml

@@ -1,4 +1,6 @@
 name: codeql/javascript-examples
-version: 0.0.3
+groups: 
+  - javascript
+  - examples
 dependencies:
   codeql/javascript-all: "*"

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/DebugResultInclusion.ql

@@ -0,0 +1,67 @@
+/**
+ * @name Debug result inclusion
+ * @description Use this query to understand why some alerts are included or excluded from the
+ *              results of boosted queries. The results for this query are the union of the alerts
+ *              generated by each boosted query. Each alert includes an explanation why it was
+ *              included or excluded for each of the four security queries.
+ * @kind problem
+ * @problem.severity error
+ * @id adaptive-threat-modeling/js/debug-result-inclusion
+ */
+
+import javascript
+import experimental.adaptivethreatmodeling.ATMConfig
+import extraction.ExtractEndpointData
+
+string getAReasonSinkExcluded(DataFlow::Node sinkCandidate, Query query) {
+  query instanceof NosqlInjectionQuery and
+  result = NosqlInjectionATM::SinkEndpointFilter::getAReasonSinkExcluded(sinkCandidate)
+  or
+  query instanceof SqlInjectionQuery and
+  result = SqlInjectionATM::SinkEndpointFilter::getAReasonSinkExcluded(sinkCandidate)
+  or
+  query instanceof TaintedPathQuery and
+  result = TaintedPathATM::SinkEndpointFilter::getAReasonSinkExcluded(sinkCandidate)
+  or
+  query instanceof XssQuery and
+  result = XssATM::SinkEndpointFilter::getAReasonSinkExcluded(sinkCandidate)
+}
+
+pragma[inline]
+string getDescriptionForAlertCandidate(
+  DataFlow::Node sourceCandidate, DataFlow::Node sinkCandidate, Query query
+) {
+  result = "excluded[reason=" + getAReasonSinkExcluded(sinkCandidate, query) + "]"
+  or
+  getATMCfg(query).isKnownSink(sinkCandidate) and
+  result = "excluded[reason=known-sink]"
+  or
+  not exists(getAReasonSinkExcluded(sinkCandidate, query)) and
+  not getDataFlowCfg(query).hasFlow(sourceCandidate, sinkCandidate) and
+  (
+    if
+      getDataFlowCfg(query).isSource(sourceCandidate) or
+      getDataFlowCfg(query).isSource(sourceCandidate, _)
+    then result = "no flow"
+    else result = "not a known source"
+  )
+  or
+  getDataFlowCfg(query).hasFlow(sourceCandidate, sinkCandidate) and
+  result = "included"
+}
+
+pragma[inline]
+string getDescriptionForAlert(DataFlow::Node sourceCandidate, DataFlow::Node sinkCandidate) {
+  result =
+    concat(Query query |
+      |
+      query.getName() + ": " +
+          getDescriptionForAlertCandidate(sourceCandidate, sinkCandidate, query), ", "
+    )
+}
+
+from DataFlow::Configuration cfg, DataFlow::Node source, DataFlow::Node sink
+where cfg.hasFlow(source, sink)
+select sink,
+  "This is an ATM result that may depend on $@ [" + getDescriptionForAlert(source, sink) + "]",
+  source, "a user-provided value"

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/EndToEndEvaluation.qll

@@ -0,0 +1,11 @@
+private import javascript
+private import extraction.Exclusions as Exclusions
+
+/**
+ * Holds if the flow from `source` to `sink` should be excluded from the results of an end-to-end
+ * evaluation query.
+ */
+pragma[inline]
+predicate isFlowExcluded(DataFlow::Node source, DataFlow::Node sink) {
+  Exclusions::isFileExcluded([source.getFile(), sink.getFile()])
+}

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/EndpointScoresIntegrationTest.ql

@@ -0,0 +1,27 @@
+/**
+ * EndpointScoresIntegrationTest.ql
+ *
+ * Extract scores for each test endpoint that is an argument to a function call in the database.
+ * This is used by integration tests to verify that QL and the modeling codebase agree on the scores
+ * of a set of test endpoints.
+ */
+
+import javascript
+import experimental.adaptivethreatmodeling.ATMConfig
+import experimental.adaptivethreatmodeling.FeaturizationConfig
+import experimental.adaptivethreatmodeling.EndpointScoring::ModelScoring as ModelScoring
+
+/**
+ * A featurization config that featurizes endpoints that are arguments to function calls.
+ *
+ * This should only be used in extraction queries and tests.
+ */
+class FunctionArgumentFeaturizationConfig extends FeaturizationConfig {
+  FunctionArgumentFeaturizationConfig() { this = "FunctionArgumentFeaturization" }
+
+  override DataFlow::Node getAnEndpointToFeaturize() {
+    exists(DataFlow::CallNode call | result = call.getAnArgument())
+  }
+}
+
+query predicate endpointScores = ModelScoring::endpointScores/3;

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/ModelCheck.ql

@@ -0,0 +1,16 @@
+/**
+ * ModelCheck.ql
+ *
+ * Returns checksums of ATM models.
+ */
+
+/**
+ * The `availableMlModels` template predicate.
+ *
+ * This is populated by the evaluator with metadata for the available machine learning models.
+ */
+external predicate availableMlModels(
+  string modelChecksum, string modelLanguage, string modelName, string modelType
+);
+
+select any(string checksum | availableMlModels(checksum, "javascript", _, _))

javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/NosqlInjection.ql

@@ -0,0 +1,24 @@
+/**
+ * NosqlInjection.ql
+ *
+ * Version of the standard NoSQL injection query with an output relation ready to plug into the
+ * evaluation pipeline.
+ */
+
+import semmle.javascript.security.dataflow.NosqlInjection
+import EndToEndEvaluation as EndToEndEvaluation
+
+from
+  DataFlow::Configuration cfg, DataFlow::Node source, DataFlow::Node sink, string filePathSink,
+  int startLineSink, int endLineSink, int startColumnSink, int endColumnSink, string filePathSource,
+  int startLineSource, int endLineSource, int startColumnSource, int endColumnSource
+where
+  cfg instanceof NosqlInjection::Configuration and
+  cfg.hasFlow(source, sink) and
+  not EndToEndEvaluation::isFlowExcluded(source, sink) and
+  sink.hasLocationInfo(filePathSink, startLineSink, startColumnSink, endLineSink, endColumnSink) and
+  source
+      .hasLocationInfo(filePathSource, startLineSource, startColumnSource, endLineSource,
+        endColumnSource)
+select source, startLineSource, startColumnSource, endLineSource, endColumnSource, filePathSource,
+  sink, startLineSink, startColumnSink, endLineSink, endColumnSink, filePathSink