wip

ATM: add hello world workflow for development of model integration te…

.github/labeler.yml

@@ -43,3 +43,11 @@ documentation:
 "QL-for-QL":
   - ql/**/*
   - .github/workflows/ql-for-ql*
+
+# Since these are all shared files that need to be synced, just pick _one_ copy of each.
+"DataFlow Library":
+  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll"
+  - "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll"

.github/workflows/atm-check-queries-run.yml

@@ -0,0 +1,56 @@
+name: ATM Check Queries Run
+
+env:
+  DB_PATH: test_db
+  ATM_MODEL_PACK: javascript/ql/experimental/adaptivethreatmodeling/src
+  QUERY_SUITE: codeql-suites/javascript-atm-code-scanning.qls
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/atm-check-queries-run.yml"
+      - "javascript/ql/experimental/adaptivethreatmodeling/**"
+  workflow_dispatch:
+
+jobs:
+  run-atm-queries:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install CodeQL CLI
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh extensions install github/gh-codeql
+          gh codeql download
+
+      - name: Install ATM model pack
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -exu
+          
+          # Install ATM model pack
+          gh codeql pack install ${ATM_MODEL_PACK}
+
+          # Retrieve model checksum
+          model_checksum=$(gh codeql resolve extensions ${ATM_MODEL_PACK}/${QUERY_SUITE} | jq -r '.models[0].checksum')
+
+          # Trust the model so that we can use it in the ATM boosted queries
+          mkdir -p "$HOME/.config/codeql"
+          echo "--insecurely-execute-ml-model-checksums ${model_checksum}" >> "$HOME/.config/codeql/config"
+
+      - name: Create test DB
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh codeql database create ${RUNNER_TEMP}/${DB_PATH} --source-root config/atm/ --language javascript 
+
+      - name: Run ATM query suite
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh codeql database run-queries -vv -- ${RUNNER_TEMP}/${DB_PATH} ${ATM_MODEL_PACK}/${QUERY_SUITE}
+      

.github/workflows/atm-model-integration-tests.yml

@@ -0,0 +1,228 @@
+name: ATM Model Integration Tests
+
+env:
+  ATM_MODEL_PACK: javascript/ql/experimental/adaptivethreatmodeling/src
+  ATM_INTEGRATION_QUERY: javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/evaluation/EndpointScoresIntegrationTest.ql
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/atm-check-queries-run.yml"
+      - "javascript/ql/experimental/adaptivethreatmodeling/**"
+  workflow_dispatch:
+
+jobs:
+  run-integration-tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+
+      - name: Install dependencies
+        run: |
+          pip install numpy pandas
+
+      - name: Install CodeQL CLI
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh extensions install github/gh-codeql
+          gh codeql download
+
+      - name: Install ATM model pack
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -exu
+          
+          # Install ATM model pack
+          gh codeql pack install ${ATM_MODEL_PACK}
+
+          # Retrieve model checksum
+          resolved_extensions=$(gh codeql resolve extensions ${ATM_INTEGRATION_QUERY})
+          model_checksum=$(jq -r '.models[0].checksum' <<< ${resolved_extensions})
+          model_path=$(jq -r '.models[0].path' <<< ${resolved_extensions})
+          echo "ML_MODEL_PATH=${model_path}" >> "${GITHUB_ENV}"
+
+          # Trust the model so that we can use it in the ATM boosted queries
+          mkdir -p "$HOME/.config/codeql"
+          echo "--insecurely-execute-ml-model-checksums ${model_checksum}" >> "$HOME/.config/codeql/config"
+
+      - name: Create test DB
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          DB_PATH=${RUNNER_TEMP}/test_db
+          echo "DB_PATH=${DB_PATH}" >> "${GITHUB_ENV}"
+          
+          # gh codeql database create "${DB_PATH}" --source-root config/atm/ --language javascript
+          
+          # TODO: hack
+          gh repo clone AmanSultanBaig/SignIn-SignUp-System-with-Nodejs -- --depth 1
+          gh codeql database create "${DB_PATH}" --source-root SignIn-SignUp-System-with-Nodejs/ --language javascript
+          
+          
+      - name: Run integration test query
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Run query
+          gh codeql query run \
+            --database "${DB_PATH}" \
+            --output "${RUNNER_TEMP}/integration_endpoint_scores.bqrs" \
+            ${ATM_INTEGRATION_QUERY}
+
+          # Decode results to csv
+          results_codeql="${RUNNER_TEMP}/integration_endpoint_scores.csv"
+          gh codeql bqrs decode \
+            --output "${results_codeql}" \
+            --entities "url" \
+            --format "csv" \
+            "${RUNNER_TEMP}/integration_endpoint_scores.bqrs"
+          echo "RESULTS_CODEQL=${results_codeql}" >> "${GITHUB_ENV}"
+
+      - name: Retrieve CodeQL and Python results
+        shell: python
+        run: |
+          import os
+          import numpy as np
+          import pandas as pd
+          from pathlib import Path
+
+          # CodeQL results
+          #---------------
+          df_codeql = pd.read_csv(os.environ['RESULTS_CODEQL'])
+          
+          # Replace 'URL for endpoint' column by 'url' column after stripping `file://`
+          # df_codeql['url'] = df_codeql['URL for endpoint'].map(lambda x: x[len('file://'):])  
+          
+          # TODO:hack
+          df_codeql['url'] = df_codeql['URL for endpoint'].map(lambda x: x.split('SignIn-SignUp-System-with-Nodejs/')[1] if 'SignIn-SignUp-System-with-Nodejs/' in x else x)
+          
+          df_codeql = df_codeql.drop(['URL for endpoint'], axis=1)
+          
+          # Remove results occuring in library code in`/opt/dist`
+          df_codeql = df_codeql[df_codeql['url'].map(lambda x: not x.startswith('/opt/dist'))]  
+
+          # Seralise it to csv
+          csv_codeql = f"{os.environ['RUNNER_TEMP']}/scored_endpoints_codeql.csv"
+          df_codeql.to_csv(csv_codeql)
+          with open(os.environ['GITHUB_ENV'], 'a') as f:
+            f.write(f"CSV_CODEQL={csv_codeql}\n")
+
+          # Python results
+          #---------------          
+          # Read endpoints locations and scores files packaged with ML model
+          model_path = Path(os.environ['ML_MODEL_PATH'])
+          df_model_locations = pd.read_csv(model_path.joinpath('model_checks', 'endpoint_locations.csv'))
+          df_model_scores = pd.read_csv(model_path.joinpath('model_checks', 'endpoint_scores.csv'))
+          
+          # Make the `url` column the same as `df_codeql` e.g. /opt/src/auth/authMiddleware.js:1:21:1:34
+          df_model_locations['url'] = \
+              df_model_locations['absolutePath'] + \
+              ':' + \
+              df_model_locations['startLine'].astype(str) + \
+              ':' + \
+              df_model_locations['startColumn'].astype(str) + \
+              ':' + \
+              df_model_locations['endLine'].astype(str) + \
+              ':' + \
+              df_model_locations['endColumn'].astype(str)
+          
+          # TODO: hack
+          df_model_locations['url'] = df_model_locations['url'].map(lambda x: x.split('/opt/src/')[1] if '/opt/src/' in x else x)
+          
+          # Merge locations and scores
+          df_model = df_model_scores.merge(df_model_locations)
+          df_model = df_model.drop(
+              [
+                  'entityName',
+                  'startLine',
+                  'startColumn',
+                  'endLine',
+                  'endColumn',
+                  'absolutePath',
+              ], axis=1
+          )
+          df_model = df_model.rename(columns={'scoreIndex': 'encodedEndpointType', 'scoreValue': 'score'})
+
+          # Seralise it to csv
+          csv_model = f"{os.environ['RUNNER_TEMP']}/scored_endpoints_pyton.csv"
+          df_model.to_csv(csv_model)
+          with open(os.environ['GITHUB_ENV'], 'a') as f:
+            f.write(f"CSV_MODEL={csv_model}\n")
+            
+      - name: Check endpoints locations
+        shell: python
+        run: |
+          # All the results in the `model_checks` directory packaged with the model appear when running 
+          # `EndpointScoresIntegrationTest.ql` against the model check DB
+
+          # NOTE: why do we have different number of results?
+          #
+          # model_check results:
+          # The model checks datataset created by the pipeline contains endpoints labelled as positive or negative
+          # examples. These endpoints are scored by the model training script once the model has been trained.
+          #
+          # codeql results:
+          # These are produced by the EndpointScoresIntegrationTest.ql query which scores all endpoints that are
+          # DataFlow::CallNode. This *happens to* encompass pretty much all possible endpoints and so will also contain
+          # the endpoints used by model_checks, along with several more.
+          #
+          # The reason we have more codeql endpoints than model_checks endpoints is thus because we do not have an easy
+          # way of selecting the same endpoints and rely on a hack to get a set of endpoints that will encompass the
+          # endpoints that we actually care about (but this is not a theoretical guarantee, more of a heuristic).
+
+          import os
+          import numpy as np
+          import pandas as pd
+          
+          # Read in data
+          df_codeql = pd.read_csv(os.environ['CSV_CODEQL'])
+          df_model = pd.read_csv(os.environ['CSV_MODEL'])
+          df_all_inner = df_codeql.merge(df_model, on=['url', 'encodedEndpointType'], suffixes=('_codeql', '_model'))
+          df_all_outer = df_codeql.merge(df_model, on=['url', 'encodedEndpointType'], suffixes=('_codeql', '_model'), how='outer', indicator=True)
+
+          # Display number of endpoints
+          print(f'Number of codeql results (i.e. from running `{os.environ["ATM_INTEGRATION_QUERY"].split("/")[-1]}`): {df_codeql.shape[0]}')
+          print(f'Number of model checks results (i.e. shipped with model): {df_model.shape[0]}')
+          print(f'Number of overlapping results: {df_all_inner.shape[0]}')
+
+          # Check all model checks results are also in codeql results
+          df_all_diff = df_all_outer[df_all_outer._merge != 'both']
+          print(f'Number of results in `model_check` but not in `codeql` (should be zero): {len(df_all_diff[df_all_diff["_merge"] == "right_only"])}')
+          print(f'Number of results in `codeql` but not in `model_checks` (expect non-zero): {len(df_all_diff[df_all_diff["_merge"] == "left_only"])}')
+          if len(df_all_diff[df_all_diff["_merge"] == "right_only"]):
+              print(f'Missing results when running `EndpointScoresIntegrationTest.ql` that are in `model_check`: {df_all_diff[df_all_diff["_merge"] == "right_only"].to_string()}')
+          assert df_model.shape[0] == df_all_inner.shape[0], f'There are missing results from `model_check` when running `EndpointScoresIntegrationTest.ql`'
+
+      - name: Check endpoints scores
+        shell: python
+        run: |
+          # The scores produced by the CodeQL query `EndpointScoresIntegrationTest.ql` should match those packaged
+          # with the model.
+          
+          import os
+          import numpy as np
+          import pandas as pd
+          
+          # Read in data
+          df_codeql = pd.read_csv(os.environ['CSV_CODEQL'])
+          df_model = pd.read_csv(os.environ['CSV_MODEL'])
+          df_all_inner = df_codeql.merge(df_model, on=['url', 'encodedEndpointType'], suffixes=('_codeql', '_model'))
+
+          # Assert the codeql and model checks scores are almost identical
+          rtol=1e-04
+          np.testing.assert_allclose(
+              df_all_inner['score_codeql'],
+              df_all_inner['score_model'],
+              rtol=rtol,
+          ), f'There are non-matching scores'
+          print(f'The scores of the {df_all_inner.shape[0]} overlapping endpoints match, using rtol={rtol}.')
+
+
+      

.github/workflows/ruby-build.yml

@@ -96,8 +96,8 @@ jobs:
       - name: Build Query Pack
         run: |
           codeql pack create ../shared/ssa --output target/packs
+          codeql pack create ../misc/suite-helpers --output target/packs
           codeql pack create ql/lib --output target/packs
-          codeql pack install ql/src
           codeql pack create ql/src --output target/packs
           PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
           codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
@@ -202,7 +202,7 @@ jobs:
           echo 'name: sample-tests
           version: 0.0.0
           dependencies:
-            codeql/ruby-all: 0.0.1
+            codeql/ruby-all: "*"
           extractor: ruby
           tests: .
           ' > qlpack.yml

.github/workflows/swift-qltest.yml

@@ -23,12 +23,23 @@ jobs:
       - uses: ./.github/actions/fetch-codeql
       - name: Check QL formatting
         run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
+  qltest-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: bazelbuild/setup-bazelisk@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version-file: 'swift/.python-version'
+      - name: Test qltest.sh
+        run: |
+          bazel test //swift/tools/test/qltest
   qltest:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os : [ubuntu-20.04, macos-latest]
+        os: [ ubuntu-20.04, macos-latest ]
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql

change-notes/1.20/analysis-javascript.md

@@ -52,7 +52,7 @@
 | Unneeded defensive code | More true positive and fewer false positive results | This query now recognizes additional defensive code patterns. |
 | Unsafe dynamic method access              | Fewer false positive results | This query no longer flags concatenated strings as unsafe method names. |
 | Unused parameter                           | Fewer false positive results | This query no longer flags parameters with leading underscore. |
-| Unused variable, import, function or class | Fewer false positive results | This query now flags fewer variables that are implictly used by JSX elements. It no longer flags variables with a leading underscore and variables in dead code. |
+| Unused variable, import, function or class | Fewer false positive results | This query now flags fewer variables that are implicitly used by JSX elements. It no longer flags variables with a leading underscore and variables in dead code. |
 | Unvalidated dynamic method call           | More true positive results | This query now flags concatenated strings as unvalidated method names in more cases. |
 | Useless assignment to property. | Fewer false positive results | This query now treats assignments with complex right-hand sides correctly. |
 | Useless conditional | Fewer results | Additional defensive coding patterns are now ignored. |

change-notes/1.23/analysis-cpp.md

@@ -19,7 +19,7 @@ The following changes in version 1.23 affect C/C++ analysis in all applications.
 | Hard-coded Japanese era start date in call (`cpp/japanese-era/constructor-or-method-with-exact-era-date`) | Deprecated | This query has been deprecated.  Use the new combined query Hard-coded Japanese era start date (`cpp/japanese-era/exact-era-date`) instead. |
 | Hard-coded Japanese era start date in struct (`cpp/japanese-era/struct-with-exact-era-date`) | Deprecated | This query has been deprecated.  Use the new combined query Hard-coded Japanese era start date (`cpp/japanese-era/exact-era-date`) instead. |
 | Hard-coded Japanese era start date (`cpp/japanese-era/exact-era-date`) | More correct results | This query now checks for the beginning date of the Reiwa era (1st May 2019). |
-| Non-constant format string (`cpp/non-constant-format`) | Fewer false positive results | Fixed false positive results triggrered by mismatching declarations of a formatting function. |
+| Non-constant format string (`cpp/non-constant-format`) | Fewer false positive results | Fixed false positive results triggered by mismatching declarations of a formatting function. |
 | Sign check of bitwise operation (`cpp/bitwise-sign-check`) | Fewer false positive results | Results involving `>=` or `<=` are no longer reported. |
 | Too few arguments to formatting function (`cpp/wrong-number-format-arguments`) | Fewer false positive results | Fixed false positive results triggered by mismatching declarations of a formatting function. |
 | Too many arguments to formatting function (`cpp/too-many-format-arguments`) | Fewer false positive results | Fixed false positive results triggered by mismatching declarations of a formatting function. |

change-notes/1.24/analysis-javascript.md

@@ -91,7 +91,7 @@
 
 ## Changes to libraries
 
-* The predicates `RegExpTerm.getSuccessor` and `RegExpTerm.getPredecessor` have been changed to reflect textual, not operational, matching order. This only makes a difference in lookbehind assertions, which are operationally matched backwards. Previously, `getSuccessor` would mimick this, so in an assertion `(?<=ab)` the term `b` would be considered the predecessor, not the successor, of `a`. Textually, however, `a` is still matched before `b`, and this is the order we now follow.
+* The predicates `RegExpTerm.getSuccessor` and `RegExpTerm.getPredecessor` have been changed to reflect textual, not operational, matching order. This only makes a difference in lookbehind assertions, which are operationally matched backwards. Previously, `getSuccessor` would mimic this, so in an assertion `(?<=ab)` the term `b` would be considered the predecessor, not the successor, of `a`. Textually, however, `a` is still matched before `b`, and this is the order we now follow.
 * An extensible model of the `EventEmitter` pattern has been implemented.
 * Taint-tracking configurations now interact differently with the `data` flow label, which may affect queries
   that combine taint-tracking and flow labels.

config/atm/ml-powered-queries-repo/add-note.js

@@ -0,0 +1,21 @@
+const mongoose = require('mongoose');
+
+Logger = require('./logger').Logger;
+Note = require('./models/note').Note;
+
+(async () => {
+  if (process.argv.length != 5) {
+    Logger.log("Creates a private note.  Usage: node add-note.js <token> <title> <body>")
+    return;
+  }
+
+  // Open the default mongoose connection
+  await mongoose.connect('mongodb://localhost:27017/notes', { useFindAndModify: false });
+
+  const [userToken, title, body] = process.argv.slice(2);
+  await Note.create({ title, body, userToken });
+
+  Logger.log(`Created private note with title ${title} and body ${body} belonging to user with token ${userToken}.`);
+
+  await mongoose.connection.close();
+})();

config/atm/ml-powered-queries-repo/app.js

@@ -0,0 +1,68 @@
+const bodyParser = require('body-parser');
+const express = require('express');
+const mongoose = require('mongoose');
+
+const notesApi = require('./notes-api');
+const usersApi = require('./users-api');
+
+const addSampleData = module.exports.addSampleData = async () => {
+  const [userA, userB] = await User.create([
+    {
+      name: "A",
+      token: "tokenA"
+    },
+    {
+      name: "B",
+      token: "tokenB"
+    }
+  ]);
+
+  await Note.create([
+    {
+      title: "Public note belonging to A",
+      body: "This is a public note belonging to A",
+      isPublic: true,
+      ownerToken: userA.token
+    },
+    {
+      title: "Public note belonging to B",
+      body: "This is a public note belonging to B",
+      isPublic: true,
+      ownerToken: userB.token
+    },
+    {
+      title: "Private note belonging to A",
+      body: "This is a private note belonging to A",
+      ownerToken: userA.token
+    },
+    {
+      title: "Private note belonging to B",
+      body: "This is a private note belonging to B",
+      ownerToken: userB.token
+    }
+  ]);
+}
+
+module.exports.startApp = async () => {
+  // Open the default mongoose connection
+  await mongoose.connect('mongodb://mongo:27017/notes', { useFindAndModify: false });
+  // Drop contents of DB
+  mongoose.connection.dropDatabase();
+  // Add some sample data
+  await addSampleData();
+
+  const app = express();
+
+  app.use(bodyParser.json());
+  app.use(bodyParser.urlencoded());
+
+  app.get('/', async (_req, res) => {
+    res.send('Hello World');
+  });
+
+  app.use('/api/notes', notesApi.router);
+  app.use('/api/users', usersApi.router);
+
+  app.listen(3000);
+  Logger.log('Express started on port 3000');
+};

config/atm/ml-powered-queries-repo/index.js

@@ -0,0 +1,7 @@
+const startApp = require('./app').startApp;
+
+Logger = require('./logger').Logger;
+Note = require('./models/note').Note;
+User = require('./models/user').User;
+
+startApp();

config/atm/ml-powered-queries-repo/logger.js

@@ -0,0 +1,5 @@
+module.exports.Logger = class {
+  log(message, ...objs) {
+    console.log(message, objs);
+  }
+};

config/atm/ml-powered-queries-repo/models/note.js

@@ -0,0 +1,8 @@
+const mongoose = require('mongoose');
+
+module.exports.Note = mongoose.model('Note', new mongoose.Schema({
+  title: String,
+  body: String,
+  ownerToken: String,
+  isPublic: Boolean
+}));

config/atm/ml-powered-queries-repo/models/user.js

@@ -0,0 +1,6 @@
+const mongoose = require('mongoose');
+
+module.exports.User = mongoose.model('User', new mongoose.Schema({
+  name: String,
+  token: String
+}));

config/atm/ml-powered-queries-repo/notes-api.js

@@ -0,0 +1,44 @@
+const express = require('express')
+
+const router = module.exports.router = express.Router();
+
+function serializeNote(note) {
+  return {
+    title: note.title,
+    body: note.body
+  };
+}
+
+router.post('/find', async (req, res) => {
+  const notes = await Note.find({
+    ownerToken: req.body.token
+  }).exec();
+  res.json({
+    notes: notes.map(serializeNote)
+  });
+});
+
+router.get('/findPublic', async (_req, res) => {
+  const notes = await Note.find({
+    isPublic: true
+  }).exec();
+  res.json({
+    notes: notes.map(serializeNote)
+  });
+});
+
+router.post('/findVisible', async (req, res) => {
+  const notes = await Note.find({
+    $or: [
+      {
+        isPublic: true
+      },
+      {
+        ownerToken: req.body.token
+      }
+    ]
+  }).exec();
+  res.json({
+    notes: notes.map(serializeNote)
+  });
+});

config/atm/ml-powered-queries-repo/read-notes.js

@@ -0,0 +1,37 @@
+const mongoose = require('mongoose');
+
+Logger = require('./logger').Logger;
+Note = require('./models/note').Note;
+User = require('./models/user').User;
+
+(async () => {
+  if (process.argv.length != 3) {
+    Logger.log("Outputs all notes visible to a user.  Usage: node read-notes.js <token>")
+    return;
+  }
+
+  // Open the default mongoose connection
+  await mongoose.connect('mongodb://localhost:27017/notes', { useFindAndModify: false });
+
+  const ownerToken = process.argv[2];
+
+  const user = await User.findOne({
+    token: ownerToken
+  }).exec();
+
+  const notes = await Note.find({
+    $or: [
+      { isPublic: true },
+      { ownerToken }
+    ]
+  }).exec();
+
+  notes.map(note => {
+    Logger.log("Title:" + note.title);
+    Logger.log("By:" + user.name);
+    Logger.log("Body:" + note.body);
+    Logger.log();
+  });
+
+  await mongoose.connection.close();
+})();

config/atm/ml-powered-queries-repo/users-api.js

@@ -0,0 +1,25 @@
+const express = require('express')
+
+Logger = require('./logger').Logger;
+const router = module.exports.router = express.Router();
+
+router.post('/updateName', async (req, res) => {
+  Logger.log("/updateName called with new name", req.body.name);
+  await User.findOneAndUpdate({
+    token: req.body.token
+  }, {
+    name: req.body.name
+  }).exec();
+  res.json({
+    name: req.body.name
+  });
+});
+
+router.post('/getName', async (req, res) => {
+  const user = await User.findOne({
+    token: req.body.token
+  }).exec();
+  res.json({
+    name: user.name
+  });
+});

cpp/ql/lib/change-notes/2022-10-22-format-literal.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Fixed bugs in the `FormatLiteral` class that were causing `getMaxConvertedLength` and related predicates to return no results when the format literal was `%e`, `%f` or `%g` and an explicit precision was specified.

cpp/ql/lib/experimental/semmle/code/cpp/semantic/analysis/ModulusAnalysis.qll

@@ -4,6 +4,12 @@
  * variable), and `v` is an integer in the range `[0 .. m-1]`.
  */
 
+/*
+ * The main recursion has base cases in both `ssaModulus` (for guarded reads) and `semExprModulus`
+ * (for constant values). The most interesting recursive case is `phiModulusRankStep`, which
+ * handles phi inputs.
+ */
+
 private import ModulusAnalysisSpecific::Private
 private import experimental.semmle.code.cpp.semantic.Semantic
 private import ConstantAnalysis
@@ -162,6 +168,11 @@ private predicate phiModulusInit(SemSsaPhiNode phi, SemBound b, int val, int mod
  */
 pragma[nomagic]
 private predicate phiModulusRankStep(SemSsaPhiNode phi, SemBound b, int val, int mod, int rix) {
+  /*
+   * base case. If any phi input is equal to `b + val` modulo `mod`, that's a potential congruence
+   * class for the phi node.
+   */
+
   rix = 0 and
   phiModulusInit(phi, b, val, mod)
   or
@@ -169,6 +180,12 @@ private predicate phiModulusRankStep(SemSsaPhiNode phi, SemBound b, int val, int
     mod != 1 and
     val = remainder(v1, mod)
   |
+    /*
+     * Recursive case. If `inp` = `b + v2` mod `m2`, we combine that with the preceding potential
+     * congruence class `b + v1` mod `m1`. The result will be the congruence class of `v1` modulo
+     * the greatest common denominator of `m1`, `m2`, and `v1 - v2`.
+     */
+
     exists(int v2, int m2 |
       rankedPhiInput(pragma[only_bind_out](phi), inp, edge, rix) and
       phiModulusRankStep(phi, b, v1, m1, rix - 1) and
@@ -176,6 +193,12 @@ private predicate phiModulusRankStep(SemSsaPhiNode phi, SemBound b, int val, int
       mod = m1.gcd(m2).gcd(v1 - v2)
     )
     or
+    /*
+     * Recursive case. If `inp` = `phi` mod `m2`, we combine that with the preceding potential
+     * congruence class `b + v1` mod `m1`. The result will be a congruence class modulo the greatest
+     * common denominator of `m1` and `m2`.
+     */
+
     exists(int m2 |
       rankedPhiInput(phi, inp, edge, rix) and
       phiModulusRankStep(phi, b, v1, m1, rix - 1) and

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.4.2
+version: 0.4.3-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/lib/semmle/code/cpp/commons/Printf.qll

@@ -1125,12 +1125,12 @@ class FormatLiteral extends Literal {
         exists(int dot, int afterdot |
           (if this.getPrecision(n) = 0 then dot = 0 else dot = 1) and
           (
-            (
-              if this.hasExplicitPrecision(n)
-              then afterdot = this.getPrecision(n)
-              else not this.hasImplicitPrecision(n)
-            ) and
-            afterdot = 6
+            if this.hasExplicitPrecision(n)
+            then afterdot = this.getPrecision(n)
+            else (
+              not this.hasImplicitPrecision(n) and
+              afterdot = 6
+            )
           ) and
           len = 1 + 309 + dot + afterdot
         ) and
@@ -1140,12 +1140,12 @@ class FormatLiteral extends Literal {
         exists(int dot, int afterdot |
           (if this.getPrecision(n) = 0 then dot = 0 else dot = 1) and
           (
-            (
-              if this.hasExplicitPrecision(n)
-              then afterdot = this.getPrecision(n)
-              else not this.hasImplicitPrecision(n)
-            ) and
-            afterdot = 6
+            if this.hasExplicitPrecision(n)
+            then afterdot = this.getPrecision(n)
+            else (
+              not this.hasImplicitPrecision(n) and
+              afterdot = 6
+            )
           ) and
           len = 1 + 1 + dot + afterdot + 1 + 1 + 3
         ) and
@@ -1155,12 +1155,12 @@ class FormatLiteral extends Literal {
         exists(int dot, int afterdot |
           (if this.getPrecision(n) = 0 then dot = 0 else dot = 1) and
           (
-            (
-              if this.hasExplicitPrecision(n)
-              then afterdot = this.getPrecision(n)
-              else not this.hasImplicitPrecision(n)
-            ) and
-            afterdot = 6
+            if this.hasExplicitPrecision(n)
+            then afterdot = this.getPrecision(n)
+            else (
+              not this.hasImplicitPrecision(n) and
+              afterdot = 6
+            )
           ) and
           // note: this could be displayed in the style %e or %f;
           //       however %f is only used when 'P > X >= -4'

cpp/ql/src/Best Practices/Likely Errors/CommaBeforeMisleadingIndentation.ql

@@ -16,15 +16,36 @@
 import cpp
 import semmle.code.cpp.commons.Exclusions
 
-/** Gets the sub-expression of 'e' with the earliest-starting Location */
+/**
+ * Gets a child of `e`, including conversions but excluding call arguments.
+ */
+pragma[inline]
+Expr getAChildWithConversions(Expr e) {
+  result.getParentWithConversions() = e and
+  not result = any(Call c).getAnArgument()
+}
+
+/**
+ * Gets the left-most column position of any transitive child of `e` (including
+ * conversions but excluding call arguments).
+ */
+int getCandidateColumn(Expr e) {
+  result = e.getLocation().getStartColumn() or
+  result = getCandidateColumn(getAChildWithConversions(e))
+}
+
+/**
+ * Gets the transitive child of `e` (including conversions but excluding call
+ * arguments) at the left-most column position, preferring less deeply nested
+ * expressions if there is a choice.
+ */
 Expr normalizeExpr(Expr e) {
-  result =
-    min(Expr child |
-      child.getParentWithConversions*() = e.getFullyConverted() and
-      not child.getParentWithConversions*() = any(Call c).getAnArgument()
-    |
-      child order by child.getLocation().getStartColumn(), count(child.getParentWithConversions*())
-    )
+  e.getLocation().getStartColumn() = min(getCandidateColumn(e)) and
+  result = e
+  or
+  not e.getLocation().getStartColumn() = min(getCandidateColumn(e)) and
+  result = normalizeExpr(getAChildWithConversions(e)) and
+  result.getLocation().getStartColumn() = min(getCandidateColumn(e))
 }
 
 predicate isParenthesized(CommaExpr ce) {
@@ -43,8 +64,8 @@ from CommaExpr ce, Expr left, Expr right, Location leftLoc, Location rightLoc
 where
   ce.fromSource() and
   not isFromMacroDefinition(ce) and
-  left = normalizeExpr(ce.getLeftOperand()) and
-  right = normalizeExpr(ce.getRightOperand()) and
+  left = normalizeExpr(ce.getLeftOperand().getFullyConverted()) and
+  right = normalizeExpr(ce.getRightOperand().getFullyConverted()) and
   leftLoc = left.getLocation() and
   rightLoc = right.getLocation() and
   not isParenthesized(ce) and

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 0.4.2
+version: 0.4.3-dev
 groups: 
   - cpp
   - queries

cpp/ql/test/TestUtilities/InlineExpectationsTest.qll

@@ -137,6 +137,7 @@ abstract class InlineExpectationsTest extends string {
   final predicate hasFailureMessage(FailureLocatable element, string message) {
     exists(ActualResult actualResult |
       actualResult.getTest() = this and
+      actualResult.getTag() = this.getARelevantTag() and
       element = actualResult and
       (
         exists(FalseNegativeExpectation falseNegative |
@@ -150,9 +151,18 @@ abstract class InlineExpectationsTest extends string {
       )
     )
     or
+    exists(ActualResult actualResult |
+      actualResult.getTest() = this and
+      not actualResult.getTag() = this.getARelevantTag() and
+      element = actualResult and
+      message =
+        "Tag mismatch: Actual result with tag '" + actualResult.getTag() +
+          "' that is not part of getARelevantTag()"
+    )
+    or
     exists(ValidExpectation expectation |
       not exists(ActualResult actualResult | expectation.matchesActualResult(actualResult)) and
-      expectation.getTag() = getARelevantTag() and
+      expectation.getTag() = this.getARelevantTag() and
       element = expectation and
       (
         expectation instanceof GoodExpectation and

cpp/ql/test/library-tests/printf/formatAttribute/AttributeFormattingFunction.cpp

@@ -0,0 +1,6 @@
+
+typedef void *va_list;
+
+int myPrintf(const char *format, ...) __attribute__((format(printf, 1, 2)));
+int mySprintf(char *buffer, const char *format, ...) __attribute__((format(__printf__, 2, 3)));
+int myVprintf(const char *format, va_list arg) __attribute__((format(printf, 1, 0)));

cpp/ql/test/library-tests/printf/formatAttribute/AttributeFormattingFunction.expected

@@ -0,0 +1,2 @@
+| AttributeFormattingFunction.cpp:4:5:4:12 | myPrintf | 0 | char | wchar_t | wchar_t |
+| AttributeFormattingFunction.cpp:5:5:5:13 | mySprintf | 1 | char | wchar_t | wchar_t |

cpp/ql/test/library-tests/printf/formatAttribute/AttributeFormattingFunction.ql

@@ -0,0 +1,5 @@
+import cpp
+
+from AttributeFormattingFunction f
+select f, f.getFormatParameterIndex(), concat(f.getDefaultCharType().toString(), ", "),
+  concat(f.getWideCharType().toString(), ", "), concat(f.getNonDefaultCharType().toString(), ", ")

cpp/ql/test/library-tests/printf/formatAttribute/FormatAttribute.expected

@@ -0,0 +1,3 @@
+| AttributeFormattingFunction.cpp:4:54:4:59 | format | printf | 0 | 1 |
+| AttributeFormattingFunction.cpp:5:69:5:74 | format | __printf__ | 1 | 2 |
+| AttributeFormattingFunction.cpp:6:63:6:68 | format | printf | 0 |  |

cpp/ql/test/library-tests/printf/formatAttribute/FormatAttribute.ql

@@ -0,0 +1,5 @@
+import cpp
+
+from FormatAttribute fa
+select fa, fa.getArchetype(), concat(fa.getFormatIndex().toString(), ", "),
+  concat(fa.getFirstFormatArgIndex().toString(), ", ")

cpp/ql/test/library-tests/printf/formatLiteral/formatLiteral.expected

@@ -0,0 +1,53 @@
+| test.c:14:9:14:10 |  | 1 |
+| test.c:15:9:15:14 |   | 2 |
+| test.c:16:9:16:12 | \t | 2 |
+| test.c:17:9:17:12 | %% | 2 |
+| test.c:20:9:20:12 | %c | 2 |
+| test.c:21:9:21:16 | %c%c%c | 4 |
+| test.c:24:9:24:23 | Hello, world! | 14 |
+| test.c:25:9:25:12 | %s | 14 |
+| test.c:26:9:26:14 | %.4s | 5 |
+| test.c:27:9:27:16 | %s, %s | 14 |
+| test.c:30:9:30:12 | %i | 12 |
+| test.c:31:9:31:14 | %lli | 12 |
+| test.c:32:9:32:12 | %i | 12 |
+| test.c:33:9:33:14 | %lli | 21 |
+| test.c:34:9:34:12 | %d | 12 |
+| test.c:35:9:35:12 | %u | 11 |
+| test.c:36:9:36:12 | %x | 9 |
+| test.c:37:9:37:12 | %X | 9 |
+| test.c:38:9:38:13 | %#x | 11 |
+| test.c:39:9:39:12 | %o | 12 |
+| test.c:40:9:40:13 | %#o | 13 |
+| test.c:43:9:43:12 | %f | 318 |
+| test.c:44:9:44:14 | %.2f | 314 |
+| test.c:45:9:45:12 | %e | 15 |
+| test.c:59:10:59:14 | %Ii | 12 |
+| test.c:66:10:66:14 | %zu | 21 |
+| test.c:67:10:67:14 | %Zu | 21 |
+| test.c:74:10:74:14 | %lc | 2 |
+| test.c:78:9:78:20 | %2$i, %1$i | 5 |
+| test.c:79:9:79:20 | %2$i, %1$i | 25 |
+| test.c:81:9:81:24 | %2$02i %1$4.2f |  |
+| test.c:85:10:85:18 | %2$*1$d |  |
+| test.c:86:10:86:19 | %2$0*1$d |  |
+| test.c:92:10:92:19 | %2$.*1$f |  |
+| test.c:99:10:99:12 | # | 2 |
+| test.c:100:10:100:13 | %% | 2 |
+| test.c:101:10:101:15 | %%%% | 3 |
+| test.c:102:10:102:15 | %%%f | 319 |
+| test.c:103:10:103:17 | %%%%%f | 320 |
+| test.c:104:10:104:18 | %4.2f%% | 315 |
+| test.c:105:10:105:17 | %%%f%% | 320 |
+| test.c:112:10:112:13 | %f | 318 |
+| test.c:113:10:113:15 | %.1f | 313 |
+| test.c:114:10:114:14 | %1f | 318 |
+| test.c:115:10:115:16 | %1.1f | 313 |
+| test.c:116:10:116:13 | %e | 15 |
+| test.c:117:10:117:15 | %.2e | 11 |
+| test.c:118:10:118:14 | %3e | 15 |
+| test.c:119:10:119:16 | %3.2e | 11 |
+| test.c:120:10:120:13 | %g | 15 |
+| test.c:121:10:121:15 | %.1g | 10 |
+| test.c:122:10:122:14 | %4g | 15 |
+| test.c:123:10:123:16 | %4.1g | 10 |

cpp/ql/test/library-tests/printf/formatLiteral/formatLiteral.ql

@@ -0,0 +1,4 @@
+import semmle.code.cpp.commons.Printf
+
+from FormatLiteral fl
+select fl, concat(fl.getMaxConvertedLength().toString(), ", ")

cpp/ql/test/library-tests/printf/formatLiteral/formatType.expected

@@ -0,0 +1,51 @@
+| test.c:20:9:20:12 | %c | 0 |  | c |  | file://:0:0:0:0 | char |
+| test.c:21:9:21:16 | %c%c%c | 0 |  | c |  | file://:0:0:0:0 | char |
+| test.c:21:9:21:16 | %c%c%c | 1 |  | c |  | file://:0:0:0:0 | char |
+| test.c:21:9:21:16 | %c%c%c | 2 |  | c |  | file://:0:0:0:0 | char |
+| test.c:25:9:25:12 | %s | 0 |  | s |  | file://:0:0:0:0 | char * |
+| test.c:26:9:26:14 | %.4s | 0 |  | s |  | file://:0:0:0:0 | char * |
+| test.c:27:9:27:16 | %s, %s | 0 |  | s |  | file://:0:0:0:0 | char * |
+| test.c:27:9:27:16 | %s, %s | 1 |  | s |  | file://:0:0:0:0 | char * |
+| test.c:30:9:30:12 | %i | 0 |  | i |  | file://:0:0:0:0 | int |
+| test.c:31:9:31:14 | %lli | 0 |  | i | ll | file://:0:0:0:0 | long long |
+| test.c:32:9:32:12 | %i | 0 |  | i |  | file://:0:0:0:0 | int |
+| test.c:33:9:33:14 | %lli | 0 |  | i | ll | file://:0:0:0:0 | long long |
+| test.c:34:9:34:12 | %d | 0 |  | d |  | file://:0:0:0:0 | int |
+| test.c:35:9:35:12 | %u | 0 |  | u |  | file://:0:0:0:0 | unsigned int |
+| test.c:36:9:36:12 | %x | 0 |  | x |  | file://:0:0:0:0 | unsigned int |
+| test.c:37:9:37:12 | %X | 0 |  | X |  | file://:0:0:0:0 | unsigned int |
+| test.c:38:9:38:13 | %#x | 0 |  | x |  | file://:0:0:0:0 | unsigned int |
+| test.c:39:9:39:12 | %o | 0 |  | o |  | file://:0:0:0:0 | unsigned int |
+| test.c:40:9:40:13 | %#o | 0 |  | o |  | file://:0:0:0:0 | unsigned int |
+| test.c:43:9:43:12 | %f | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:44:9:44:14 | %.2f | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:45:9:45:12 | %e | 0 |  | e |  | file://:0:0:0:0 | double |
+| test.c:59:10:59:14 | %Ii | 0 |  | i |  | file://:0:0:0:0 | int |
+| test.c:66:10:66:14 | %zu | 0 |  | u | z | test.c:50:27:50:32 | size_t |
+| test.c:67:10:67:14 | %Zu | 0 |  | u | Z | test.c:50:27:50:32 | size_t |
+| test.c:74:10:74:14 | %lc | 0 |  | c | l | file://:0:0:0:0 | wchar_t |
+| test.c:78:9:78:20 | %2$i, %1$i | 0 | 2$ | i |  | file://:0:0:0:0 | int |
+| test.c:78:9:78:20 | %2$i, %1$i | 1 | 1$ | i |  | file://:0:0:0:0 | int |
+| test.c:79:9:79:20 | %2$i, %1$i | 0 | 2$ | i |  | file://:0:0:0:0 | int |
+| test.c:79:9:79:20 | %2$i, %1$i | 1 | 1$ | i |  | file://:0:0:0:0 | int |
+| test.c:81:9:81:24 | %2$02i %1$4.2f | 0 | 2$ | i |  | file://:0:0:0:0 | int |
+| test.c:81:9:81:24 | %2$02i %1$4.2f | 1 | 1$ | f |  | file://:0:0:0:0 | double |
+| test.c:85:10:85:18 | %2$*1$d | 0 | 2$ | d |  | file://:0:0:0:0 | int |
+| test.c:86:10:86:19 | %2$0*1$d | 0 | 2$ | d |  | file://:0:0:0:0 | int |
+| test.c:92:10:92:19 | %2$.*1$f | 0 | 2$ | f |  | file://:0:0:0:0 | double |
+| test.c:102:10:102:15 | %%%f | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:103:10:103:17 | %%%%%f | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:104:10:104:18 | %4.2f%% | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:105:10:105:17 | %%%f%% | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:112:10:112:13 | %f | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:113:10:113:15 | %.1f | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:114:10:114:14 | %1f | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:115:10:115:16 | %1.1f | 0 |  | f |  | file://:0:0:0:0 | double |
+| test.c:116:10:116:13 | %e | 0 |  | e |  | file://:0:0:0:0 | double |
+| test.c:117:10:117:15 | %.2e | 0 |  | e |  | file://:0:0:0:0 | double |
+| test.c:118:10:118:14 | %3e | 0 |  | e |  | file://:0:0:0:0 | double |
+| test.c:119:10:119:16 | %3.2e | 0 |  | e |  | file://:0:0:0:0 | double |
+| test.c:120:10:120:13 | %g | 0 |  | g |  | file://:0:0:0:0 | double |
+| test.c:121:10:121:15 | %.1g | 0 |  | g |  | file://:0:0:0:0 | double |
+| test.c:122:10:122:14 | %4g | 0 |  | g |  | file://:0:0:0:0 | double |
+| test.c:123:10:123:16 | %4.1g | 0 |  | g |  | file://:0:0:0:0 | double |

cpp/ql/test/library-tests/printf/formatLiteral/formatType.ql

@@ -0,0 +1,6 @@
+import semmle.code.cpp.commons.Printf
+
+from FormatLiteral fl, int i
+select fl, i, concat(fl.getParameterField(i).toString(), ", "), fl.getConversionChar(i),
+  fl.getLength(i), concat(fl.getConversionType(i).getLocation().toString(), ", "),
+  concat(fl.getConversionType(i).toString(), ", ")

cpp/ql/test/library-tests/printf/formatLiteral/test.c

@@ -0,0 +1,125 @@
+/** standard printf functions */
+
+int printf(const char *format, ...);
+
+/** test program */
+
+int main(int argc, char *argv[])
+{
+	long long int lli;
+	double d;
+	int i;
+
+	// constant expressions
+	printf("");
+	printf("\x20");
+	printf("\t");
+	printf("%%");
+
+	// characters
+	printf("%c", 'a');
+	printf("%c%c%c", 'a', 'b', 'c');
+
+	// strings
+	printf("Hello, world!");
+	printf("%s", "Hello, world!");
+	printf("%.4s", "Hello, world!");
+	printf("%s, %s", "Hello", "world!");
+
+	// integers
+	printf("%i", i);
+	printf("%lli", i);
+	printf("%i", lli);
+	printf("%lli", lli);
+	printf("%d", i);
+	printf("%u", i);
+	printf("%x", i);
+	printf("%X", i);
+	printf("%#x", i);
+	printf("%o", i);
+	printf("%#o", i);
+
+	// doubles
+	printf("%f", d);
+	printf("%.2f", d);
+	printf("%e", d);
+
+	return 0;
+}
+
+typedef long unsigned int size_t;
+typedef unsigned int wint_t;
+
+void more_cases(int a, int b)
+{
+	// integers
+	{
+		int i;
+
+		printf("%Ii", i); // glibc 2.2 'I' prefix
+	}
+
+	// size_t
+	{
+		size_t st;
+
+		printf("%zu", st); // size_t
+		printf("%Zu", st); // non-standard synonym for 'z'
+	}
+
+	// wint_t
+	{
+		wint_t wt;
+
+		printf("%lc", wt); // wide character
+	}
+
+	// posix indexed format arguments
+	printf("%2$i, %1$i", 1, 2); // '2, 1'
+	printf("%2$i, %1$i", a, b);
+	
+	printf("%2$02i %1$4.2f", 3.3333f, 6); // 06, 3.33
+	{
+		int width, num;
+
+		printf("%2$*1$d", width, num);
+		printf("%2$0*1$d", width, num);
+	}
+	{
+		int precision;
+		float num;
+
+		printf("%2$.*1$f", precision, num);
+	}
+
+	// %%
+	{
+		float num;
+
+		printf("#");
+		printf("%%");
+		printf("%%%%");
+		printf("%%%f", num);
+		printf("%%%%%f", num);
+		printf("%4.2f%%", num);
+		printf("%%%f%%", num);
+	}
+
+	// more tests of width and precision
+	{
+		float num;
+
+		printf("%f", num);
+		printf("%.1f", num);
+		printf("%1f", num);
+		printf("%1.1f", num);
+		printf("%e", num);
+		printf("%.2e", num);
+		printf("%3e", num);
+		printf("%3.2e", num);
+		printf("%g", num);
+		printf("%.1g", num);
+		printf("%4g", num);
+		printf("%4.1g", num);
+	}
+}

cpp/ql/test/library-tests/templates/CPP-223/decls.expected

@@ -13,6 +13,7 @@
 | test.cpp:4:26:4:26 | c<<expression>> |
 | test.cpp:4:26:4:26 | c<<unnamed>> |
 | test.cpp:5:29:5:29 | e |
+| test.cpp:6:24:6:24 | f |
 | test.cpp:6:26:6:26 | (unnamed parameter 0) |
 | test.cpp:6:29:6:31 | (unnamed parameter 1) |
 | test.cpp:7:20:7:20 | f |

cpp/ql/test/query-tests/Likely Bugs/Format/NonConstantFormat/NonConstantFormat.expected

@@ -3,19 +3,20 @@
 | nested.cpp:21:23:21:26 | fmt0 | The format string argument to snprintf should be constant to prevent security issues and other potential errors. |
 | nested.cpp:79:32:79:38 | call to get_fmt | The format string argument to diagnostic should be constant to prevent security issues and other potential errors. |
 | nested.cpp:87:18:87:20 | fmt | The format string argument to diagnostic should be constant to prevent security issues and other potential errors. |
-| test.cpp:50:10:50:21 | call to make_message | The format string argument to printf should be constant to prevent security issues and other potential errors. |
-| test.cpp:56:12:56:16 | hello | The format string argument to printf should be constant to prevent security issues and other potential errors. |
-| test.cpp:59:12:59:21 | call to const_wash | The format string argument to printf should be constant to prevent security issues and other potential errors. |
-| test.cpp:60:12:60:26 | ... + ... | The format string argument to printf should be constant to prevent security issues and other potential errors. |
-| test.cpp:61:12:61:17 | + ... | The format string argument to printf should be constant to prevent security issues and other potential errors. |
-| test.cpp:62:12:62:18 | * ... | The format string argument to printf should be constant to prevent security issues and other potential errors. |
-| test.cpp:63:12:63:18 | & ... | The format string argument to printf should be constant to prevent security issues and other potential errors. |
-| test.cpp:64:12:64:39 | ... + ... | The format string argument to printf should be constant to prevent security issues and other potential errors. |
-| test.cpp:66:10:66:35 | ... + ... | The format string argument to printf should be constant to prevent security issues and other potential errors. |
-| test.cpp:69:12:69:20 | ... + ... | The format string argument to printf should be constant to prevent security issues and other potential errors. |
-| test.cpp:75:12:75:16 | hello | The format string argument to printf should be constant to prevent security issues and other potential errors. |
-| test.cpp:81:12:81:16 | hello | The format string argument to printf should be constant to prevent security issues and other potential errors. |
-| test.cpp:87:12:87:16 | hello | The format string argument to printf should be constant to prevent security issues and other potential errors. |
-| test.cpp:92:12:92:18 | ++ ... | The format string argument to printf should be constant to prevent security issues and other potential errors. |
-| test.cpp:109:12:109:24 | new[] | The format string argument to printf should be constant to prevent security issues and other potential errors. |
-| test.cpp:129:20:129:26 | access to array | The format string argument to sprintf should be constant to prevent security issues and other potential errors. |
+| test.cpp:51:10:51:21 | call to make_message | The format string argument to printf should be constant to prevent security issues and other potential errors. |
+| test.cpp:57:12:57:16 | hello | The format string argument to printf should be constant to prevent security issues and other potential errors. |
+| test.cpp:60:12:60:21 | call to const_wash | The format string argument to printf should be constant to prevent security issues and other potential errors. |
+| test.cpp:61:12:61:26 | ... + ... | The format string argument to printf should be constant to prevent security issues and other potential errors. |
+| test.cpp:62:12:62:17 | + ... | The format string argument to printf should be constant to prevent security issues and other potential errors. |
+| test.cpp:63:12:63:18 | * ... | The format string argument to printf should be constant to prevent security issues and other potential errors. |
+| test.cpp:64:12:64:18 | & ... | The format string argument to printf should be constant to prevent security issues and other potential errors. |
+| test.cpp:65:12:65:39 | ... + ... | The format string argument to printf should be constant to prevent security issues and other potential errors. |
+| test.cpp:67:10:67:35 | ... + ... | The format string argument to printf should be constant to prevent security issues and other potential errors. |
+| test.cpp:70:12:70:20 | ... + ... | The format string argument to printf should be constant to prevent security issues and other potential errors. |
+| test.cpp:76:12:76:16 | hello | The format string argument to printf should be constant to prevent security issues and other potential errors. |
+| test.cpp:82:12:82:16 | hello | The format string argument to printf should be constant to prevent security issues and other potential errors. |
+| test.cpp:88:12:88:16 | hello | The format string argument to printf should be constant to prevent security issues and other potential errors. |
+| test.cpp:93:12:93:18 | ++ ... | The format string argument to printf should be constant to prevent security issues and other potential errors. |
+| test.cpp:110:12:110:24 | new[] | The format string argument to printf should be constant to prevent security issues and other potential errors. |
+| test.cpp:130:20:130:26 | access to array | The format string argument to sprintf should be constant to prevent security issues and other potential errors. |
+| test.cpp:157:12:157:15 | data | The format string argument to printf should be constant to prevent security issues and other potential errors. |

cpp/ql/test/query-tests/Likely Bugs/Format/NonConstantFormat/test.cpp

@@ -1,6 +1,7 @@
 extern "C" int printf(const char *fmt, ...);
 extern "C" int sprintf(char *buf, const char *fmt, ...);
-extern "C" char *gettext (const char *);
+extern "C" char *gettext(const char *);
+extern "C" char *strcpy(char *dst, const char *src);
 
 #define MYSPRINTF sprintf
 
@@ -150,3 +151,8 @@ void print_ith_message() {
   set_value_of(&i);
   printf(messages[i], 1U); // GOOD
 }
+
+void fmt_via_strcpy(char *data) {
+    strcpy(data, "some string");
+    printf(data); // BAD
+}

csharp/ql/campaigns/Solorigate/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-all
-version: 1.3.2
+version: 1.3.3-dev
 groups:
     - csharp
     - solorigate

csharp/ql/campaigns/Solorigate/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-queries
-version: 1.3.2
+version: 1.3.3-dev
 groups:
     - csharp
     - solorigate

csharp/ql/integration-tests/qlpack.yml

@@ -1,2 +1,2 @@
-libraryPathDependencies:
-  - codeql-csharp
+dependencies:
+  codeql/csharp-all: '*'

csharp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-all
-version: 0.4.2
+version: 0.4.3-dev
 groups: csharp
 dbscheme: semmlecode.csharp.dbscheme
 extractor: csharp

csharp/ql/lib/semmle/code/csharp/dataflow/internal/SsaImpl.qll

@@ -1067,6 +1067,59 @@ private predicate variableReadPseudo(ControlFlow::BasicBlock bb, int i, Ssa::Sou
   capturedReadIn(bb, i, v, _, _, _)
 }
 
+pragma[noinline]
+private predicate adjacentDefRead(
+  Definition def, SsaInput::BasicBlock bb1, int i1, SsaInput::BasicBlock bb2, int i2,
+  SsaInput::SourceVariable v
+) {
+  adjacentDefRead(def, bb1, i1, bb2, i2) and
+  v = def.getSourceVariable()
+}
+
+private predicate adjacentDefReachesRead(
+  Definition def, SsaInput::BasicBlock bb1, int i1, SsaInput::BasicBlock bb2, int i2
+) {
+  exists(SsaInput::SourceVariable v | adjacentDefRead(def, bb1, i1, bb2, i2, v) |
+    def.definesAt(v, bb1, i1)
+    or
+    SsaInput::variableRead(bb1, i1, v, true)
+  )
+  or
+  exists(SsaInput::BasicBlock bb3, int i3 |
+    adjacentDefReachesRead(def, bb1, i1, bb3, i3) and
+    SsaInput::variableRead(bb3, i3, _, false) and
+    adjacentDefRead(def, bb3, i3, bb2, i2)
+  )
+}
+
+/** Same as `adjacentDefRead`, but skips uncertain reads. */
+pragma[nomagic]
+private predicate adjacentDefSkipUncertainReads(
+  Definition def, SsaInput::BasicBlock bb1, int i1, SsaInput::BasicBlock bb2, int i2
+) {
+  adjacentDefReachesRead(def, bb1, i1, bb2, i2) and
+  SsaInput::variableRead(bb2, i2, _, true)
+}
+
+private predicate adjacentDefReachesUncertainRead(
+  Definition def, SsaInput::BasicBlock bb1, int i1, SsaInput::BasicBlock bb2, int i2
+) {
+  adjacentDefReachesRead(def, bb1, i1, bb2, i2) and
+  SsaInput::variableRead(bb2, i2, _, false)
+}
+
+/** Same as `lastRefRedef`, but skips uncertain reads. */
+pragma[nomagic]
+private predicate lastRefSkipUncertainReads(Definition def, SsaInput::BasicBlock bb, int i) {
+  lastRef(def, bb, i) and
+  not SsaInput::variableRead(bb, i, def.getSourceVariable(), false)
+  or
+  exists(SsaInput::BasicBlock bb0, int i0 |
+    lastRef(def, bb0, i0) and
+    adjacentDefReachesUncertainRead(def, bb, i, bb0, i0)
+  )
+}
+
 cached
 private module Cached {
   cached
@@ -1237,7 +1290,7 @@ private module Cached {
   predicate firstReadSameVar(Definition def, ControlFlow::Node cfn) {
     exists(ControlFlow::BasicBlock bb1, int i1, ControlFlow::BasicBlock bb2, int i2 |
       def.definesAt(_, bb1, i1) and
-      adjacentDefNoUncertainReads(def, bb1, i1, bb2, i2) and
+      adjacentDefSkipUncertainReads(def, bb1, i1, bb2, i2) and
       cfn = bb2.getNode(i2)
     )
   }
@@ -1252,20 +1305,27 @@ private module Cached {
     exists(ControlFlow::BasicBlock bb1, int i1, ControlFlow::BasicBlock bb2, int i2 |
       cfn1 = bb1.getNode(i1) and
       variableReadActual(bb1, i1, _) and
-      adjacentDefNoUncertainReads(def, bb1, i1, bb2, i2) and
+      adjacentDefSkipUncertainReads(def, bb1, i1, bb2, i2) and
       cfn2 = bb2.getNode(i2)
     )
   }
 
+  /** Same as `lastRefRedef`, but skips uncertain reads. */
   cached
   predicate lastRefBeforeRedef(Definition def, ControlFlow::BasicBlock bb, int i, Definition next) {
-    lastRefRedefNoUncertainReads(def, bb, i, next)
+    lastRefRedef(def, bb, i, next) and
+    not SsaInput::variableRead(bb, i, def.getSourceVariable(), false)
+    or
+    exists(SsaInput::BasicBlock bb0, int i0 |
+      lastRefRedef(def, bb0, i0, next) and
+      adjacentDefReachesUncertainRead(def, bb, i, bb0, i0)
+    )
   }
 
   cached
   predicate lastReadSameVar(Definition def, ControlFlow::Node cfn) {
     exists(ControlFlow::BasicBlock bb, int i |
-      lastRefNoUncertainReads(def, bb, i) and
+      lastRefSkipUncertainReads(def, bb, i) and
       variableReadActual(bb, i, _) and
       cfn = bb.getNode(i)
     )

csharp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-queries
-version: 0.4.2
+version: 0.4.3-dev
 groups: 
   - csharp
   - queries

csharp/ql/test/TestUtilities/InlineExpectationsTest.qll

@@ -137,6 +137,7 @@ abstract class InlineExpectationsTest extends string {
   final predicate hasFailureMessage(FailureLocatable element, string message) {
     exists(ActualResult actualResult |
       actualResult.getTest() = this and
+      actualResult.getTag() = this.getARelevantTag() and
       element = actualResult and
       (
         exists(FalseNegativeExpectation falseNegative |
@@ -150,9 +151,18 @@ abstract class InlineExpectationsTest extends string {
       )
     )
     or
+    exists(ActualResult actualResult |
+      actualResult.getTest() = this and
+      not actualResult.getTag() = this.getARelevantTag() and
+      element = actualResult and
+      message =
+        "Tag mismatch: Actual result with tag '" + actualResult.getTag() +
+          "' that is not part of getARelevantTag()"
+    )
+    or
     exists(ValidExpectation expectation |
       not exists(ActualResult actualResult | expectation.matchesActualResult(actualResult)) and
-      expectation.getTag() = getARelevantTag() and
+      expectation.getTag() = this.getARelevantTag() and
       element = expectation and
       (
         expectation instanceof GoodExpectation and

docs/change-notes.md

@@ -45,7 +45,7 @@ The valid YAML properties in the metadata are:
 After the `---` line following the metadata, the rest of the markdown file is the user-visible content of the change note. This should usually be a single markdown bullet list entry (starting with `*`), although it is acceptable to have multiple bullet entries in the same change note if there are multiple changes that are closely related and have the same category metadata.
 
 ## Change categories
-Each change note must specifiy a `category` property in its metadata. This category servers two purposes: It determines how the change affects the version number of the next release of the pack, and it is used to group related changes in the final changelog. There is one set of available categories for query packs, and another set of available categories for library packs.
+Each change note must specify a `category` property in its metadata. This category servers two purposes: It determines how the change affects the version number of the next release of the pack, and it is used to group related changes in the final changelog. There is one set of available categories for query packs, and another set of available categories for library packs.
 
 ### Query pack change categories
 | Category       | SemVer effect      | Description |

docs/codeql/codeql-for-visual-studio-code/exploring-data-flow-with-path-queries.rst

@@ -28,7 +28,7 @@ Running path queries in VS Code
 #. Once the query has finished running, you can see the results in the Results view as usual (under ``alerts`` in the dropdown menu). Each query result describes the flow of information between a source and a sink.
 #. Expand the result to see the individual steps that the data follows. 
 #. Click each step to jump to it in the source code and investigate the problem further.
-#. To navigate the path from your keyboard, you can bind shortcuts to the **CodeQL: Show Previous Step on Path** and **CodeQL: Show Next Step on Path** commands.
+#. To navigate the results from your keyboard, you can bind shortcuts to the **CodeQL: Navigate Up/Down/Left/Right in Result Viewer** commands.
 
 Further reading
 -----------------

docs/codeql/ql-language-reference/modules.rst

@@ -273,4 +273,71 @@ The ``<module_expression>`` itself can be a module name, a selection, or a quali
 reference. For more information, see ":ref:`name-resolution`."
 
 For information about how import statements are looked up, see "`Module resolution <https://codeql.github.com/docs/ql-language-reference/ql-language-specification/#module-resolution>`__"
-in the QL language specification. 
+in the QL language specification. 
+
+Built-in modules
+****************
+
+QL defines a ``QlBuiltins`` module that is always in scope.
+Currently, it defines a single parameterized sub-module
+``EquivalenceRelation``, that provides an efficient abstraction for working with
+(partial) equivalence relations in QL.
+
+Equivalence relations
+=====================
+
+The built-in ``EquivalenceRelation`` module is parameterized by a type ``T`` and a
+binary base relation ``base`` on ``T``. The symmetric and transitive closure of ``base``
+induces a partial equivalence relation on ``T``. If every value of ``T`` appears in
+``base``, then the induced relation is an equivalence relation on ``T``.
+
+The ``EquivalenceRelation`` module exports a ``getEquivalenceClass`` predicate that
+gets the equivalence class, if any, associated with a given ``T`` element by the
+(partial) equivalence relation induced by ``base``.
+
+The following example illustrates an application of the ``EquivalenceRelation``
+module to generate a custom equivalence relation:
+
+.. code-block:: ql
+
+  class Node extends int {
+    Node() { this in [1 .. 6] }
+  }
+
+  predicate base(Node x, Node y) {
+    x = 1 and y = 2
+    or
+    x = 3 and y = 4
+  }
+
+  module Equiv = QlBuiltins::EquivalenceRelation<Node, base/2>;
+
+  from int x, int y
+  where Equiv::getEquivalenceClass(x) = Equiv::getEquivalenceClass(y)
+  select x, y
+
+Since ``base`` does not relate ``5`` or ``6`` to any nodes, the induced
+relation is a partial equivalence relation on ``Node`` and does not relate ``5``
+or ``6`` to any nodes either.
+
+The above select clause returns the following partial equivalence relation:
+
++---+---+
+| x | y |
++===+===+
+| 1 | 1 |
++---+---+
+| 1 | 2 |
++---+---+
+| 2 | 1 |
++---+---+
+| 2 | 2 |
++---+---+
+| 3 | 3 |
++---+---+
+| 3 | 4 |
++---+---+
+| 4 | 3 |
++---+---+
+| 4 | 4 |
++---+---+

docs/codeql/ql-language-reference/ql-language-specification.rst

@@ -639,7 +639,7 @@ Various kinds of syntax can have *annotations* applied to them. Annotations are
                     |   "override"
                     |   "query"
 
-   argsAnnotation ::= "pragma" "[" ("inline" | "noinline" | "nomagic" | "noopt") "]"
+   argsAnnotation ::= "pragma" "[" ("inline" | "noinline" | "nomagic" | "noopt" | "assume_small_delta") "]"
                   |   "language" "[" "monotonicAggregates" "]"
                   |   "bindingset" "[" (variable ( "," variable)*)? "]"
 
@@ -687,17 +687,19 @@ Parameterized annotations take some additional arguments.
 
 The parameterized annotation ``pragma`` supplies compiler pragmas, and may be applied in various contexts depending on the pragma in question.
 
-+--------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
-| Pragma       | Classes | Characters | Member predicates | Non-member predicates | Imports | Fields | Modules | Aliases |
-+==============+=========+============+===================+=======================+=========+========+=========+=========+
-| ``inline``   |         | yes        | yes               | yes                   |         |        |         |         |
-+--------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
-| ``noinline`` |         | yes        | yes               | yes                   |         |        |         |         |
-+--------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
-| ``nomagic``  |         | yes        | yes               | yes                   |         |        |         |         |
-+--------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
-| ``noopt``    |         | yes        | yes               | yes                   |         |        |         |         |
-+--------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
++---------------------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
+| Pragma                    | Classes | Characters | Member predicates | Non-member predicates | Imports | Fields | Modules | Aliases |
++===========================+=========+============+===================+=======================+=========+========+=========+=========+
+| ``inline``                |         | yes        | yes               | yes                   |         |        |         |         |
++---------------------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
+| ``noinline``              |         | yes        | yes               | yes                   |         |        |         |         |
++---------------------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
+| ``nomagic``               |         | yes        | yes               | yes                   |         |        |         |         |
++---------------------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
+| ``noopt``                 |         | yes        | yes               | yes                   |         |        |         |         |
++---------------------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
+| ``assume_small_delta``    |         | yes        | yes               | yes                   |         |        |         |         |
++---------------------------+---------+------------+-------------------+-----------------------+---------+--------+---------+---------+
 
 The parameterized annotation ``language`` supplies language pragmas which change the behavior of the language. Language pragmas apply at the scope level, and are inherited by nested scopes.
 
@@ -2048,7 +2050,7 @@ The complete grammar for QL is as follows:
                     |   "override"
                     |   "query"
 
-   argsAnnotation ::= "pragma" "[" ("noinline" | "nomagic" | "noopt") "]"
+   argsAnnotation ::= "pragma" "[" ("inline" | "noinline" | "nomagic" | "noopt" | "assume_small_delta") "]"
                   |   "language" "[" "monotonicAggregates" "]"
                   |   "bindingset" "[" (variable ( "," variable)*)? "]"
 

docs/codeql/ql-language-reference/signatures.rst

@@ -40,10 +40,11 @@ Type signatures
 ===============
 
 Type signatures declare module parameters that will be substituted with types when the module is instantiated.
-Type signatures are used to specify supertypes and are the simplest category of signatures.
+Type signatures may specify supertypes and required member predicates (in addition to those member predicates that are
+implied by the supertypes).
 
 The substitution of type signatures relies on structural typing. That is, types do not have to be explicitly defined as
-implementing a type signature - they just need to have the specified (transitive) supertypes.
+implementing a type signature - they just need to have the specified (transitive) supertypes and member predicates.
 
 In detail, a type signature definition consists of:
 
@@ -52,7 +53,8 @@ In detail, a type signature definition consists of:
 #. The name of the type signature. This is an `identifier <https://codeql.github.com/docs/ql-language-reference/ql-language-specification/#identifiers>`_
    starting with a uppercase letter.
 #. Optionally, the keyword ``extends`` followed by a list of types, separated by commas.
-#. A semicolon ``;``.
+#. Either a semicolon ``;`` or a list of predicate signatures enclosed in braces.
+   The ``signature`` keyword is omitted for these contained signatures.
 
 For example:
 
@@ -60,6 +62,10 @@ For example:
 
     signature class ExtendsInt extends int;
 
+    signature class CanBePrinted {
+      string toString();
+    }
+
 Module signatures
 =================
 

docs/ql-libraries/dataflow/dataflow.md

@@ -294,8 +294,8 @@ through an additional step targeting a `PostUpdateNode`).
 
 It is recommended to introduce `PostUpdateNode`s for all `ArgumentNode`s (this
 can be skipped for immutable arguments), and all field qualifiers for both
-reads and stores. Note also that in the case of compund arguments, such as
-`b ? x : y`, it is recommented to have post-update nodes for `x` and `y` (and
+reads and stores. Note also that in the case of compound arguments, such as
+`b ? x : y`, it is recommended to have post-update nodes for `x` and `y` (and
 not the compound argument itself), and let `[post update] x` have both `x`
 and `b ? x : y` as pre-update nodes (and similarly for `[post update] y`).
 

go/old-change-notes/2020-11-12-zipslip-sanitizers.md

@@ -1,2 +1,2 @@
 lgtm,codescanning
-* Improved recongition of sanitizer functions for the `go/zipslip` query. This may reduce false-positives (but also perhaps false-negatives) when application code attempts to check a zip header entry does not contain an illegal path traversal attempt.
+* Improved recognition of sanitizer functions for the `go/zipslip` query. This may reduce false-positives (but also perhaps false-negatives) when application code attempts to check a zip header entry does not contain an illegal path traversal attempt.

go/old-change-notes/2021-01-12-model-couchbase.md

@@ -1,2 +1,2 @@
 lgtm,codescanning
-* Added support for [the offical Couchbase Go SDK library](https://github.com/couchbase/gocb), v1 and v2. The `go/sql-injection` query (which also handles non-SQL databases such as Couchbase) will now identify Couchbase queries built from untrusted external input.
+* Added support for [the official Couchbase Go SDK library](https://github.com/couchbase/gocb), v1 and v2. The `go/sql-injection` query (which also handles non-SQL databases such as Couchbase) will now identify Couchbase queries built from untrusted external input.

go/ql/config/legacy-support/qlpack.yml

@@ -1,3 +1,4 @@
 name: legacy-libraries-go
 version: 0.0.0
+# Note libraryPathDependencies is obsolete and should not be used in new qlpacks.
 libraryPathDependencies: codeql-go

go/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/go-all
-version: 0.3.2
+version: 0.3.3-dev
 groups: go
 dbscheme: go.dbscheme
 extractor: go

go/ql/lib/semmle/go/frameworks/Protobuf.qll

@@ -123,21 +123,21 @@ module Protobuf {
   }
 
   /** A `Get` method of a protobuf `Message` type. */
-  private class GetMethod extends DataFlow::FunctionModel, Method {
+  class GetMethod extends TaintTracking::FunctionModel, Method {
     GetMethod() {
       exists(string name | name.matches("Get%") | this = any(MessageType msg).getMethod(name))
     }
 
-    override predicate hasDataFlow(FunctionInput inp, FunctionOutput outp) {
+    override predicate hasTaintFlow(FunctionInput inp, FunctionOutput outp) {
       inp.isReceiver() and outp.isResult()
     }
   }
 
   /** A `ProtoReflect` method of a protobuf `Message` type. */
-  private class ProtoReflectMethod extends DataFlow::FunctionModel, Method {
+  private class ProtoReflectMethod extends TaintTracking::FunctionModel, Method {
     ProtoReflectMethod() { this = any(MessageType msg).getMethod("ProtoReflect") }
 
-    override predicate hasDataFlow(FunctionInput inp, FunctionOutput outp) {
+    override predicate hasTaintFlow(FunctionInput inp, FunctionOutput outp) {
       inp.isReceiver() and outp.isResult()
     }
   }

go/ql/lib/semmle/go/security/CleartextLogging.qll

@@ -48,8 +48,12 @@ module CleartextLogging {
         write.writesField(trg.(DataFlow::PostUpdateNode).getPreUpdateNode(), _, src)
       )
       or
-      // taint steps that do not include flow through fields
-      TaintTracking::localTaintStep(src, trg) and not TaintTracking::fieldReadStep(src, trg)
+      // taint steps that do not include flow through fields. Field reads would produce FPs due to
+      // the additional taint step above that taints whole structs from individual field writes.
+      TaintTracking::localTaintStep(src, trg) and
+      not TaintTracking::fieldReadStep(src, trg) and
+      // Also exclude protobuf field fetches, since they amount to single field reads.
+      not any(Protobuf::GetMethod gm).taintStep(src, trg)
     }
   }
 }

go/ql/src/Diagnostics/SuccessfullyExtractedFiles.ql

@@ -9,5 +9,7 @@
 import go
 
 from File f
-where not exists(Error e | e.getFile() = f)
-select f.getRelativePath()
+where
+  not exists(Error e | e.getFile() = f) and
+  exists(f.getRelativePath())
+select f, ""

go/ql/src/change-notes/2022-10-28-protobuf-cleartext-logging.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Query `go/clear-text-logging` now excludes `GetX` methods of protobuf `Message` structs, except where taint is specifically known to belong to the right field. This is to avoid FPs where taint is written to one field and then spuriously read from another.

go/ql/src/qlpack.lock.yml

@@ -1,6 +1,4 @@
 ---
-dependencies:
-  codeql/suite-helpers:
-    version: 0.0.2
+dependencies: {}
 compiled: false
 lockVersion: 1.0.0

go/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/go-queries
-version: 0.3.2
+version: 0.3.3-dev
 groups:
   - go
   - queries

go/ql/test/TestUtilities/InlineExpectationsTest.qll

@@ -137,6 +137,7 @@ abstract class InlineExpectationsTest extends string {
   final predicate hasFailureMessage(FailureLocatable element, string message) {
     exists(ActualResult actualResult |
       actualResult.getTest() = this and
+      actualResult.getTag() = this.getARelevantTag() and
       element = actualResult and
       (
         exists(FalseNegativeExpectation falseNegative |
@@ -150,9 +151,18 @@ abstract class InlineExpectationsTest extends string {
       )
     )
     or
+    exists(ActualResult actualResult |
+      actualResult.getTest() = this and
+      not actualResult.getTag() = this.getARelevantTag() and
+      element = actualResult and
+      message =
+        "Tag mismatch: Actual result with tag '" + actualResult.getTag() +
+          "' that is not part of getARelevantTag()"
+    )
+    or
     exists(ValidExpectation expectation |
       not exists(ActualResult actualResult | expectation.matchesActualResult(actualResult)) and
-      expectation.getTag() = getARelevantTag() and
+      expectation.getTag() = this.getARelevantTag() and
       element = expectation and
       (
         expectation instanceof GoodExpectation and

go/ql/test/qlpack.lock.yml

@@ -1,6 +1,4 @@
 ---
-dependencies:
-  codeql/suite-helpers:
-    version: 0.0.2
+dependencies: {}
 compiled: false
 lockVersion: 1.0.0

go/ql/test/query-tests/Diagnostics/SuccessfullyExtractedFiles.expected

@@ -1 +1 @@
-| query-tests/Diagnostics/util.go |
+| util.go:0:0:0:0 | util.go |  |

go/ql/test/query-tests/Security/CWE-312/CleartextLogging.expected

@@ -24,6 +24,16 @@ edges
 | passwords.go:122:13:122:25 | call to getPassword : string | passwords.go:125:14:125:19 | config |
 | passwords.go:126:14:126:19 | config [x] : string | passwords.go:126:14:126:21 | selection of x |
 | passwords.go:127:14:127:19 | config [y] : string | passwords.go:127:14:127:21 | selection of y |
+| protobuf.go:11:2:11:6 | definition of query [pointer, Description] : string | protobuf.go:12:2:12:6 | query [pointer, Description] : string |
+| protobuf.go:11:2:11:6 | definition of query [pointer, Description] : string | protobuf.go:14:14:14:18 | query [pointer, Description] : string |
+| protobuf.go:12:2:12:6 | implicit dereference [Description] : string | protobuf.go:11:2:11:6 | definition of query [pointer, Description] : string |
+| protobuf.go:12:2:12:6 | query [pointer, Description] : string | protobuf.go:12:2:12:6 | implicit dereference [Description] : string |
+| protobuf.go:12:22:12:29 | password : string | protobuf.go:12:2:12:6 | implicit dereference [Description] : string |
+| protobuf.go:14:14:14:18 | query [pointer, Description] : string | protobuf.go:14:14:14:35 | call to GetDescription |
+| protobuf.go:14:14:14:18 | query [pointer, Description] : string | protos/query/query.pb.go:117:7:117:7 | definition of x [pointer, Description] : string |
+| protos/query/query.pb.go:117:7:117:7 | definition of x [pointer, Description] : string | protos/query/query.pb.go:119:10:119:10 | x [pointer, Description] : string |
+| protos/query/query.pb.go:119:10:119:10 | implicit dereference [Description] : string | protos/query/query.pb.go:119:10:119:22 | selection of Description : string |
+| protos/query/query.pb.go:119:10:119:10 | x [pointer, Description] : string | protos/query/query.pb.go:119:10:119:10 | implicit dereference [Description] : string |
 | util.go:16:9:16:18 | selection of password : string | passwords.go:28:14:28:28 | call to getPassword |
 nodes
 | klog.go:20:30:20:37 | selection of Header : Header | semmle.label | selection of Header : Header |
@@ -77,8 +87,19 @@ nodes
 | passwords.go:126:14:126:21 | selection of x | semmle.label | selection of x |
 | passwords.go:127:14:127:19 | config [y] : string | semmle.label | config [y] : string |
 | passwords.go:127:14:127:21 | selection of y | semmle.label | selection of y |
+| protobuf.go:11:2:11:6 | definition of query [pointer, Description] : string | semmle.label | definition of query [pointer, Description] : string |
+| protobuf.go:12:2:12:6 | implicit dereference [Description] : string | semmle.label | implicit dereference [Description] : string |
+| protobuf.go:12:2:12:6 | query [pointer, Description] : string | semmle.label | query [pointer, Description] : string |
+| protobuf.go:12:22:12:29 | password : string | semmle.label | password : string |
+| protobuf.go:14:14:14:18 | query [pointer, Description] : string | semmle.label | query [pointer, Description] : string |
+| protobuf.go:14:14:14:35 | call to GetDescription | semmle.label | call to GetDescription |
+| protos/query/query.pb.go:117:7:117:7 | definition of x [pointer, Description] : string | semmle.label | definition of x [pointer, Description] : string |
+| protos/query/query.pb.go:119:10:119:10 | implicit dereference [Description] : string | semmle.label | implicit dereference [Description] : string |
+| protos/query/query.pb.go:119:10:119:10 | x [pointer, Description] : string | semmle.label | x [pointer, Description] : string |
+| protos/query/query.pb.go:119:10:119:22 | selection of Description : string | semmle.label | selection of Description : string |
 | util.go:16:9:16:18 | selection of password : string | semmle.label | selection of password : string |
 subpaths
+| protobuf.go:14:14:14:18 | query [pointer, Description] : string | protos/query/query.pb.go:117:7:117:7 | definition of x [pointer, Description] : string | protos/query/query.pb.go:119:10:119:22 | selection of Description : string | protobuf.go:14:14:14:35 | call to GetDescription : string |
 #select
 | klog.go:22:15:22:20 | header | klog.go:20:30:20:37 | selection of Header : Header | klog.go:22:15:22:20 | header | $@ flows to a logging call. | klog.go:20:30:20:37 | selection of Header | Sensitive data returned by HTTP request headers |
 | klog.go:28:13:28:41 | call to Get | klog.go:28:13:28:20 | selection of Header : Header | klog.go:28:13:28:41 | call to Get | $@ flows to a logging call. | klog.go:28:13:28:20 | selection of Header | Sensitive data returned by HTTP request headers |
@@ -111,3 +132,4 @@ subpaths
 | passwords.go:125:14:125:19 | config | passwords.go:122:13:122:25 | call to getPassword : string | passwords.go:125:14:125:19 | config | $@ flows to a logging call. | passwords.go:122:13:122:25 | call to getPassword | Sensitive data returned by a call to getPassword |
 | passwords.go:126:14:126:21 | selection of x | passwords.go:121:13:121:20 | password : string | passwords.go:126:14:126:21 | selection of x | $@ flows to a logging call. | passwords.go:121:13:121:20 | password | Sensitive data returned by an access to password |
 | passwords.go:127:14:127:21 | selection of y | passwords.go:122:13:122:25 | call to getPassword : string | passwords.go:127:14:127:21 | selection of y | $@ flows to a logging call. | passwords.go:122:13:122:25 | call to getPassword | Sensitive data returned by a call to getPassword |
+| protobuf.go:14:14:14:35 | call to GetDescription | protobuf.go:12:22:12:29 | password : string | protobuf.go:14:14:14:35 | call to GetDescription | $@ flows to a logging call. | protobuf.go:12:22:12:29 | password | Sensitive data returned by an access to password |

go/ql/test/query-tests/Security/CWE-312/go.mod

@@ -6,4 +6,6 @@ require (
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/sirupsen/logrus v1.5.0
 	k8s.io/klog v1.0.0
+        github.com/golang/protobuf v1.4.2
+        google.golang.org/protobuf v1.23.0
 )

go/ql/test/query-tests/Security/CWE-312/protobuf.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"log"
+	"main/protos/query"
+)
+
+func testProtobuf() {
+	password := "P@ssw0rd"
+
+	query := &query.Query{}
+	query.Description = password
+
+	log.Println(query.GetDescription()) // NOT OK
+	log.Println(query.GetId())          // OK
+}

go/ql/test/query-tests/Security/CWE-312/protos/query.proto

@@ -0,0 +1,25 @@
+syntax = "proto3";
+option go_package = "protos/query";
+
+message Query {
+  string description = 1;
+  string id = 2;
+
+  enum Severity {
+    ERROR = 0;
+    WARNING = 1;
+  }
+
+  message Alert {
+    string msg = 1;
+    int64 loc = 2;
+  }
+
+  repeated Alert alerts = 4;
+
+  map<int32, string> keyValuePairs = 5;
+}
+
+message QuerySuite {
+  repeated Query queries = 1;
+}

go/ql/test/query-tests/Security/CWE-312/protos/query/query.pb.go

@@ -0,0 +1,371 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.25.0-devel
+// 	protoc        v3.12.4
+// source: query.proto
+
+package query
+
+import (
+	proto "github.com/golang/protobuf/proto"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// This is a compile-time assertion that a sufficiently up-to-date version
+// of the legacy proto package is being used.
+const _ = proto.ProtoPackageIsVersion4
+
+type Query_Severity int32
+
+const (
+	Query_ERROR   Query_Severity = 0
+	Query_WARNING Query_Severity = 1
+)
+
+// Enum value maps for Query_Severity.
+var (
+	Query_Severity_name = map[int32]string{
+		0: "ERROR",
+		1: "WARNING",
+	}
+	Query_Severity_value = map[string]int32{
+		"ERROR":   0,
+		"WARNING": 1,
+	}
+)
+
+func (x Query_Severity) Enum() *Query_Severity {
+	p := new(Query_Severity)
+	*p = x
+	return p
+}
+
+func (x Query_Severity) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (Query_Severity) Descriptor() protoreflect.EnumDescriptor {
+	return file_query_proto_enumTypes[0].Descriptor()
+}
+
+func (Query_Severity) Type() protoreflect.EnumType {
+	return &file_query_proto_enumTypes[0]
+}
+
+func (x Query_Severity) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use Query_Severity.Descriptor instead.
+func (Query_Severity) EnumDescriptor() ([]byte, []int) {
+	return file_query_proto_rawDescGZIP(), []int{0, 0}
+}
+
+type Query struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Description   string           `protobuf:"bytes,1,opt,name=description,proto3" json:"description,omitempty"`
+	Id            string           `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"`
+	Alerts        []*Query_Alert   `protobuf:"bytes,4,rep,name=alerts,proto3" json:"alerts,omitempty"`
+	KeyValuePairs map[int32]string `protobuf:"bytes,5,rep,name=keyValuePairs,proto3" json:"keyValuePairs,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+}
+
+func (x *Query) Reset() {
+	*x = Query{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_query_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Query) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Query) ProtoMessage() {}
+
+func (x *Query) ProtoReflect() protoreflect.Message {
+	mi := &file_query_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Query.ProtoReflect.Descriptor instead.
+func (*Query) Descriptor() ([]byte, []int) {
+	return file_query_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *Query) GetDescription() string {
+	if x != nil {
+		return x.Description
+	}
+	return ""
+}
+
+func (x *Query) GetId() string {
+	if x != nil {
+		return x.Id
+	}
+	return ""
+}
+
+func (x *Query) GetAlerts() []*Query_Alert {
+	if x != nil {
+		return x.Alerts
+	}
+	return nil
+}
+
+func (x *Query) GetKeyValuePairs() map[int32]string {
+	if x != nil {
+		return x.KeyValuePairs
+	}
+	return nil
+}
+
+type QuerySuite struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Queries []*Query `protobuf:"bytes,1,rep,name=queries,proto3" json:"queries,omitempty"`
+}
+
+func (x *QuerySuite) Reset() {
+	*x = QuerySuite{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_query_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *QuerySuite) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*QuerySuite) ProtoMessage() {}
+
+func (x *QuerySuite) ProtoReflect() protoreflect.Message {
+	mi := &file_query_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use QuerySuite.ProtoReflect.Descriptor instead.
+func (*QuerySuite) Descriptor() ([]byte, []int) {
+	return file_query_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *QuerySuite) GetQueries() []*Query {
+	if x != nil {
+		return x.Queries
+	}
+	return nil
+}
+
+type Query_Alert struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Msg string `protobuf:"bytes,1,opt,name=msg,proto3" json:"msg,omitempty"`
+	Loc int64  `protobuf:"varint,2,opt,name=loc,proto3" json:"loc,omitempty"`
+}
+
+func (x *Query_Alert) Reset() {
+	*x = Query_Alert{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_query_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Query_Alert) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Query_Alert) ProtoMessage() {}
+
+func (x *Query_Alert) ProtoReflect() protoreflect.Message {
+	mi := &file_query_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Query_Alert.ProtoReflect.Descriptor instead.
+func (*Query_Alert) Descriptor() ([]byte, []int) {
+	return file_query_proto_rawDescGZIP(), []int{0, 0}
+}
+
+func (x *Query_Alert) GetMsg() string {
+	if x != nil {
+		return x.Msg
+	}
+	return ""
+}
+
+func (x *Query_Alert) GetLoc() int64 {
+	if x != nil {
+		return x.Loc
+	}
+	return 0
+}
+
+var File_query_proto protoreflect.FileDescriptor
+
+var file_query_proto_rawDesc = []byte{
+	0x0a, 0x0b, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb3, 0x02,
+	0x0a, 0x05, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72,
+	0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65,
+	0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x24, 0x0a, 0x06, 0x61, 0x6c, 0x65,
+	0x72, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x51, 0x75, 0x65, 0x72,
+	0x79, 0x2e, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x52, 0x06, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x73, 0x12,
+	0x3f, 0x0a, 0x0d, 0x6b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x50, 0x61, 0x69, 0x72, 0x73,
+	0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x4b,
+	0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x50, 0x61, 0x69, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72,
+	0x79, 0x52, 0x0d, 0x6b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x50, 0x61, 0x69, 0x72, 0x73,
+	0x1a, 0x2b, 0x0a, 0x05, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x73, 0x67,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6d, 0x73, 0x67, 0x12, 0x10, 0x0a, 0x03, 0x6c,
+	0x6f, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x6c, 0x6f, 0x63, 0x1a, 0x40, 0x0a,
+	0x12, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x50, 0x61, 0x69, 0x72, 0x73, 0x45, 0x6e,
+	0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05,
+	0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22,
+	0x22, 0x0a, 0x08, 0x53, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x12, 0x09, 0x0a, 0x05, 0x45,
+	0x52, 0x52, 0x4f, 0x52, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x57, 0x41, 0x52, 0x4e, 0x49, 0x4e,
+	0x47, 0x10, 0x01, 0x22, 0x2e, 0x0a, 0x0a, 0x51, 0x75, 0x65, 0x72, 0x79, 0x53, 0x75, 0x69, 0x74,
+	0x65, 0x12, 0x20, 0x0a, 0x07, 0x71, 0x75, 0x65, 0x72, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03,
+	0x28, 0x0b, 0x32, 0x06, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x07, 0x71, 0x75, 0x65, 0x72,
+	0x69, 0x65, 0x73, 0x42, 0x0e, 0x5a, 0x0c, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x71, 0x75,
+	0x65, 0x72, 0x79, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_query_proto_rawDescOnce sync.Once
+	file_query_proto_rawDescData = file_query_proto_rawDesc
+)
+
+func file_query_proto_rawDescGZIP() []byte {
+	file_query_proto_rawDescOnce.Do(func() {
+		file_query_proto_rawDescData = protoimpl.X.CompressGZIP(file_query_proto_rawDescData)
+	})
+	return file_query_proto_rawDescData
+}
+
+var file_query_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
+var file_query_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
+var file_query_proto_goTypes = []interface{}{
+	(Query_Severity)(0), // 0: Query.Severity
+	(*Query)(nil),       // 1: Query
+	(*QuerySuite)(nil),  // 2: QuerySuite
+	(*Query_Alert)(nil), // 3: Query.Alert
+	nil,                 // 4: Query.KeyValuePairsEntry
+}
+var file_query_proto_depIdxs = []int32{
+	3, // 0: Query.alerts:type_name -> Query.Alert
+	4, // 1: Query.keyValuePairs:type_name -> Query.KeyValuePairsEntry
+	1, // 2: QuerySuite.queries:type_name -> Query
+	3, // [3:3] is the sub-list for method output_type
+	3, // [3:3] is the sub-list for method input_type
+	3, // [3:3] is the sub-list for extension type_name
+	3, // [3:3] is the sub-list for extension extendee
+	0, // [0:3] is the sub-list for field type_name
+}
+
+func init() { file_query_proto_init() }
+func file_query_proto_init() {
+	if File_query_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_query_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Query); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_query_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*QuerySuite); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_query_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Query_Alert); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_query_proto_rawDesc,
+			NumEnums:      1,
+			NumMessages:   4,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_query_proto_goTypes,
+		DependencyIndexes: file_query_proto_depIdxs,
+		EnumInfos:         file_query_proto_enumTypes,
+		MessageInfos:      file_query_proto_msgTypes,
+	}.Build()
+	File_query_proto = out.File
+	file_query_proto_rawDesc = nil
+	file_query_proto_goTypes = nil
+	file_query_proto_depIdxs = nil
+}

go/ql/test/query-tests/Security/CWE-312/vendor/github.com/golang/protobuf/LICENSE

@@ -0,0 +1,28 @@
+Copyright 2010 The Go Authors.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

go/ql/test/query-tests/Security/CWE-312/vendor/github.com/golang/protobuf/proto/stub.go

@@ -0,0 +1,29 @@
+// This is a simple stub for github.com/golang/protobuf/proto, strictly for use in testing.
+
+// See the LICENSE file for information about the licensing of the original library.
+// Source: github.com/golang/protobuf/proto (exports: Message; functions: Marshal,Unmarshal,ProtoPackageIsVersion4)
+
+// Package proto is a stub of github.com/golang/protobuf/proto
+package proto
+
+import (
+	protoiface "google.golang.org/protobuf/runtime/protoiface"
+)
+
+func Marshal(_ interface{}) ([]byte, error) {
+	return nil, nil
+}
+
+type Message = protoiface.MessageV1
+
+const ProtoPackageIsVersion4 bool = false
+
+func Unmarshal(_ []byte, _ interface{}) error {
+	return nil
+}
+
+func Clone(_ Message) Message {
+	return nil
+}
+
+func Merge(_, _ Message) {}

go/ql/test/query-tests/Security/CWE-312/vendor/google.golang.org/protobuf/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2018 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

go/ql/test/query-tests/Security/CWE-312/vendor/google.golang.org/protobuf/internal/impl/stub.go

@@ -0,0 +1,132 @@
+// This is a simple stub for google.golang.org/protobuf/internal/impl, strictly for use in testing.
+
+// See the LICENSE file for information about the licensing of the original library.
+// Source: google.golang.org/protobuf/internal/impl (exports: MessageState,Pointer; functions: )
+
+// Package impl is a stub of google.golang.org/protobuf/internal/impl.
+package impl
+
+import (
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+type MessageState struct {
+	NoUnkeyedLiterals interface{}
+	DoNotCompare      interface{}
+	DoNotCopy         interface{}
+}
+
+type Pointer interface{}
+
+type MessageInfo struct {
+	Exporter interface{}
+}
+
+func (*MessageInfo) MessageOf(_ interface{}) protoreflect.Message { return nil }
+
+type EnumInfo struct{}
+
+func (_ *EnumInfo) Descriptor() protoreflect.EnumDescriptor         { return nil }
+func (_ *EnumInfo) New(_ protoreflect.EnumNumber) protoreflect.Enum { return nil }
+
+type DescBuilder struct {
+	GoPackagePath string
+	RawDescriptor []byte
+	NumEnums      int
+	NumMessages   int
+	NumExtensions int
+	NumServices   int
+}
+
+type TypeBuilder struct {
+	File              DescBuilder
+	GoTypes           []interface{}
+	DependencyIndexes []int32
+	EnumInfos         []EnumInfo
+	MessageInfos      []MessageInfo
+}
+
+type BuilderOut struct {
+	File protoreflect.FileDescriptor
+}
+
+func (tb TypeBuilder) Build() BuilderOut {
+	return BuilderOut{nil}
+}
+
+func (ms *MessageState) LoadMessageInfo() *MessageInfo    { return nil }
+func (ms *MessageState) StoreMessageInfo(mi *MessageInfo) {}
+
+func (ms *MessageState) Clear(_ protoreflect.FieldDescriptor)       {}
+func (ms *MessageState) Descriptor() protoreflect.MessageDescriptor { return nil }
+func (ms *MessageState) Get(_ protoreflect.FieldDescriptor) protoreflect.Value {
+	return protoreflect.Value{}
+}
+func (ms *MessageState) GetUnknown() protoreflect.RawFields      { return nil }
+func (ms *MessageState) Has(_ protoreflect.FieldDescriptor) bool { return false }
+func (ms *MessageState) Interface() protoreflect.ProtoMessage    { return nil }
+func (ms *MessageState) IsValid() bool                           { return false }
+func (ms *MessageState) Mutable(_ protoreflect.FieldDescriptor) protoreflect.Value {
+	return protoreflect.Value{}
+}
+func (ms *MessageState) New() protoreflect.Message { return nil }
+func (ms *MessageState) NewField(_ protoreflect.FieldDescriptor) protoreflect.Value {
+	return protoreflect.Value{}
+}
+func (ms *MessageState) ProtoMethods() *struct {
+	NoUnkeyedLiterals interface{}
+	Flags             uint64
+	Size              func(struct {
+		NoUnkeyedLiterals interface{}
+		Message           protoreflect.Message
+		Flags             byte
+	}) struct {
+		NoUnkeyedLiterals interface{}
+		Size              int
+	}
+	Marshal func(struct {
+		NoUnkeyedLiterals interface{}
+		Message           protoreflect.Message
+		Buf               []byte
+		Flags             byte
+	}) (struct {
+		NoUnkeyedLiterals interface{}
+		Buf               []byte
+	}, error)
+	Unmarshal func(struct {
+		NoUnkeyedLiterals interface{}
+		Message           protoreflect.Message
+		Buf               []byte
+		Flags             byte
+		Resolver          interface {
+			FindExtensionByName(_ protoreflect.FullName) (protoreflect.ExtensionType, error)
+			FindExtensionByNumber(_ protoreflect.FullName, _ interface{}) (protoreflect.ExtensionType, error)
+		}
+	}) (struct {
+		NoUnkeyedLiterals interface{}
+		Flags             byte
+	}, error)
+	Merge func(struct {
+		NoUnkeyedLiterals interface{}
+		Source            protoreflect.Message
+		Destination       protoreflect.Message
+	}) struct {
+		NoUnkeyedLiterals interface{}
+		Flags             byte
+	}
+	CheckInitialized func(struct {
+		NoUnkeyedLiterals interface{}
+		Message           protoreflect.Message
+	}) (struct {
+		NoUnkeyedLiterals interface{}
+	}, error)
+} {
+	return nil
+}
+func (ms *MessageState) Range(_ func(protoreflect.FieldDescriptor, protoreflect.Value) bool) {}
+func (ms *MessageState) Set(_ protoreflect.FieldDescriptor, _ protoreflect.Value)            {}
+func (ms *MessageState) SetUnknown(_ protoreflect.RawFields)                                 {}
+func (ms *MessageState) Type() protoreflect.MessageType                                      { return nil }
+func (ms *MessageState) WhichOneof(_ protoreflect.OneofDescriptor) protoreflect.FieldDescriptor {
+	return nil
+}

go/ql/test/query-tests/Security/CWE-312/vendor/google.golang.org/protobuf/proto/stub.go

@@ -0,0 +1,68 @@
+// This is a simple stub for github.com/golang/protobuf/proto, strictly for use in testing.
+
+// See the LICENSE file for information about the licensing of the original library.
+// Source: github.com/golang/protobuf/proto (exports: Message; functions: Marshal,Unmarshal,ProtoPackageIsVersion4)
+
+// Package proto is a stub of github.com/golang/protobuf/proto.
+package proto
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoiface "google.golang.org/protobuf/runtime/protoiface"
+)
+
+func Marshal(_ interface{}) ([]byte, error) {
+	return nil, nil
+}
+
+type Message = protoreflect.ProtoMessage
+
+var ProtoPackageIsVersion4 bool = false
+
+func Unmarshal(_ []byte, _ interface{}) error {
+	return nil
+}
+
+type MarshalOptions struct {
+	AllowPartial  bool
+	Deterministic bool
+	UseCachedSize bool
+}
+
+func (_ MarshalOptions) Marshal(_ Message) ([]byte, error)                 { return nil, nil }
+func (_ MarshalOptions) MarshalAppend(b []byte, m Message) ([]byte, error) { return nil, nil }
+func (_ MarshalOptions) MarshalState(in protoiface.MarshalInput) (protoiface.MarshalOutput, error) {
+	return protoiface.MarshalOutput{nil}, nil
+}
+
+type UnmarshalOptions struct {
+	// Merge merges the input into the destination message.
+	// The default behavior is to always reset the message before unmarshaling,
+	// unless Merge is specified.
+	Merge bool
+
+	// AllowPartial accepts input for messages that will result in missing
+	// required fields. If AllowPartial is false (the default), Unmarshal will
+	// return an error if there are any missing required fields.
+	AllowPartial bool
+
+	// If DiscardUnknown is set, unknown fields are ignored.
+	DiscardUnknown bool
+
+	// Resolver is used for looking up types when unmarshaling extension fields.
+	// If nil, this defaults to using protoregistry.GlobalTypes.
+	Resolver interface {
+		FindExtensionByName(field protoreflect.FullName) (protoreflect.ExtensionType, error)
+		FindExtensionByNumber(message protoreflect.FullName, field protoreflect.FieldNumber) (protoreflect.ExtensionType, error)
+	}
+}
+
+func (o UnmarshalOptions) Unmarshal(b []byte, m Message) error {
+	return nil
+}
+
+func Clone(_ Message) Message {
+	return nil
+}
+
+func Merge(_, _ Message) {}

go/ql/test/query-tests/Security/CWE-312/vendor/google.golang.org/protobuf/reflect/protoreflect/stub.go

@@ -0,0 +1,683 @@
+// Code generated by depstubber. DO NOT EDIT.
+// This is a simple stub for google.golang.org/protobuf/reflect/protoreflect, strictly for use in testing.
+
+// See the LICENSE file for information about the licensing of the original library.
+// Source: google.golang.org/protobuf/reflect/protoreflect (exports: EnumDescriptor,EnumType,EnumNumber,Message,FileDescriptor; functions: )
+
+// Package protoreflect is a stub of google.golang.org/protobuf/reflect/protoreflect, generated by depstubber.
+package protoreflect
+
+import ()
+
+type Cardinality int8
+
+func (_ Cardinality) GoString() string {
+	return ""
+}
+
+func (_ Cardinality) IsValid() bool {
+	return false
+}
+
+func (_ Cardinality) String() string {
+	return ""
+}
+
+type Descriptor interface {
+	FullName() FullName
+	Index() int
+	IsPlaceholder() bool
+	Name() Name
+	Options() ProtoMessage
+	Parent() Descriptor
+	ParentFile() FileDescriptor
+	ProtoInternal(_ interface{})
+	Syntax() Syntax
+}
+
+type Enum interface {
+	Descriptor() EnumDescriptor
+	Number() EnumNumber
+	Type() EnumType
+}
+
+type EnumDescriptor interface {
+	FullName() FullName
+	Index() int
+	IsPlaceholder() bool
+	Name() Name
+	Options() ProtoMessage
+	Parent() Descriptor
+	ParentFile() FileDescriptor
+	ProtoInternal(_ interface{})
+	ProtoType(_ EnumDescriptor)
+	ReservedNames() Names
+	ReservedRanges() EnumRanges
+	Syntax() Syntax
+	Values() EnumValueDescriptors
+}
+
+type EnumDescriptors interface {
+	ByName(_ Name) EnumDescriptor
+	Get(_ int) EnumDescriptor
+	Len() int
+	ProtoInternal(_ interface{})
+}
+
+type EnumNumber int32
+
+type EnumRanges interface {
+	Get(_ int) [2]EnumNumber
+	Has(_ EnumNumber) bool
+	Len() int
+	ProtoInternal(_ interface{})
+}
+
+type EnumType interface {
+	Descriptor() EnumDescriptor
+	New(_ EnumNumber) Enum
+}
+
+type EnumValueDescriptor interface {
+	FullName() FullName
+	Index() int
+	IsPlaceholder() bool
+	Name() Name
+	Number() EnumNumber
+	Options() ProtoMessage
+	Parent() Descriptor
+	ParentFile() FileDescriptor
+	ProtoInternal(_ interface{})
+	ProtoType(_ EnumValueDescriptor)
+	Syntax() Syntax
+}
+
+type EnumValueDescriptors interface {
+	ByName(_ Name) EnumValueDescriptor
+	ByNumber(_ EnumNumber) EnumValueDescriptor
+	Get(_ int) EnumValueDescriptor
+	Len() int
+	ProtoInternal(_ interface{})
+}
+
+type ExtensionDescriptors interface {
+	ByName(_ Name) FieldDescriptor
+	Get(_ int) FieldDescriptor
+	Len() int
+	ProtoInternal(_ interface{})
+}
+
+type ExtensionType interface {
+	InterfaceOf(_ Value) interface{}
+	IsValidInterface(_ interface{}) bool
+	IsValidValue(_ Value) bool
+	New() Value
+	TypeDescriptor() ExtensionTypeDescriptor
+	ValueOf(_ interface{}) Value
+	Zero() Value
+}
+
+type ExtensionTypeDescriptor interface {
+	Cardinality() Cardinality
+	ContainingMessage() MessageDescriptor
+	ContainingOneof() OneofDescriptor
+	Default() Value
+	DefaultEnumValue() EnumValueDescriptor
+	Descriptor() FieldDescriptor
+	Enum() EnumDescriptor
+	FullName() FullName
+	HasDefault() bool
+	HasJSONName() bool
+	HasOptionalKeyword() bool
+	HasPresence() bool
+	Index() int
+	IsExtension() bool
+	IsList() bool
+	IsMap() bool
+	IsPacked() bool
+	IsPlaceholder() bool
+	IsWeak() bool
+	JSONName() string
+	Kind() Kind
+	MapKey() FieldDescriptor
+	MapValue() FieldDescriptor
+	Message() MessageDescriptor
+	Name() Name
+	Number() interface{}
+	Options() ProtoMessage
+	Parent() Descriptor
+	ParentFile() FileDescriptor
+	ProtoInternal(_ interface{})
+	ProtoType(_ FieldDescriptor)
+	Syntax() Syntax
+	Type() ExtensionType
+}
+
+type FieldDescriptor interface {
+	Cardinality() Cardinality
+	ContainingMessage() MessageDescriptor
+	ContainingOneof() OneofDescriptor
+	Default() Value
+	DefaultEnumValue() EnumValueDescriptor
+	Enum() EnumDescriptor
+	FullName() FullName
+	HasDefault() bool
+	HasJSONName() bool
+	HasOptionalKeyword() bool
+	HasPresence() bool
+	Index() int
+	IsExtension() bool
+	IsList() bool
+	IsMap() bool
+	IsPacked() bool
+	IsPlaceholder() bool
+	IsWeak() bool
+	JSONName() string
+	Kind() Kind
+	MapKey() FieldDescriptor
+	MapValue() FieldDescriptor
+	Message() MessageDescriptor
+	Name() Name
+	Number() interface{}
+	Options() ProtoMessage
+	Parent() Descriptor
+	ParentFile() FileDescriptor
+	ProtoInternal(_ interface{})
+	ProtoType(_ FieldDescriptor)
+	Syntax() Syntax
+}
+
+type FieldDescriptors interface {
+	ByJSONName(_ string) FieldDescriptor
+	ByName(_ Name) FieldDescriptor
+	ByNumber(_ interface{}) FieldDescriptor
+	Get(_ int) FieldDescriptor
+	Len() int
+	ProtoInternal(_ interface{})
+}
+
+type FieldNumber int32
+
+type FieldNumbers interface {
+	Get(_ int) interface{}
+	Has(_ interface{}) bool
+	Len() int
+	ProtoInternal(_ interface{})
+}
+
+type FieldRanges interface {
+	Get(_ int) [2]interface{}
+	Has(_ interface{}) bool
+	Len() int
+	ProtoInternal(_ interface{})
+}
+
+type FileDescriptor interface {
+	Enums() EnumDescriptors
+	Extensions() ExtensionDescriptors
+	FullName() FullName
+	Imports() FileImports
+	Index() int
+	IsPlaceholder() bool
+	Messages() MessageDescriptors
+	Name() Name
+	Options() ProtoMessage
+	Package() FullName
+	Parent() Descriptor
+	ParentFile() FileDescriptor
+	Path() string
+	ProtoInternal(_ interface{})
+	ProtoType(_ FileDescriptor)
+	Services() ServiceDescriptors
+	SourceLocations() SourceLocations
+	Syntax() Syntax
+}
+
+type FileImport struct {
+	FileDescriptor FileDescriptor
+	IsPublic       bool
+	IsWeak         bool
+}
+
+func (_ FileImport) Enums() EnumDescriptors {
+	return nil
+}
+
+func (_ FileImport) Extensions() ExtensionDescriptors {
+	return nil
+}
+
+func (_ FileImport) FullName() FullName {
+	return ""
+}
+
+func (_ FileImport) Imports() FileImports {
+	return nil
+}
+
+func (_ FileImport) Index() int {
+	return 0
+}
+
+func (_ FileImport) IsPlaceholder() bool {
+	return false
+}
+
+func (_ FileImport) Messages() MessageDescriptors {
+	return nil
+}
+
+func (_ FileImport) Name() Name {
+	return ""
+}
+
+func (_ FileImport) Options() ProtoMessage {
+	return nil
+}
+
+func (_ FileImport) Package() FullName {
+	return ""
+}
+
+func (_ FileImport) Parent() Descriptor {
+	return nil
+}
+
+func (_ FileImport) ParentFile() FileDescriptor {
+	return nil
+}
+
+func (_ FileImport) Path() string {
+	return ""
+}
+
+func (_ FileImport) ProtoInternal(_ interface{}) {}
+
+func (_ FileImport) ProtoType(_ FileDescriptor) {}
+
+func (_ FileImport) Services() ServiceDescriptors {
+	return nil
+}
+
+func (_ FileImport) SourceLocations() SourceLocations {
+	return nil
+}
+
+func (_ FileImport) Syntax() Syntax {
+	return 0
+}
+
+type FileImports interface {
+	Get(_ int) FileImport
+	Len() int
+	ProtoInternal(_ interface{})
+}
+
+type FullName string
+
+func (_ FullName) Append(_ Name) FullName {
+	return ""
+}
+
+func (_ FullName) IsValid() bool {
+	return false
+}
+
+func (_ FullName) Name() Name {
+	return ""
+}
+
+func (_ FullName) Parent() FullName {
+	return ""
+}
+
+type Kind int8
+
+func (_ Kind) GoString() string {
+	return ""
+}
+
+func (_ Kind) IsValid() bool {
+	return false
+}
+
+func (_ Kind) String() string {
+	return ""
+}
+
+type List interface {
+	Append(_ Value)
+	AppendMutable() Value
+	Get(_ int) Value
+	IsValid() bool
+	Len() int
+	NewElement() Value
+	Set(_ int, _ Value)
+	Truncate(_ int)
+}
+
+type Map interface {
+	Clear(_ MapKey)
+	Get(_ MapKey) Value
+	Has(_ MapKey) bool
+	IsValid() bool
+	Len() int
+	Mutable(_ MapKey) Value
+	NewValue() Value
+	Range(_ func(MapKey, Value) bool)
+	Set(_ MapKey, _ Value)
+}
+
+type MapKey struct {
+	DoNotCompare interface{}
+}
+
+func (_ MapKey) Bool() bool {
+	return false
+}
+
+func (_ MapKey) Int() int64 {
+	return 0
+}
+
+func (_ MapKey) Interface() interface{} {
+	return nil
+}
+
+func (_ MapKey) IsValid() bool {
+	return false
+}
+
+func (_ MapKey) String() string {
+	return ""
+}
+
+func (_ MapKey) Uint() uint64 {
+	return 0
+}
+
+func (_ MapKey) Value() Value {
+	return Value{}
+}
+
+type Message interface {
+	Clear(_ FieldDescriptor)
+	Descriptor() MessageDescriptor
+	Get(_ FieldDescriptor) Value
+	GetUnknown() RawFields
+	Has(_ FieldDescriptor) bool
+	Interface() ProtoMessage
+	IsValid() bool
+	Mutable(_ FieldDescriptor) Value
+	New() Message
+	NewField(_ FieldDescriptor) Value
+	ProtoMethods() *struct {
+		NoUnkeyedLiterals interface{}
+		Flags             uint64
+		Size              func(struct {
+			NoUnkeyedLiterals interface{}
+			Message           Message
+			Flags             byte
+		}) struct {
+			NoUnkeyedLiterals interface{}
+			Size              int
+		}
+		Marshal func(struct {
+			NoUnkeyedLiterals interface{}
+			Message           Message
+			Buf               []byte
+			Flags             byte
+		}) (struct {
+			NoUnkeyedLiterals interface{}
+			Buf               []byte
+		}, error)
+		Unmarshal func(struct {
+			NoUnkeyedLiterals interface{}
+			Message           Message
+			Buf               []byte
+			Flags             byte
+			Resolver          interface {
+				FindExtensionByName(_ FullName) (ExtensionType, error)
+				FindExtensionByNumber(_ FullName, _ interface{}) (ExtensionType, error)
+			}
+		}) (struct {
+			NoUnkeyedLiterals interface{}
+			Flags             byte
+		}, error)
+		Merge func(struct {
+			NoUnkeyedLiterals interface{}
+			Source            Message
+			Destination       Message
+		}) struct {
+			NoUnkeyedLiterals interface{}
+			Flags             byte
+		}
+		CheckInitialized func(struct {
+			NoUnkeyedLiterals interface{}
+			Message           Message
+		}) (struct {
+			NoUnkeyedLiterals interface{}
+		}, error)
+	}
+	Range(_ func(FieldDescriptor, Value) bool)
+	Set(_ FieldDescriptor, _ Value)
+	SetUnknown(_ RawFields)
+	Type() MessageType
+	WhichOneof(_ OneofDescriptor) FieldDescriptor
+}
+
+type MessageDescriptor interface {
+	Enums() EnumDescriptors
+	ExtensionRangeOptions(_ int) ProtoMessage
+	ExtensionRanges() FieldRanges
+	Extensions() ExtensionDescriptors
+	Fields() FieldDescriptors
+	FullName() FullName
+	Index() int
+	IsMapEntry() bool
+	IsPlaceholder() bool
+	Messages() MessageDescriptors
+	Name() Name
+	Oneofs() OneofDescriptors
+	Options() ProtoMessage
+	Parent() Descriptor
+	ParentFile() FileDescriptor
+	ProtoInternal(_ interface{})
+	ProtoType(_ MessageDescriptor)
+	RequiredNumbers() FieldNumbers
+	ReservedNames() Names
+	ReservedRanges() FieldRanges
+	Syntax() Syntax
+}
+
+type MessageDescriptors interface {
+	ByName(_ Name) MessageDescriptor
+	Get(_ int) MessageDescriptor
+	Len() int
+	ProtoInternal(_ interface{})
+}
+
+type MessageType interface {
+	Descriptor() MessageDescriptor
+	New() Message
+	Zero() Message
+}
+
+type MethodDescriptor interface {
+	FullName() FullName
+	Index() int
+	Input() MessageDescriptor
+	IsPlaceholder() bool
+	IsStreamingClient() bool
+	IsStreamingServer() bool
+	Name() Name
+	Options() ProtoMessage
+	Output() MessageDescriptor
+	Parent() Descriptor
+	ParentFile() FileDescriptor
+	ProtoInternal(_ interface{})
+	ProtoType(_ MethodDescriptor)
+	Syntax() Syntax
+}
+
+type MethodDescriptors interface {
+	ByName(_ Name) MethodDescriptor
+	Get(_ int) MethodDescriptor
+	Len() int
+	ProtoInternal(_ interface{})
+}
+
+type Name string
+
+func (_ Name) IsValid() bool {
+	return false
+}
+
+type Names interface {
+	Get(_ int) Name
+	Has(_ Name) bool
+	Len() int
+	ProtoInternal(_ interface{})
+}
+
+type OneofDescriptor interface {
+	Fields() FieldDescriptors
+	FullName() FullName
+	Index() int
+	IsPlaceholder() bool
+	IsSynthetic() bool
+	Name() Name
+	Options() ProtoMessage
+	Parent() Descriptor
+	ParentFile() FileDescriptor
+	ProtoInternal(_ interface{})
+	ProtoType(_ OneofDescriptor)
+	Syntax() Syntax
+}
+
+type OneofDescriptors interface {
+	ByName(_ Name) OneofDescriptor
+	Get(_ int) OneofDescriptor
+	Len() int
+	ProtoInternal(_ interface{})
+}
+
+type ProtoMessage interface {
+	ProtoReflect() Message
+}
+
+type RawFields []byte
+
+func (_ RawFields) IsValid() bool {
+	return false
+}
+
+type ServiceDescriptor interface {
+	FullName() FullName
+	Index() int
+	IsPlaceholder() bool
+	Methods() MethodDescriptors
+	Name() Name
+	Options() ProtoMessage
+	Parent() Descriptor
+	ParentFile() FileDescriptor
+	ProtoInternal(_ interface{})
+	ProtoType(_ ServiceDescriptor)
+	Syntax() Syntax
+}
+
+type ServiceDescriptors interface {
+	ByName(_ Name) ServiceDescriptor
+	Get(_ int) ServiceDescriptor
+	Len() int
+	ProtoInternal(_ interface{})
+}
+
+type SourceLocation struct {
+	Path                    SourcePath
+	StartLine               int
+	StartColumn             int
+	EndLine                 int
+	EndColumn               int
+	LeadingDetachedComments []string
+	LeadingComments         string
+	TrailingComments        string
+}
+
+type SourceLocations interface {
+	Get(_ int) SourceLocation
+	Len() int
+	ProtoInternal(_ interface{})
+}
+
+type SourcePath []int32
+
+type Syntax int8
+
+func (_ Syntax) GoString() string {
+	return ""
+}
+
+func (_ Syntax) IsValid() bool {
+	return false
+}
+
+func (_ Syntax) String() string {
+	return ""
+}
+
+type Value struct {
+	DoNotCompare interface{}
+}
+
+func (_ Value) Bool() bool {
+	return false
+}
+
+func (_ Value) Bytes() []byte {
+	return nil
+}
+
+func (_ Value) Enum() EnumNumber {
+	return 0
+}
+
+func (_ Value) Float() float64 {
+	return 0
+}
+
+func (_ Value) Int() int64 {
+	return 0
+}
+
+func (_ Value) Interface() interface{} {
+	return nil
+}
+
+func (_ Value) IsValid() bool {
+	return false
+}
+
+func (_ Value) List() List {
+	return nil
+}
+
+func (_ Value) Map() Map {
+	return nil
+}
+
+func (_ Value) MapKey() MapKey {
+	return MapKey{}
+}
+
+func (_ Value) Message() Message {
+	return nil
+}
+
+func (_ Value) String() string {
+	return ""
+}
+
+func (_ Value) Uint() uint64 {
+	return 0
+}

go/ql/test/query-tests/Security/CWE-312/vendor/google.golang.org/protobuf/runtime/protoiface/stub.go

@@ -0,0 +1,29 @@
+// This is a simple stub for google.golang.org/protobuf/runtime/protoiface, strictly for use in testing.
+
+// See the LICENSE file for information about the licensing of the original library.
+// Source: google.golang.org/protobuf/runtime/protoiface (exports: MessageV1; functions: )
+
+// Package protoiface is a stub of google.golang.org/protobuf/runtime/protoiface.
+package protoiface
+
+import (
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+type MessageV1 interface {
+	ProtoMessage()
+	Reset()
+	String() string
+}
+
+type MarshalInputFlags = uint8
+
+type MarshalInput struct {
+	Message protoreflect.Message
+	Buf     []byte // output is appended to this buffer
+	Flags   MarshalInputFlags
+}
+
+type MarshalOutput struct {
+	Buf []byte // contains marshaled message
+}

go/ql/test/query-tests/Security/CWE-312/vendor/google.golang.org/protobuf/runtime/protoimpl/stub.go

@@ -0,0 +1,107 @@
+// This is a simple stub for google.golang.org/protobuf/runtime/protoimpl, strictly for use in testing.
+
+// See the LICENSE file for information about the licensing of the original library.
+// Source: google.golang.org/protobuf/runtime/protoimpl (exports: MessageState,SizeCache,UnknownFields,Pointer,EnforceVersion; functions: MinVersion,MaxVersion,UnsafeEnabled,X)
+
+// Package protoimpl is a stub of google.golang.org/protobuf/runtime/protoimpl.
+package protoimpl
+
+import (
+	impl "google.golang.org/protobuf/internal/impl"
+)
+
+type EnforceVersion uint
+
+const MaxVersion int = 20
+
+type MessageState = impl.MessageState
+
+const MinVersion int = 20
+
+type Pointer = impl.Pointer
+
+type SizeCache = int32
+
+type UnknownFields = []byte
+
+var UnsafeEnabled bool = false
+
+// Export is a zero-length named type that exists only to export a set of
+// functions that we do not want to appear in godoc.
+type Export struct{}
+
+var X Export = Export{}
+
+func (Export) NewError(f string, x ...interface{}) error {
+	return nil
+}
+
+type enum = interface{}
+
+func (Export) EnumOf(e enum) interface{} {
+	return nil
+}
+
+func (Export) EnumDescriptorOf(e enum) interface{} {
+	return nil
+}
+
+func (Export) EnumTypeOf(e enum) interface{} {
+	return nil
+}
+
+func (Export) EnumStringOf(ed interface{}, n interface{}) string {
+	return ""
+}
+
+type message = interface{}
+
+type legacyMessageWrapper struct{ m interface{} }
+
+func (m legacyMessageWrapper) Reset()         {}
+func (m legacyMessageWrapper) String() string { return "" }
+func (m legacyMessageWrapper) ProtoMessage()  {}
+
+func (Export) ProtoMessageV1Of(m message) interface{} {
+	return nil
+}
+
+func (Export) protoMessageV2Of(m message) interface{} {
+	return nil
+}
+
+func (Export) ProtoMessageV2Of(m message) interface{} {
+	return nil
+}
+
+func (Export) MessageOf(m message) interface{} {
+	return nil
+}
+
+func (Export) MessageDescriptorOf(m message) interface{} {
+	return nil
+}
+
+func (Export) MessageTypeOf(m message) interface{} {
+	return nil
+}
+
+func (Export) MessageStringOf(m interface{}) string {
+	return ""
+}
+
+func (Export) MessageStateOf(p Pointer) *MessageState {
+	return nil
+}
+
+func (Export) CompressGZIP(_ []byte) []byte {
+	return nil
+}
+
+type EnumInfo = impl.EnumInfo
+
+type MessageInfo = impl.MessageInfo
+
+type TypeBuilder = impl.TypeBuilder
+
+type DescBuilder = impl.DescBuilder

go/ql/test/query-tests/Security/CWE-312/vendor/modules.txt

@@ -7,3 +7,9 @@ github.com/sirupsen/logrus
 # k8s.io/klog v1.0.0
 ## explicit
 k8s.io/klog
+# github.com/golang/protobuf v1.4.2
+## explicit
+github.com/golang/protobuf
+# google.golang.org/protobuf v1.23.0
+## explicit
+google.golang.org/protobuf

java/documentation/library-coverage/coverage.rst

@@ -8,16 +8,20 @@ Java framework & library support
 
    Framework / library,Package,Flow sources,Taint & value steps,Sinks (total),`CWE‑022` :sub:`Path injection`,`CWE‑036` :sub:`Path traversal`,`CWE‑079` :sub:`Cross-site scripting`,`CWE‑089` :sub:`SQL injection`,`CWE‑090` :sub:`LDAP injection`,`CWE‑094` :sub:`Code injection`,`CWE‑319` :sub:`Cleartext transmission`
    Android,``android.*``,52,479,116,,,3,67,,,
+   Android extensions,``androidx.*``,5,183,8,,,,,,,
    `Apache Commons Collections <https://commons.apache.org/proper/commons-collections/>`_,"``org.apache.commons.collections``, ``org.apache.commons.collections4``",,1600,,,,,,,,
    `Apache Commons IO <https://commons.apache.org/proper/commons-io/>`_,``org.apache.commons.io``,,556,106,91,,,,,,15
    `Apache Commons Lang <https://commons.apache.org/proper/commons-lang/>`_,``org.apache.commons.lang3``,,424,,,,,,,,
    `Apache Commons Text <https://commons.apache.org/proper/commons-text/>`_,``org.apache.commons.text``,,272,,,,,,,,
    `Apache HttpComponents <https://hc.apache.org/>`_,"``org.apache.hc.core5.*``, ``org.apache.http``",5,136,28,,,3,,,,25
+   `Apache Log4j 2 <https://logging.apache.org/log4j/2.0/>`_,``org.apache.logging.log4j``,,8,359,,,,,,,
    `Google Guava <https://guava.dev/>`_,``com.google.common.*``,,728,39,,6,,,,,
+   JBoss Logging,``org.jboss.logging``,,,324,,,,,,,
    `JSON-java <https://github.com/stleary/JSON-java>`_,``org.json``,,236,,,,,,,,
    Java Standard Library,``java.*``,3,589,130,28,,,7,,,10
    Java extensions,"``javax.*``, ``jakarta.*``",63,609,32,,,4,,1,1,2
+   Kotlin Standard Library,``kotlin*``,,1835,12,10,,,,,,2
    `Spring <https://spring.io/>`_,``org.springframework.*``,29,477,101,,,,19,14,,29
-   Others,"``androidx.core.app``, ``androidx.slice``, ``cn.hutool.core.codec``, ``com.esotericsoftware.kryo.io``, ``com.esotericsoftware.kryo5.io``, ``com.fasterxml.jackson.core``, ``com.fasterxml.jackson.databind``, ``com.hubspot.jinjava``, ``com.mitchellbosecke.pebble``, ``com.opensymphony.xwork2.ognl``, ``com.rabbitmq.client``, ``com.unboundid.ldap.sdk``, ``com.zaxxer.hikari``, ``flexjson``, ``freemarker.cache``, ``freemarker.template``, ``groovy.lang``, ``groovy.util``, ``jodd.json``, ``kotlin``, ``net.sf.saxon.s9api``, ``ognl``, ``okhttp3``, ``org.apache.commons.codec``, ``org.apache.commons.jexl2``, ``org.apache.commons.jexl3``, ``org.apache.commons.logging``, ``org.apache.commons.ognl``, ``org.apache.directory.ldap.client.api``, ``org.apache.ibatis.jdbc``, ``org.apache.log4j``, ``org.apache.logging.log4j``, ``org.apache.shiro.codec``, ``org.apache.shiro.jndi``, ``org.apache.velocity.app``, ``org.apache.velocity.runtime``, ``org.codehaus.groovy.control``, ``org.dom4j``, ``org.hibernate``, ``org.jboss.logging``, ``org.jdbi.v3.core``, ``org.jooq``, ``org.mvel2``, ``org.scijava.log``, ``org.slf4j``, ``org.thymeleaf``, ``org.xml.sax``, ``org.xmlpull.v1``, ``play.mvc``, ``ratpack.core.form``, ``ratpack.core.handling``, ``ratpack.core.http``, ``ratpack.exec``, ``ratpack.form``, ``ratpack.func``, ``ratpack.handling``, ``ratpack.http``, ``ratpack.util``, ``retrofit2``",65,2326,972,10,,,14,18,,5
+   Others,"``cn.hutool.core.codec``, ``com.esotericsoftware.kryo.io``, ``com.esotericsoftware.kryo5.io``, ``com.fasterxml.jackson.core``, ``com.fasterxml.jackson.databind``, ``com.hubspot.jinjava``, ``com.mitchellbosecke.pebble``, ``com.opensymphony.xwork2.ognl``, ``com.rabbitmq.client``, ``com.unboundid.ldap.sdk``, ``com.zaxxer.hikari``, ``flexjson``, ``freemarker.cache``, ``freemarker.template``, ``groovy.lang``, ``groovy.util``, ``jodd.json``, ``net.sf.saxon.s9api``, ``ognl``, ``okhttp3``, ``org.apache.commons.codec``, ``org.apache.commons.jexl2``, ``org.apache.commons.jexl3``, ``org.apache.commons.logging``, ``org.apache.commons.ognl``, ``org.apache.directory.ldap.client.api``, ``org.apache.ibatis.jdbc``, ``org.apache.log4j``, ``org.apache.shiro.codec``, ``org.apache.shiro.jndi``, ``org.apache.velocity.app``, ``org.apache.velocity.runtime``, ``org.codehaus.groovy.control``, ``org.dom4j``, ``org.hibernate``, ``org.jdbi.v3.core``, ``org.jooq``, ``org.mvel2``, ``org.scijava.log``, ``org.slf4j``, ``org.thymeleaf``, ``org.xml.sax``, ``org.xmlpull.v1``, ``play.mvc``, ``ratpack.core.form``, ``ratpack.core.handling``, ``ratpack.core.http``, ``ratpack.exec``, ``ratpack.form``, ``ratpack.func``, ``ratpack.handling``, ``ratpack.http``, ``ratpack.util``, ``retrofit2``",60,300,269,,,,14,18,,3
    Totals,,217,8432,1524,129,6,10,107,33,1,86
 

java/documentation/library-coverage/frameworks.csv

@@ -1,12 +1,16 @@
 Framework name,URL,Package prefixes
 Java Standard Library,,java.*
 Java extensions,,javax.* jakarta.*
+Kotlin Standard Library,,kotlin*
+Android,,android.*
+Android extensions,,androidx.*
 Apache Commons Collections,https://commons.apache.org/proper/commons-collections/,org.apache.commons.collections org.apache.commons.collections4
 Apache Commons IO,https://commons.apache.org/proper/commons-io/,org.apache.commons.io
 Apache Commons Lang,https://commons.apache.org/proper/commons-lang/,org.apache.commons.lang3
 Apache Commons Text,https://commons.apache.org/proper/commons-text/,org.apache.commons.text
 Apache HttpComponents,https://hc.apache.org/,org.apache.hc.core5.* org.apache.http
-Android,,android.*
+Apache Log4j 2,https://logging.apache.org/log4j/2.0/,org.apache.logging.log4j
 Google Guava,https://guava.dev/,com.google.common.*
+JBoss Logging,,org.jboss.logging
 JSON-java,https://github.com/stleary/JSON-java,org.json
 Spring,https://spring.io/,org.springframework.*

java/kotlin-extractor/kotlin_plugin_versions.py

@@ -49,7 +49,7 @@ def get_single_version(fakeVersionOutput = None):
     matching_minor_versions.sort(reverse = True)
 
     for version in matching_minor_versions:
-        if version <= current_version:
+        if version[0:3] <= current_version[0:3]:
             return version_tuple_to_string(version)
 
     return version_tuple_to_string(matching_minor_versions[-1])

java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java

@@ -277,7 +277,7 @@ public class OdasaOutput {
 				// Only re-write an existing trap file if we encountered a newer version of the same class.
 				TrapClassVersion trapVersion = readVersionInfo(trap);
 				if (!currVersion.isValid()) {
-					log.warn("Not rewriting trap file for: " + shortName + " " + trapVersion + " " + currVersion + " " + trap);
+					log.trace("Not rewriting trap file for: " + shortName + " " + trapVersion + " " + currVersion + " " + trap);
 				} else if (currVersion.newerThan(trapVersion)) {
 					log.trace("Rewriting trap file for: " + shortName + " " + trapVersion + " " + currVersion + " " + trap);
 					deleteTrapFileAndDependencies(sym, signature);
@@ -291,7 +291,7 @@ public class OdasaOutput {
 			// If the TRAP file already exists then we
 			// don't need to write it.
 			if (trap.exists()) {
-				log.warn("Not rewriting trap file for " + trap.toString() + " as it exists");
+				log.trace("Not rewriting trap file for " + trap.toString() + " as it exists");
 				return null;
 			}
 			// If the TRAP file was written in the past, and
@@ -301,7 +301,7 @@ public class OdasaOutput {
 			File trapFileDir = trap.getParentFile();
 			File trapOld = new File(trapFileDir, trap.getName().replace(".trap.gz", ".trap-old.gz"));
 			if (trapOld.exists()) {
-				log.warn("Not rewriting trap file for " + trap.toString() + " as the trap-old exists");
+				log.trace("Not rewriting trap file for " + trap.toString() + " as the trap-old exists");
 				return null;
 			}
 			// Otherwise, if any newer TRAP file has already
@@ -316,7 +316,7 @@ public class OdasaOutput {
 					if (m.matches() && m.group(1).equals(trapFileBaseName)) {
 						TrapClassVersion v = new TrapClassVersion(Integer.valueOf(m.group(2)), Integer.valueOf(m.group(3)), Long.valueOf(m.group(4)), m.group(5));
 						if (v.newerThan(trapFileVersion)) {
-							log.warn("Not rewriting trap file for " + trap.toString() + " as " + f.toString() + " exists");
+							log.trace("Not rewriting trap file for " + trap.toString() + " as " + f.toString() + " exists");
 							return null;
 						}
 					}

java/kotlin-extractor/src/main/java/com/semmle/util/trap/pathtransformers/PathTransformer.java

@@ -14,7 +14,15 @@ public abstract class PathTransformer {
 	 * canonical, absolute, strings and normalises away Unix/Windows differences.
 	 */
 	public String fileAsDatabaseString(File file) {
-		String path;
+		String path = file.getPath();
+		// For /!unknown-binary-location/... and /modules/...
+		// paths, on Windows the standard code wants to
+		// normalise them to e.g. C:/!unknown-binary-location/...
+		// which is particularly annoying for cross-platform test
+		// output. We therefore handle them specially here.
+		if (path.matches("^[/\\\\](!unknown-binary-location|modules)[/\\\\].*")) {
+			return path.replace('\\', '/');
+		}
 		if (Boolean.valueOf(Env.systemEnv().get(Var.SEMMLE_PRESERVE_SYMLINKS)))
 			path = FileUtil.simplifyPath(file);
 		else
@@ -43,4 +51,4 @@ public abstract class PathTransformer {
 	public static PathTransformer std() {
 		return DEFAULT_TRANSFORMER;
 	}
-}
+}

java/kotlin-extractor/src/main/kotlin/ExternalDeclExtractor.kt

@@ -35,8 +35,6 @@ class ExternalDeclExtractor(val logger: FileLogger, val invocationTrapFile: Stri
         if (ret) externalDeclWorkList.add(Pair(d, signature))
         return ret
     }
-    fun extractLater(p: IrProperty) = extractLater(p, propertySignature)
-    fun extractLater(f: IrField) = extractLater(f, fieldSignature)
     fun extractLater(c: IrClass) = extractLater(c, "")
 
     fun extractExternalClasses() {

java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt

@@ -2,6 +2,7 @@ package com.github.codeql
 
 import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
 import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
+import org.jetbrains.kotlin.config.KotlinCompilerVersion
 import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.ir.util.*
 import org.jetbrains.kotlin.ir.IrElement
@@ -139,6 +140,8 @@ class KotlinExtractorExtension(
             logger.flush()
             logger.info("Extraction for invocation TRAP file $invocationTrapFile")
             logger.flush()
+            logger.info("Kotlin version ${KotlinCompilerVersion.getVersion()}")
+            logger.flush()
             logPeakMemoryUsage(logger, "before extractor")
             if (System.getenv("CODEQL_EXTRACTOR_JAVA_KOTLIN_DUMP") == "true") {
                 logger.info("moduleFragment:\n" + moduleFragment.dump())

java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt

@@ -17,6 +17,7 @@ import org.jetbrains.kotlin.ir.IrStatement
 import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI
 import org.jetbrains.kotlin.ir.backend.js.utils.realOverrideTarget
 import org.jetbrains.kotlin.ir.declarations.*
+import org.jetbrains.kotlin.ir.declarations.lazy.IrLazyFunction
 import org.jetbrains.kotlin.ir.expressions.*
 import org.jetbrains.kotlin.ir.expressions.impl.IrConstImpl
 import org.jetbrains.kotlin.ir.symbols.*
@@ -27,6 +28,7 @@ import org.jetbrains.kotlin.load.java.structure.JavaClass
 import org.jetbrains.kotlin.load.java.structure.JavaMethod
 import org.jetbrains.kotlin.load.java.structure.JavaTypeParameter
 import org.jetbrains.kotlin.load.java.structure.JavaTypeParameterListOwner
+import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.types.Variance
 import org.jetbrains.kotlin.util.OperatorNameConventions
@@ -98,15 +100,29 @@ open class KotlinFileExtractor(
         }
     }
 
+    private fun javaBinaryDeclaresMethod(c: IrClass, name: String) =
+        ((c.source as? JavaSourceElement)?.javaElement as? BinaryJavaClass)?.methods?.any { it.name.asString() == name }
+
+    private fun isJavaBinaryDeclaration(f: IrFunction) =
+        f.parentClassOrNull?.let { javaBinaryDeclaresMethod(it, f.name.asString()) } ?: false
+
+    private fun isJavaBinaryObjectMethodRedeclaration(d: IrDeclaration) =
+        when (d) {
+            is IrFunction ->
+                when (d.name.asString()) {
+                    "toString" -> d.valueParameters.isEmpty()
+                    "hashCode" -> d.valueParameters.isEmpty()
+                    "equals" -> d.valueParameters.singleOrNull()?.type?.isNullableAny() ?: false
+                    else -> false
+                } && isJavaBinaryDeclaration(d)
+            else -> false
+        }
+
     @OptIn(ObsoleteDescriptorBasedAPI::class)
     private fun isFake(d: IrDeclarationWithVisibility): Boolean {
-        val visibility = d.visibility
-        if (visibility is DelegatedDescriptorVisibility && visibility.delegate == Visibilities.InvisibleFake) {
+        val hasFakeVisibility = d.visibility.let { it is DelegatedDescriptorVisibility && it.delegate == Visibilities.InvisibleFake } || d.isFakeOverride
+        if (hasFakeVisibility && !isJavaBinaryObjectMethodRedeclaration(d))
             return true
-        }
-        if (d.isFakeOverride) {
-            return true
-        }
         try {
             if ((d as? IrFunction)?.descriptor?.isHiddenToOvercomeSignatureClash == true) {
                 return true
@@ -305,7 +321,7 @@ open class KotlinFileExtractor(
                 val kind = c.kind
                 if (kind == ClassKind.ENUM_CLASS) {
                     tw.writeIsEnumType(classId)
-                } else if (kind != ClassKind.CLASS && kind != ClassKind.OBJECT) {
+                } else if (kind != ClassKind.CLASS && kind != ClassKind.OBJECT && kind != ClassKind.ENUM_ENTRY) {
                     logger.errorElement("Unrecognised class kind $kind", c)
                 }
             }
@@ -452,7 +468,7 @@ open class KotlinFileExtractor(
                     val kind = c.kind
                     if (kind == ClassKind.ENUM_CLASS) {
                         tw.writeIsEnumType(classId)
-                    } else if (kind != ClassKind.CLASS && kind != ClassKind.OBJECT) {
+                    } else if (kind != ClassKind.CLASS && kind != ClassKind.OBJECT && kind != ClassKind.ENUM_ENTRY) {
                         logger.warnElement("Unrecognised class kind $kind", c)
                     }
 
@@ -676,7 +692,7 @@ open class KotlinFileExtractor(
                     null
             } ?: vp.type
             val javaType = (vp.parent as? IrFunction)?.let { getJavaCallable(it)?.let { jCallable -> getJavaValueParameterType(jCallable, idx) } }
-            val typeWithWildcards = addJavaLoweringWildcards(maybeAlteredType, !hasWildcardSuppressionAnnotation(vp), javaType)
+            val typeWithWildcards = addJavaLoweringWildcards(maybeAlteredType, !getInnermostWildcardSupppressionAnnotation(vp), javaType)
             val substitutedType = typeSubstitution?.let { it(typeWithWildcards, TypeContext.OTHER, pluginContext) } ?: typeWithWildcards
             val id = useValueParameter(vp, parent)
             if (extractTypeAccess) {
@@ -908,7 +924,9 @@ open class KotlinFileExtractor(
             else
                 null
         } else {
-            forceExtractFunction(f, parentId, extractBody, extractMethodAndParameterTypeAccesses, typeSubstitution, classTypeArgsIncludingOuterClasses).also {
+            // Work around an apparent bug causing redeclarations of `fun toString(): String` specifically in interfaces loaded from Java classes show up like fake overrides.
+            val overriddenVisibility = if (f.isFakeOverride && isJavaBinaryObjectMethodRedeclaration(f)) OverriddenFunctionAttributes(visibility = DescriptorVisibilities.PUBLIC) else null
+            forceExtractFunction(f, parentId, extractBody, extractMethodAndParameterTypeAccesses, typeSubstitution, classTypeArgsIncludingOuterClasses, overriddenAttributes = overriddenVisibility).also {
                 // The defaults-forwarder function is a static utility, not a member, so we only need to extract this for the unspecialised instance of this class.
                 if (classTypeArgsIncludingOuterClasses.isNullOrEmpty())
                     extractDefaultsFunction(f, parentId, extractBody, extractMethodAndParameterTypeAccesses)
@@ -1171,22 +1189,16 @@ open class KotlinFileExtractor(
                             id
 
                 val extReceiver = f.extensionReceiverParameter
-                val idxOffset = if (extReceiver != null) 1 else 0
-                val fParameters = overriddenAttributes?.valueParameters ?: f.valueParameters
+                val fParameters = listOfNotNull(extReceiver) + (overriddenAttributes?.valueParameters ?: f.valueParameters)
                 val paramTypes = fParameters.mapIndexed { i, vp ->
-                    extractValueParameter(vp, id, i + idxOffset, typeSubstitution, sourceDeclaration, classTypeArgsIncludingOuterClasses, extractTypeAccess = extractMethodAndParameterTypeAccesses, overriddenAttributes?.sourceLoc)
+                    extractValueParameter(vp, id, i, typeSubstitution, sourceDeclaration, classTypeArgsIncludingOuterClasses, extractTypeAccess = extractMethodAndParameterTypeAccesses, overriddenAttributes?.sourceLoc)
                 }
-                val allParamTypes = if (extReceiver != null) {
-                    val extendedType = useType(extReceiver.type)
+                if (extReceiver != null) {
+                    val extendedType = paramTypes[0]
                     tw.writeKtExtensionFunctions(id.cast<DbMethod>(), extendedType.javaResult.id, extendedType.kotlinResult.id)
-
-                    val t = extractValueParameter(extReceiver, id, 0, null, sourceDeclaration, classTypeArgsIncludingOuterClasses, extractTypeAccess = extractMethodAndParameterTypeAccesses, overriddenAttributes?.sourceLoc)
-                    listOf(t) + paramTypes
-                } else {
-                    paramTypes
                 }
 
-                val paramsSignature = allParamTypes.joinToString(separator = ",", prefix = "(", postfix = ")") { signatureOrWarn(it.javaResult, f) }
+                val paramsSignature = paramTypes.joinToString(separator = ",", prefix = "(", postfix = ")") { signatureOrWarn(it.javaResult, f) }
 
                 val adjustedReturnType = addJavaLoweringWildcards(getAdjustedReturnType(f), false, (javaCallable as? JavaMethod)?.returnType)
                 val substReturnType = typeSubstitution?.let { it(adjustedReturnType, TypeContext.RETURN, pluginContext) } ?: adjustedReturnType
@@ -1482,7 +1494,7 @@ open class KotlinFileExtractor(
         }
     }
 
-    private fun extractVariableExpr(v: IrVariable, callable: Label<out DbCallable>, parent: Label<out DbExprparent>, idx: Int, enclosingStmt: Label<out DbStmt>) {
+    private fun extractVariableExpr(v: IrVariable, callable: Label<out DbCallable>, parent: Label<out DbExprparent>, idx: Int, enclosingStmt: Label<out DbStmt>, extractInitializer: Boolean = true) {
         with("variable expr", v) {
             val varId = useVariable(v)
             val exprId = tw.getFreshIdLabel<DbLocalvariabledeclexpr>()
@@ -1497,7 +1509,7 @@ open class KotlinFileExtractor(
             tw.writeCallableEnclosingExpr(exprId, callable)
             tw.writeStatementEnclosingExpr(exprId, enclosingStmt)
             val i = v.initializer
-            if (i != null) {
+            if (i != null && extractInitializer) {
                 extractExpressionExpr(i, callable, exprId, 0, enclosingStmt)
             }
             if (!v.isVar) {
@@ -1884,7 +1896,7 @@ open class KotlinFileExtractor(
             IrConstImpl.defaultValueForType(0, 0, getDefaultsMethodLastArgType(callTarget))
         )
 
-        extractCallValueArguments(id, valueArgsWithDummies + extraArgs, enclosingStmt, enclosingCallable, nextIdx)
+        extractCallValueArguments(id, valueArgsWithDummies + extraArgs, enclosingStmt, enclosingCallable, nextIdx, extractVarargAsArray = true)
     }
 
     private fun getFunctionInvokeMethod(typeArgs: List<IrTypeArgument>): IrFunction? {
@@ -1945,6 +1957,18 @@ open class KotlinFileExtractor(
         }
     }
 
+    private fun getCalleeRealOverrideTarget(f: IrFunction): IrFunction {
+        val target = f.target.realOverrideTarget
+        return if (overridesCollectionsMethodWithAlteredParameterTypes(f))
+        // Cope with the case where an inherited callee can be rewritten with substituted parameter types
+        // if the child class uses it to implement a collections interface
+        // (for example, `class A { boolean contains(Object o) { ... } }; class B<T> extends A implements Set<T> { ... }`
+        // leads to generating a function `A.contains(B::T)`, with `initialSignatureFunction` pointing to `A.contains(Object)`.
+            (target as? IrLazyFunction)?.initialSignatureFunction ?: target
+        else
+            target
+    }
+
     fun extractRawMethodAccess(
         syntacticCallTarget: IrFunction,
         locElement: IrElement,
@@ -1961,8 +1985,12 @@ open class KotlinFileExtractor(
         superQualifierSymbol: IrClassSymbol? = null) {
 
         val locId = tw.getLocation(locElement)
+        val varargParam = syntacticCallTarget.valueParameters.withIndex().find { it.value.isVararg }
+        // If the vararg param is the only one not specified, and it has no default value, then we don't need to call a $default method,
+        // as omitting it already implies passing an empty vararg array.
+        val nullAllowedIdx = if (varargParam != null && varargParam.value.defaultValue == null) varargParam.index else -1
 
-        if (valueArguments.any { it == null }) {
+        if (valueArguments.withIndex().any { (index, it) -> it == null && index != nullAllowedIdx }) {
             extractsDefaultsCall(
                 syntacticCallTarget,
                 locId,
@@ -2013,7 +2041,7 @@ open class KotlinFileExtractor(
         extractClassTypeArguments: Boolean = false,
         superQualifierSymbol: IrClassSymbol? = null) {
 
-        val callTarget = syntacticCallTarget.target.realOverrideTarget
+        val callTarget = getCalleeRealOverrideTarget(syntacticCallTarget)
         val methodId = getCalleeMethodId(callTarget, drType, extractClassTypeArguments)
         if (methodId == null) {
             logger.warn("No method to bind call to for raw method access")
@@ -2082,11 +2110,11 @@ open class KotlinFileExtractor(
     private fun extractCallValueArguments(callId: Label<out DbExprparent>, call: IrFunctionAccessExpression, enclosingStmt: Label<out DbStmt>, enclosingCallable: Label<out DbCallable>, idxOffset: Int) =
         extractCallValueArguments(callId, (0 until call.valueArgumentsCount).map { call.getValueArgument(it) }, enclosingStmt, enclosingCallable, idxOffset)
 
-    private fun extractCallValueArguments(callId: Label<out DbExprparent>, valueArguments: List<IrExpression?>, enclosingStmt: Label<out DbStmt>, enclosingCallable: Label<out DbCallable>, idxOffset: Int) {
+    private fun extractCallValueArguments(callId: Label<out DbExprparent>, valueArguments: List<IrExpression?>, enclosingStmt: Label<out DbStmt>, enclosingCallable: Label<out DbCallable>, idxOffset: Int, extractVarargAsArray: Boolean = false) {
         var i = 0
         valueArguments.forEach { arg ->
             if(arg != null) {
-                if (arg is IrVararg) {
+                if (arg is IrVararg && !extractVarargAsArray) {
                     arg.elements.forEachIndexed { varargNo, vararg -> extractVarargElement(vararg, enclosingCallable, callId, i + idxOffset + varargNo, enclosingStmt) }
                     i += arg.elements.size
                 } else {
@@ -3153,6 +3181,90 @@ open class KotlinFileExtractor(
         }
     }
 
+    /**
+     * This method tries to extract a block as an enhanced for loop.
+     * It returns true if it succeeds, and false otherwise.
+     */
+    private fun tryExtractForLoop(e: IrContainerExpression, callable: Label<out DbCallable>, parent: StmtExprParent): Boolean {
+        /*
+         * We're expecting the pattern
+         * {
+         *   val iterator = [expr].iterator()
+         *   while (iterator.hasNext()) {
+         *    val [loopVar] = iterator.next()
+         *    [block]
+         *   }
+         * }
+         */
+
+        if (e.origin != IrStatementOrigin.FOR_LOOP ||
+            e.statements.size != 2) {
+            return false
+        }
+
+        val iteratorVariable = e.statements[0] as? IrVariable
+        val innerWhile = e.statements[1] as? IrWhileLoop
+
+        if (iteratorVariable == null ||
+            iteratorVariable.origin != IrDeclarationOrigin.FOR_LOOP_ITERATOR ||
+            innerWhile == null ||
+            innerWhile.origin != IrStatementOrigin.FOR_LOOP_INNER_WHILE) {
+            return false
+        }
+
+        val initializer = iteratorVariable.initializer as? IrCall
+        if (initializer == null ||
+            initializer.origin != IrStatementOrigin.FOR_LOOP_ITERATOR ||
+            initializer.symbol.owner.name.asString() != "iterator") {
+            return false
+        }
+
+        val expr = initializer.dispatchReceiver
+        val cond = innerWhile.condition as? IrCall
+        val body = innerWhile.body as? IrBlock
+
+        if (expr == null ||
+            cond == null ||
+            cond.origin != IrStatementOrigin.FOR_LOOP_HAS_NEXT ||
+            (cond.dispatchReceiver as? IrGetValue)?.symbol?.owner != iteratorVariable ||
+            body == null ||
+            body.origin != IrStatementOrigin.FOR_LOOP_INNER_WHILE ||
+            body.statements.size < 2) {
+            return false
+        }
+
+        val loopVar = body.statements[0] as? IrVariable
+        val nextCall = loopVar?.initializer as? IrCall
+
+        if (loopVar == null ||
+            !(loopVar.origin == IrDeclarationOrigin.FOR_LOOP_VARIABLE || loopVar.origin == IrDeclarationOrigin.IR_TEMPORARY_VARIABLE) ||
+            nextCall == null ||
+            nextCall.origin != IrStatementOrigin.FOR_LOOP_NEXT ||
+            (nextCall.dispatchReceiver as? IrGetValue)?.symbol?.owner != iteratorVariable) {
+            return false
+        }
+
+        val id = extractLoop(innerWhile, null, parent, callable) { p, idx ->
+            tw.getFreshIdLabel<DbEnhancedforstmt>().also {
+                tw.writeStmts_enhancedforstmt(it, p, idx, callable)
+            }
+        }
+
+        extractVariableExpr(loopVar, callable, id, 0, id, extractInitializer = false)
+        extractExpressionExpr(expr, callable, id, 1, id)
+        val block = body.statements[1] as? IrBlock
+        if (body.statements.size == 2 && block != null) {
+            // Extract the body that was given to us by the compiler
+            extractExpressionStmt(block, callable, id, 2)
+        } else {
+            // Extract a block with all but the first (loop variable declaration) statement
+            extractBlock(body, body.statements.takeLast(body.statements.size - 1), id, 2, callable)
+        }
+
+        return true
+    }
+
+
     /**
      * This tried to extract a block as an array update.
      * It returns true if it succeeds, and false otherwise.
@@ -3384,22 +3496,17 @@ open class KotlinFileExtractor(
                     }
                 }
                 is IrContainerExpression -> {
-                    if(!tryExtractArrayUpdate(e, callable, parent)) {
-                        val stmtParent = parent.stmt(e, callable)
-                        val id = tw.getFreshIdLabel<DbBlock>()
-                        val locId = tw.getLocation(e)
-                        tw.writeStmts_block(id, stmtParent.parent, stmtParent.idx, callable)
-                        tw.writeHasLocation(id, locId)
-                        e.statements.forEachIndexed { i, s ->
-                            extractStatement(s, callable, id, i)
-                        }
+                    if (!tryExtractArrayUpdate(e, callable, parent) &&
+                        !tryExtractForLoop(e, callable, parent)) {
+
+                        extractBlock(e, e.statements, parent, callable)
                     }
                 }
                 is IrWhileLoop -> {
-                    extractLoop(e, parent, callable)
+                    extractLoopWithCondition(e, parent, callable)
                 }
                 is IrDoWhileLoop -> {
-                    extractLoop(e, parent, callable)
+                    extractLoopWithCondition(e, parent, callable)
                 }
                 is IrInstanceInitializerCall -> {
                     val irConstructor = declarationStack.peek().first as? IrConstructor
@@ -3864,6 +3971,32 @@ open class KotlinFileExtractor(
         }
     }
 
+    private fun extractBlock(
+        e: IrContainerExpression,
+        statements: List<IrStatement>,
+        parent: StmtExprParent,
+        callable: Label<out DbCallable>
+    ) {
+        val stmtParent = parent.stmt(e, callable)
+        extractBlock(e, statements, stmtParent.parent, stmtParent.idx, callable)
+    }
+
+    private fun extractBlock(
+        e: IrElement,
+        statements: List<IrStatement>,
+        parent: Label<out DbStmtparent>,
+        idx: Int,
+        callable: Label<out DbCallable>
+    ) {
+        val id = tw.getFreshIdLabel<DbBlock>()
+        val locId = tw.getLocation(e)
+        tw.writeStmts_block(id, parent, idx, callable)
+        tw.writeHasLocation(id, locId)
+        statements.forEachIndexed { i, s ->
+            extractStatement(s, callable, id, i)
+        }
+    }
+
     private inline fun <D: DeclarationDescriptor, reified B: IrSymbolOwner> getBoundSymbolOwner(symbol: IrBindableSymbol<D, B>, e: IrExpression): B? {
         if (symbol.isBound) {
             return symbol.owner
@@ -3972,9 +4105,11 @@ open class KotlinFileExtractor(
 
     private fun extractLoop(
         loop: IrLoop,
+        bodyIdx: Int?,
         stmtExprParent: StmtExprParent,
-        callable: Label<out DbCallable>
-    ) {
+        callable: Label<out DbCallable>,
+        getId: (Label<out DbStmtparent>, Int) -> Label<out DbStmt>
+    ) : Label<out DbStmt> {
         val stmtParent = stmtExprParent.stmt(loop, callable)
         val locId = tw.getLocation(loop)
 
@@ -3995,22 +4130,34 @@ open class KotlinFileExtractor(
             parent = stmtParent.parent
         }
 
-        val id = if (loop is IrWhileLoop) {
-            val id = tw.getFreshIdLabel<DbWhilestmt>()
-            tw.writeStmts_whilestmt(id, parent, idx, callable)
-            id
-        } else {
-            val id = tw.getFreshIdLabel<DbDostmt>()
-            tw.writeStmts_dostmt(id, parent, idx, callable)
-            id
+        val id = getId(parent, idx)
+        tw.writeHasLocation(id, locId)
+
+        val body = loop.body
+        if (body != null && bodyIdx != null) {
+            extractExpressionStmt(body, callable, id, bodyIdx)
         }
 
-        tw.writeHasLocation(id, locId)
-        extractExpressionExpr(loop.condition, callable, id, 0, id)
-        val body = loop.body
-        if (body != null) {
-            extractExpressionStmt(body, callable, id, 1)
+        return id
+    }
+
+    private fun extractLoopWithCondition(
+        loop: IrLoop,
+        stmtExprParent: StmtExprParent,
+        callable: Label<out DbCallable>
+    ) {
+        val id = extractLoop(loop, 1, stmtExprParent, callable) { parent, idx ->
+            if (loop is IrWhileLoop) {
+                tw.getFreshIdLabel<DbWhilestmt>().also {
+                    tw.writeStmts_whilestmt(it, parent, idx, callable)
+                }
+            } else {
+                tw.getFreshIdLabel<DbDostmt>().also {
+                    tw.writeStmts_dostmt(it, parent, idx, callable)
+                }
+            }
         }
+        extractExpressionExpr(loop.condition, callable, id, 0, id)
     }
 
     private fun IrValueParameter.isExtensionReceiver(): Boolean {

java/kotlin-extractor/src/main/kotlin/KotlinUsesExtractor.kt

@@ -2,6 +2,7 @@ package com.github.codeql
 
 import com.github.codeql.utils.*
 import com.github.codeql.utils.versions.codeQlWithHasQuestionMark
+import com.github.codeql.utils.versions.getKotlinType
 import com.github.codeql.utils.versions.isRawType
 import com.semmle.extractor.java.OdasaOutput
 import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
@@ -22,6 +23,7 @@ import org.jetbrains.kotlin.load.java.BuiltinMethodsWithSpecialGenericSignature
 import org.jetbrains.kotlin.load.java.JvmAbi
 import org.jetbrains.kotlin.load.java.sources.JavaSourceElement
 import org.jetbrains.kotlin.load.java.structure.*
+import org.jetbrains.kotlin.load.java.typeEnhancement.hasEnhancedNullability
 import org.jetbrains.kotlin.load.kotlin.getJvmModuleNameForDeserializedDescriptor
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.NameUtils
@@ -253,19 +255,24 @@ open class KotlinUsesExtractor(
         }
     }
 
+    private fun propertySignature(p: IrProperty) =
+        ((p.getter ?: p.setter)?.extensionReceiverParameter?.let { useType(erase(it.type)).javaResult.signature } ?: "")
+
     private fun extractPropertyLaterIfExternalFileMember(p: IrProperty) {
         if (isExternalFileClassMember(p)) {
             extractExternalClassLater(p.parentAsClass)
-            dependencyCollector?.addDependency(p, externalClassExtractor.propertySignature)
-            externalClassExtractor.extractLater(p)
+            val signature = propertySignature(p) + externalClassExtractor.propertySignature
+            dependencyCollector?.addDependency(p, signature)
+            externalClassExtractor.extractLater(p, signature)
         }
     }
 
     private fun extractFieldLaterIfExternalFileMember(f: IrField) {
         if (isExternalFileClassMember(f)) {
             extractExternalClassLater(f.parentAsClass)
-            dependencyCollector?.addDependency(f, externalClassExtractor.fieldSignature)
-            externalClassExtractor.extractLater(f)
+            val signature = (f.correspondingPropertySymbol?.let { propertySignature(it.owner) } ?: "") + externalClassExtractor.fieldSignature
+            dependencyCollector?.addDependency(f, signature)
+            externalClassExtractor.extractLater(f, signature)
         }
     }
 
@@ -669,7 +676,8 @@ open class KotlinUsesExtractor(
                           otherIsPrimitive: Boolean,
                           javaClass: IrClass,
                           kotlinPackageName: String, kotlinClassName: String): TypeResults {
-            val javaResult = if ((context == TypeContext.RETURN || (context == TypeContext.OTHER && otherIsPrimitive)) && !s.isNullable() && primitiveName != null) {
+            // Note the use of `hasEnhancedNullability` here covers cases like `@NotNull Integer`, which must be extracted as `Integer` not `int`.
+            val javaResult = if ((context == TypeContext.RETURN || (context == TypeContext.OTHER && otherIsPrimitive)) && !s.isNullable() && getKotlinType(s)?.hasEnhancedNullability() != true && primitiveName != null) {
                     val label: Label<DbPrimitive> = tw.getLabelFor("@\"type;$primitiveName\"", {
                         tw.writePrimitives(it, primitiveName)
                     })
@@ -813,7 +821,7 @@ open class KotlinUsesExtractor(
                 OperatorNameConventions.INVOKE.asString())
 
         fun getSuffixIfInternal() =
-            if (f.visibility == DescriptorVisibilities.INTERNAL) {
+            if (f.visibility == DescriptorVisibilities.INTERNAL && f !is IrConstructor) {
                 "\$" + getJvmModuleName(f)
             } else {
                 ""
@@ -952,27 +960,43 @@ open class KotlinUsesExtractor(
             ((t as? IrSimpleType)?.classOrNull?.owner?.isFinalClass) != true
         }
 
-    private fun wildcardAdditionAllowed(v: Variance, t: IrType, addByDefault: Boolean) =
+    private fun wildcardAdditionAllowed(v: Variance, t: IrType, addByDefault: Boolean, javaVariance: Variance?) =
         when {
             t.hasAnnotation(jvmWildcardAnnotation) -> true
             !addByDefault -> false
-            t.hasAnnotation(jvmWildcardSuppressionAnnotation) -> false
+            // If a Java declaration specifies a variance, introduce it even if it's pointless (e.g. ? extends FinalClass, or ? super Object)
+            javaVariance == v -> true
             v == Variance.IN_VARIANCE -> !(t.isNullableAny() || t.isAny())
             v == Variance.OUT_VARIANCE -> extendsAdditionAllowed(t)
             else -> false
         }
 
+    // Returns true if `t` has `@JvmSuppressWildcards` or `@JvmSuppressWildcards(true)`,
+    // false if it has `@JvmSuppressWildcards(false)`,
+    // and null if the annotation is not present.
+    @Suppress("UNCHECKED_CAST")
+    private fun getWildcardSuppressionDirective(t: IrAnnotationContainer) =
+        t.getAnnotation(jvmWildcardSuppressionAnnotation)?.let { (it.getValueArgument(0) as? IrConst<Boolean>)?.value ?: true }
+
     private fun addJavaLoweringArgumentWildcards(p: IrTypeParameter, t: IrTypeArgument, addByDefault: Boolean, javaType: JavaType?): IrTypeArgument =
         (t as? IrTypeProjection)?.let {
-            val newBase = addJavaLoweringWildcards(it.type, addByDefault, javaType)
+            val newAddByDefault = getWildcardSuppressionDirective(it.type)?.not() ?: addByDefault
+            val newBase = addJavaLoweringWildcards(it.type, newAddByDefault, javaType)
+            // Note javaVariance == null means we don't have a Java type to conform to -- for example if this is a Kotlin source definition.
+            val javaVariance = javaType?.let { jType ->
+                when (jType) {
+                    is JavaWildcardType -> if (jType.isExtends) Variance.OUT_VARIANCE else Variance.IN_VARIANCE
+                    else -> Variance.INVARIANT
+                }
+            }
             val newVariance =
                 if (it.variance == Variance.INVARIANT &&
                     p.variance != Variance.INVARIANT &&
                     // The next line forbids inferring a wildcard type when we have a corresponding Java type with conflicting variance.
                     // For example, Java might declare f(Comparable<CharSequence> cs), in which case we shouldn't add a `? super ...`
                     // wildcard. Note if javaType is unknown (e.g. this is a Kotlin source element), we assume wildcards should be added.
-                    (javaType?.let { jt -> jt is JavaWildcardType && jt.isExtends == (p.variance == Variance.OUT_VARIANCE) } != false) &&
-                    wildcardAdditionAllowed(p.variance, it.type, addByDefault))
+                    (javaVariance == null || javaVariance == p.variance) &&
+                    wildcardAdditionAllowed(p.variance, it.type, newAddByDefault, javaVariance))
                     p.variance
                 else
                     it.variance
@@ -991,12 +1015,13 @@ open class KotlinUsesExtractor(
 
     fun addJavaLoweringWildcards(t: IrType, addByDefault: Boolean, javaType: JavaType?): IrType =
         (t as? IrSimpleType)?.let {
+            val newAddByDefault = getWildcardSuppressionDirective(t)?.not() ?: addByDefault
             val typeParams = it.classOrNull?.owner?.typeParameters ?: return t
             val newArgs = typeParams.zip(it.arguments).mapIndexed { idx, pair ->
                 addJavaLoweringArgumentWildcards(
                     pair.first,
                     pair.second,
-                    addByDefault,
+                    newAddByDefault,
                     javaType?.let { jt -> getJavaTypeArgument(jt, idx) }
                 )
             }
@@ -1044,7 +1069,7 @@ open class KotlinUsesExtractor(
             classTypeArgsIncludingOuterClasses,
             overridesCollectionsMethodWithAlteredParameterTypes(f),
             getJavaCallable(f),
-            !hasWildcardSuppressionAnnotation(f)
+            !getInnermostWildcardSupppressionAnnotation(f)
         )
 
     /*
@@ -1203,10 +1228,11 @@ open class KotlinUsesExtractor(
         else -> null
     }
 
-    fun hasWildcardSuppressionAnnotation(d: IrDeclaration) =
-        d.hasAnnotation(jvmWildcardSuppressionAnnotation) ||
+    fun getInnermostWildcardSupppressionAnnotation(d: IrDeclaration) =
+        getWildcardSuppressionDirective(d) ?:
         // Note not using `parentsWithSelf` as that only works if `d` is an IrDeclarationParent
-        d.parents.any { (it as? IrAnnotationContainer)?.hasAnnotation(jvmWildcardSuppressionAnnotation) == true }
+        d.parents.filterIsInstance<IrAnnotationContainer>().mapNotNull { getWildcardSuppressionDirective(it) }.firstOrNull() ?:
+        false
 
     /**
      * Class to hold labels for generated classes around local functions, lambdas, function references, and property references.
@@ -1273,6 +1299,7 @@ open class KotlinUsesExtractor(
                         }
                         // Look for an exact type match...
                         javaClass.declarations.findSubType<IrFunction> { decl ->
+                            !decl.isFakeOverride &&
                             decl.name.asString() == jvmName &&
                             decl.valueParameters.size == f.valueParameters.size &&
                             decl.valueParameters.zip(f.valueParameters).all { p -> erase(p.first.type).classifierOrNull == erase(p.second.type).classifierOrNull }

java/kotlin-extractor/src/main/kotlin/LinesOfCode.kt

@@ -1,9 +1,10 @@
 package com.github.codeql
 
-import com.github.codeql.utils.versions.Psi2Ir
+import com.github.codeql.utils.versions.getPsi2Ir
 import com.intellij.psi.PsiComment
 import com.intellij.psi.PsiElement
 import com.intellij.psi.PsiWhiteSpace
+import org.jetbrains.kotlin.config.KotlinCompilerVersion
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.kdoc.psi.api.KDocElement
@@ -15,9 +16,16 @@ class LinesOfCode(
     val tw: FileTrapWriter,
     val file: IrFile
 ) {
-    val psi2Ir = Psi2Ir(logger)
+    val psi2Ir = getPsi2Ir(logger).also {
+        if (it == null) {
+            logger.warn("Lines of code will not be populated as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})")
+        }
+    }
 
     fun linesOfCodeInFile(id: Label<DbFile>) {
+        if (psi2Ir == null) {
+            return
+        }
         val ktFile = psi2Ir.getKtFile(file)
         if (ktFile == null) {
             return
@@ -26,6 +34,9 @@ class LinesOfCode(
     }
 
     fun linesOfCodeInDeclaration(d: IrDeclaration, id: Label<out DbSourceline>) {
+        if (psi2Ir == null) {
+            return
+        }
         val p = psi2Ir.findPsiElement(d, file)
         if (p == null) {
             return

java/kotlin-extractor/src/main/kotlin/TrapWriter.kt

@@ -1,7 +1,6 @@
 package com.github.codeql
 
 import com.github.codeql.KotlinUsesExtractor.LocallyVisibleFunctionLabels
-import com.github.codeql.utils.versions.FileEntry
 import java.io.BufferedWriter
 import java.io.File
 import org.jetbrains.kotlin.ir.IrElement
@@ -15,6 +14,7 @@ import org.jetbrains.kotlin.ir.util.SYNTHETIC_OFFSET
 
 import com.semmle.extractor.java.PopulateFile
 import com.semmle.util.unicode.UTF8Util
+import org.jetbrains.kotlin.ir.expressions.IrCall
 
 /**
  * Each `.trap` file has a `TrapLabelManager` while we are writing it.
@@ -269,11 +269,42 @@ open class FileTrapWriter (
      */
     val fileId = mkFileId(filePath, populateFileTables)
 
+    private fun offsetMinOf(default: Int, vararg options: Int?): Int {
+        if (default == UNDEFINED_OFFSET || default == SYNTHETIC_OFFSET) {
+            return default
+        }
+
+        var currentMin = default
+        for (option in options) {
+            if (option != null && option != UNDEFINED_OFFSET && option != SYNTHETIC_OFFSET && option < currentMin) {
+                currentMin = option
+            }
+        }
+
+        return currentMin
+    }
+
+    private fun getStartOffset(e: IrElement): Int {
+        return when (e) {
+            is IrCall -> {
+                // Calls have incorrect startOffset, so we adjust them:
+                val dr = e.dispatchReceiver?.let { getStartOffset(it) }
+                val er = e.extensionReceiver?.let { getStartOffset(it) }
+                offsetMinOf(e.startOffset, dr, er)
+            }
+            else -> e.startOffset
+        }
+    }
+
+    private fun getEndOffset(e: IrElement): Int {
+        return e.endOffset
+    }
+
     /**
      * Gets a label for the location of `e`.
      */
     fun getLocation(e: IrElement): Label<DbLocation> {
-        return getLocation(e.startOffset, e.endOffset)
+        return getLocation(getStartOffset(e), getEndOffset(e))
     }
     /**
      * Gets a label for the location corresponding to `startOffset` and

java/kotlin-extractor/src/main/kotlin/comments/CommentExtractor.kt

@@ -3,9 +3,11 @@ package com.github.codeql.comments
 import com.github.codeql.*
 import com.github.codeql.utils.IrVisitorLookup
 import com.github.codeql.utils.isLocalFunction
-import com.github.codeql.utils.versions.Psi2Ir
+import com.github.codeql.utils.versions.getPsi2Ir
+import com.github.codeql.utils.versions.Psi2IrFacade
 import com.intellij.psi.PsiComment
 import com.intellij.psi.PsiElement
+import org.jetbrains.kotlin.config.KotlinCompilerVersion
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.ir.expressions.IrBody
@@ -21,18 +23,23 @@ import org.jetbrains.kotlin.psi.psiUtil.startOffset
 class CommentExtractor(private val fileExtractor: KotlinFileExtractor, private val file: IrFile, private val fileLabel: Label<out DbFile>) {
     private val tw = fileExtractor.tw
     private val logger = fileExtractor.logger
-    private val psi2Ir = Psi2Ir(logger)
-    private val ktFile = psi2Ir.getKtFile(file)
 
     fun extract() {
+        val psi2Ir = getPsi2Ir(logger)
+        if (psi2Ir == null) {
+            logger.warn("Comments will not be extracted as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})")
+            return
+        }
+        val ktFile = psi2Ir.getKtFile(file)
         if (ktFile == null) {
             logger.warn("Comments are not being processed in ${file.path}.")
-        } else {
-            ktFile.accept(commentVisitor)
+            return
         }
+        val commentVisitor = mkCommentVisitor(psi2Ir)
+        ktFile.accept(commentVisitor)
     }
 
-    private val commentVisitor =
+    private fun mkCommentVisitor(psi2Ir: Psi2IrFacade): KtVisitor<Unit, Unit> =
         object : KtVisitor<Unit, Unit>() {
             override fun visitElement(element: PsiElement) {
                 element.acceptChildren(this)

java/kotlin-extractor/src/main/kotlin/utils/IrVisitorLookup.kt

@@ -1,6 +1,6 @@
 package com.github.codeql.utils
 
-import com.github.codeql.utils.versions.Psi2Ir
+import com.github.codeql.utils.versions.Psi2IrFacade
 import com.intellij.psi.PsiElement
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.IrDeclaration
@@ -8,7 +8,7 @@ import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.ir.util.isFakeOverride
 import org.jetbrains.kotlin.ir.visitors.IrElementVisitor
 
-class IrVisitorLookup(private val psi2Ir: Psi2Ir, private val psi: PsiElement, private val file: IrFile) :
+class IrVisitorLookup(private val psi2Ir: Psi2IrFacade, private val psi: PsiElement, private val file: IrFile) :
     IrElementVisitor<Unit, MutableCollection<IrElement>> {
     private val location = psi.getLocation()
 

java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_4_32/Psi2Ir.kt

@@ -1,19 +1,5 @@
 package com.github.codeql.utils.versions
 
 import com.github.codeql.FileLogger
-import com.intellij.psi.PsiElement
-import org.jetbrains.kotlin.ir.IrElement
-import org.jetbrains.kotlin.ir.declarations.IrFile
-import org.jetbrains.kotlin.psi.KtFile
 
-class Psi2Ir(private val logger: FileLogger) : Psi2IrFacade {
-    override fun getKtFile(irFile: IrFile): KtFile? {
-        logger.warn("Comment extraction is not supported for Kotlin < 1.5.20")
-        return null
-    }
-
-    override fun findPsiElement(irElement: IrElement, irFile: IrFile): PsiElement? {
-        logger.error("Attempted comment extraction for Kotlin < 1.5.20")
-        return null
-    }
-}
+fun getPsi2Ir(@Suppress("UNUSED_PARAMETER") logger: FileLogger): Psi2IrFacade? = null

java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_4_32/getKotlinType.kt

@@ -0,0 +1,6 @@
+package com.github.codeql.utils.versions
+
+import org.jetbrains.kotlin.ir.types.IrSimpleType
+import org.jetbrains.kotlin.ir.types.impl.IrTypeBase
+
+fun getKotlinType(s: IrSimpleType) = (s as? IrTypeBase)?.kotlinType

java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_5_20/Psi2Ir.kt

@@ -8,7 +8,9 @@ import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.psi.KtFile
 
-class Psi2Ir(private val logger: FileLogger): Psi2IrFacade {
+fun getPsi2Ir(logger: FileLogger): Psi2IrFacade? = Psi2Ir(logger)
+
+private class Psi2Ir(private val logger: FileLogger): Psi2IrFacade {
     override fun getKtFile(irFile: IrFile): KtFile? {
         return irFile.getKtFile()
     }

java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_7_0/getKotlinType.kt

@@ -0,0 +1,5 @@
+package com.github.codeql.utils.versions
+
+import org.jetbrains.kotlin.ir.types.IrSimpleType
+
+fun getKotlinType(s: IrSimpleType) = s.kotlinType

java/old-change-notes/2021-02-23-deprecated-jcenter-bintray.md

@@ -1,4 +1,4 @@
 lgtm,codescanning
 * A new query "Depending upon JCenter/Bintray as an artifact repository" 
   (`java/maven/dependency-upon-bintray`) has been added.
-  This query finds uses of the deprecated JCenter/Bintray artifact respositories.
+  This query finds uses of the deprecated JCenter/Bintray artifact repositories.

java/ql/consistency-queries/diags.ql

@@ -0,0 +1,34 @@
+import semmle.code.java.Diagnostics
+
+/*
+ * This query fails if any unexpected diagnostics are recorded in the
+ * database. By putting
+ *    // Diagnostic Matches: PAT
+ * in any source files, you can declare that diagnostics matching PAT
+ * (in the string.matches(string) sense) are expected.
+ */
+
+class DiagnosticException extends Top {
+  string pattern;
+
+  DiagnosticException() {
+    this.(KtComment).getText() = "// Diagnostic Matches: " + pattern
+    or
+    this.(Javadoc).toString() = "// Diagnostic Matches: " + pattern
+  }
+
+  Diagnostic getException() { diagnosticMessage(result).matches(pattern) }
+}
+
+string diagnosticMessage(Diagnostic d) {
+  if d.getFullMessage() != "" then result = d.getFullMessage() else result = d.getMessage()
+}
+
+// Check that there aren't any old DiagnosticExceptions left after
+// something is fixed.
+query predicate unusedDiagnosticException(DiagnosticException de) { not exists(de.getException()) }
+
+query predicate unexpectedDiagnostic(Diagnostic d, string s) {
+  s = diagnosticMessage(d) and
+  not d = any(DiagnosticException de).getException()
+}

java/ql/consistency-queries/javaEquivalent.ql

@@ -1,6 +0,0 @@
-import java
-import semmle.code.java.Diagnostics
-
-from Diagnostic d
-where exists(d.getMessage().indexOf("Couldn't find a Java equivalent function to "))
-select d

java/ql/consistency-queries/qlpack.yml

@@ -1,4 +1,4 @@
 name: codeql-java-consistency-queries
 version: 0.0.0
-libraryPathDependencies:
-  - codeql-java
+dependencies:
+  codeql/java-all: '*'