use ATM model from training run classification_1665164532_df3f5e7d

New atm features rebased

cpp/ql/lib/CHANGELOG.md

@@ -1,18 +1,3 @@
-## 0.4.0
-
-### Deprecated APIs
-
-* Some classes/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
-  The old name still exists as a deprecated alias.
-
-### New Features
-
-* Added subclasses of `BuiltInOperations` for `__is_same`, `__is_function`, `__is_layout_compatible`, `__is_pointer_interconvertible_base_of`, `__is_array`, `__array_rank`, `__array_extent`, `__is_arithmetic`, `__is_complete_type`, `__is_compound`, `__is_const`, `__is_floating_point`, `__is_fundamental`, `__is_integral`, `__is_lvalue_reference`, `__is_member_function_pointer`, `__is_member_object_pointer`, `__is_member_pointer`, `__is_object`, `__is_pointer`, `__is_reference`, `__is_rvalue_reference`, `__is_scalar`, `__is_signed`, `__is_unsigned`, `__is_void`, and `__is_volatile`.
-
-### Bug Fixes
-
-* Fixed an issue in the taint tracking analysis where implicit reads were not allowed by default in sinks or additional taint steps that used flow states.
-
 ## 0.3.5
 
 ## 0.3.4

cpp/ql/lib/change-notes/2022-09-06-additional-builtin-support.md

@@ -1,14 +1,4 @@
-## 0.4.0
-
-### Deprecated APIs
-
-* Some classes/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
-  The old name still exists as a deprecated alias.
-
-### New Features
-
+---
+category: feature
+---
 * Added subclasses of `BuiltInOperations` for `__is_same`, `__is_function`, `__is_layout_compatible`, `__is_pointer_interconvertible_base_of`, `__is_array`, `__array_rank`, `__array_extent`, `__is_arithmetic`, `__is_complete_type`, `__is_compound`, `__is_const`, `__is_floating_point`, `__is_fundamental`, `__is_integral`, `__is_lvalue_reference`, `__is_member_function_pointer`, `__is_member_object_pointer`, `__is_member_pointer`, `__is_object`, `__is_pointer`, `__is_reference`, `__is_rvalue_reference`, `__is_scalar`, `__is_signed`, `__is_unsigned`, `__is_void`, and `__is_volatile`.
-
-### Bug Fixes
-
-* Fixed an issue in the taint tracking analysis where implicit reads were not allowed by default in sinks or additional taint steps that used flow states.

cpp/ql/lib/change-notes/2022-09-08-implicit-read-flowstates.md

@@ -0,0 +1,4 @@
+---
+category: fix
+---
+* Fixed an issue in the taint tracking analysis where implicit reads were not allowed by default in sinks or additional taint steps that used flow states.

cpp/ql/lib/change-notes/2022-09-12-uppercase.md

@@ -1,6 +1,5 @@
-## 0.3.0
-
-### Deprecated APIs
-
+---
+category: deprecated
+---
 * Some classes/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
-  The old name still exists as a deprecated alias.
+  The old name still exists as a deprecated alias.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.4.0
+lastReleaseVersion: 0.3.5

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll

@@ -709,7 +709,8 @@ private module Cached {
      */
     pragma[nomagic]
     private DataFlowCallable viableImplInCallContextExt(DataFlowCall call, DataFlowCall ctx) {
-      result = viableImplInCallContext(call, ctx)
+      result = viableImplInCallContext(call, ctx) and
+      result = viableCallable(call)
       or
       result = viableCallableLambda(call, TDataFlowCallSome(ctx))
       or

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll

@@ -38,6 +38,13 @@ module Consistency {
 
     /** Holds if `n` should be excluded from the consistency test `uniquePostUpdate`. */
     predicate uniquePostUpdateExclude(Node n) { none() }
+
+    /** Holds if `(call, ctx)` should be excluded from the consistency test `viableImplInCallContextTooLargeExclude`. */
+    predicate viableImplInCallContextTooLargeExclude(
+      DataFlowCall call, DataFlowCall ctx, DataFlowCallable callable
+    ) {
+      none()
+    }
   }
 
   private class RelevantNode extends Node {
@@ -217,4 +224,12 @@ module Consistency {
     not any(ConsistencyConfiguration c).postWithInFlowExclude(n) and
     msg = "PostUpdateNode should not be the target of local flow."
   }
+
+  query predicate viableImplInCallContextTooLarge(
+    DataFlowCall call, DataFlowCall ctx, DataFlowCallable callable
+  ) {
+    callable = viableImplInCallContext(call, ctx) and
+    not callable = viableCallable(call) and
+    not any(ConsistencyConfiguration c).viableImplInCallContextTooLargeExclude(call, ctx, callable)
+  }
 }

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.4.0
+version: 0.4.0-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll

@@ -709,7 +709,8 @@ private module Cached {
      */
     pragma[nomagic]
     private DataFlowCallable viableImplInCallContextExt(DataFlowCall call, DataFlowCall ctx) {
-      result = viableImplInCallContext(call, ctx)
+      result = viableImplInCallContext(call, ctx) and
+      result = viableCallable(call)
       or
       result = viableCallableLambda(call, TDataFlowCallSome(ctx))
       or

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplConsistency.qll

@@ -38,6 +38,13 @@ module Consistency {
 
     /** Holds if `n` should be excluded from the consistency test `uniquePostUpdate`. */
     predicate uniquePostUpdateExclude(Node n) { none() }
+
+    /** Holds if `(call, ctx)` should be excluded from the consistency test `viableImplInCallContextTooLargeExclude`. */
+    predicate viableImplInCallContextTooLargeExclude(
+      DataFlowCall call, DataFlowCall ctx, DataFlowCallable callable
+    ) {
+      none()
+    }
   }
 
   private class RelevantNode extends Node {
@@ -217,4 +224,12 @@ module Consistency {
     not any(ConsistencyConfiguration c).postWithInFlowExclude(n) and
     msg = "PostUpdateNode should not be the target of local flow."
   }
+
+  query predicate viableImplInCallContextTooLarge(
+    DataFlowCall call, DataFlowCall ctx, DataFlowCallable callable
+  ) {
+    callable = viableImplInCallContext(call, ctx) and
+    not callable = viableCallable(call) and
+    not any(ConsistencyConfiguration c).viableImplInCallContextTooLargeExclude(call, ctx, callable)
+  }
 }

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll

@@ -709,7 +709,8 @@ private module Cached {
      */
     pragma[nomagic]
     private DataFlowCallable viableImplInCallContextExt(DataFlowCall call, DataFlowCall ctx) {
-      result = viableImplInCallContext(call, ctx)
+      result = viableImplInCallContext(call, ctx) and
+      result = viableCallable(call)
       or
       result = viableCallableLambda(call, TDataFlowCallSome(ctx))
       or

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll

@@ -38,6 +38,13 @@ module Consistency {
 
     /** Holds if `n` should be excluded from the consistency test `uniquePostUpdate`. */
     predicate uniquePostUpdateExclude(Node n) { none() }
+
+    /** Holds if `(call, ctx)` should be excluded from the consistency test `viableImplInCallContextTooLargeExclude`. */
+    predicate viableImplInCallContextTooLargeExclude(
+      DataFlowCall call, DataFlowCall ctx, DataFlowCallable callable
+    ) {
+      none()
+    }
   }
 
   private class RelevantNode extends Node {
@@ -217,4 +224,12 @@ module Consistency {
     not any(ConsistencyConfiguration c).postWithInFlowExclude(n) and
     msg = "PostUpdateNode should not be the target of local flow."
   }
+
+  query predicate viableImplInCallContextTooLarge(
+    DataFlowCall call, DataFlowCall ctx, DataFlowCallable callable
+  ) {
+    callable = viableImplInCallContext(call, ctx) and
+    not callable = viableCallable(call) and
+    not any(ConsistencyConfiguration c).viableImplInCallContextTooLargeExclude(call, ctx, callable)
+  }
 }

cpp/ql/src/CHANGELOG.md

@@ -1,14 +1,3 @@
-## 0.4.0
-
-### New Queries
-
-* Added a new medium-precision query, `cpp/missing-check-scanf`, which detects `scanf` output variables that are used without a proper return-value check to see that they were actually written. A variation of this query was originally contributed as an [experimental query by @ihsinme](https://github.com/github/codeql/pull/8246).
-
-### Minor Analysis Improvements
-
-* Modernizations from "Cleartext storage of sensitive information in buffer" (`cpp/cleartext-storage-buffer`) have been ported to the "Cleartext storage of sensitive information in file" (`cpp/cleartext-storage-file`), "Cleartext transmission of sensitive information" (`cpp/cleartext-transmission`) and "Cleartext storage of sensitive information in an SQLite database" (`cpp/cleartext-storage-database`) queries. These changes may result in more correct results and fewer false positive results from these queries.
-* The alert message of many queries have been changed to make the message consistent with other languages.
-
 ## 0.3.4
 
 ## 0.3.3

cpp/ql/src/change-notes/2022-08-23-alert-messages.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The alert message of many queries have been changed to make the message consistent with other languages.

cpp/ql/src/change-notes/2022-08-24-missing-check-scanf.md

@@ -0,0 +1,4 @@
+---
+category: newQuery
+---
+* Added a new medium-precision query, `cpp/missing-check-scanf`, which detects `scanf` output variables that are used without a proper return-value check to see that they were actually written. A variation of this query was originally contributed as an [experimental query by @ihsinme](https://github.com/github/codeql/pull/8246).

cpp/ql/src/change-notes/2022-09-05-cleartext-queries.md

@@ -1,10 +1,4 @@
-## 0.4.0
-
-### New Queries
-
-* Added a new medium-precision query, `cpp/missing-check-scanf`, which detects `scanf` output variables that are used without a proper return-value check to see that they were actually written. A variation of this query was originally contributed as an [experimental query by @ihsinme](https://github.com/github/codeql/pull/8246).
-
-### Minor Analysis Improvements
-
+---
+category: minorAnalysis
+---
 * Modernizations from "Cleartext storage of sensitive information in buffer" (`cpp/cleartext-storage-buffer`) have been ported to the "Cleartext storage of sensitive information in file" (`cpp/cleartext-storage-file`), "Cleartext transmission of sensitive information" (`cpp/cleartext-transmission`) and "Cleartext storage of sensitive information in an SQLite database" (`cpp/cleartext-storage-database`) queries. These changes may result in more correct results and fewer false positive results from these queries.
-* The alert message of many queries have been changed to make the message consistent with other languages.

cpp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.4.0
+lastReleaseVersion: 0.3.4

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 0.4.0
+version: 0.4.0-dev
 groups: 
   - cpp
   - queries

cpp/ql/test/library-tests/dataflow/dataflow-tests/dataflow-consistency.expected

@@ -87,3 +87,4 @@ postWithInFlow
 | test.cpp:465:3:465:4 | * ... [post update] | PostUpdateNode should not be the target of local flow. |
 | test.cpp:465:4:465:4 | p [inner post update] | PostUpdateNode should not be the target of local flow. |
 | test.cpp:470:22:470:22 | x [inner post update] | PostUpdateNode should not be the target of local flow. |
+viableImplInCallContextTooLarge

cpp/ql/test/library-tests/dataflow/dataflow-tests/dataflow-ir-consistency.expected

@@ -627,3 +627,4 @@ postWithInFlow
 | true_upon_entry.cpp:98:7:98:7 | VariableAddress [post update] | PostUpdateNode should not be the target of local flow. |
 | true_upon_entry.cpp:101:18:101:18 | VariableAddress [post update] | PostUpdateNode should not be the target of local flow. |
 | true_upon_entry.cpp:102:5:102:5 | x [post update] | PostUpdateNode should not be the target of local flow. |
+viableImplInCallContextTooLarge

cpp/ql/test/library-tests/dataflow/fields/dataflow-consistency.expected

@@ -155,3 +155,4 @@ postWithInFlow
 | simple.cpp:92:7:92:7 | i [post update] | PostUpdateNode should not be the target of local flow. |
 | struct_init.c:24:11:24:12 | ab [inner post update] | PostUpdateNode should not be the target of local flow. |
 | struct_init.c:36:17:36:24 | nestedAB [inner post update] | PostUpdateNode should not be the target of local flow. |
+viableImplInCallContextTooLarge

cpp/ql/test/library-tests/dataflow/fields/dataflow-ir-consistency.expected

@@ -1323,3 +1323,4 @@ postWithInFlow
 | struct_init.c:46:16:46:24 | FieldAddress [post update] | PostUpdateNode should not be the target of local flow. |
 | struct_init.c:46:16:46:24 | pointerAB [post update] | PostUpdateNode should not be the target of local flow. |
 | struct_init.c:46:16:46:24 | pointerAB [post update] | PostUpdateNode should not be the target of local flow. |
+viableImplInCallContextTooLarge

cpp/ql/test/library-tests/syntax-zoo/dataflow-consistency.expected

@@ -124,3 +124,4 @@ postWithInFlow
 | static_init_templates.cpp:3:2:3:4 | ref [post update] | PostUpdateNode should not be the target of local flow. |
 | static_init_templates.cpp:21:2:21:4 | val [post update] | PostUpdateNode should not be the target of local flow. |
 | try_catch.cpp:7:8:7:8 | call to exception | PostUpdateNode should not be the target of local flow. |
+viableImplInCallContextTooLarge

cpp/ql/test/library-tests/syntax-zoo/dataflow-ir-consistency.expected

@@ -2710,3 +2710,4 @@ postWithInFlow
 | whilestmt.c:11:5:11:8 | done [post update] | PostUpdateNode should not be the target of local flow. |
 | whilestmt.c:40:7:40:7 | VariableAddress [post update] | PostUpdateNode should not be the target of local flow. |
 | whilestmt.c:42:7:42:7 | VariableAddress [post update] | PostUpdateNode should not be the target of local flow. |
+viableImplInCallContextTooLarge

csharp/ql/campaigns/Solorigate/lib/CHANGELOG.md

@@ -1,7 +1,3 @@
-## 1.3.0
-
-No user-facing changes.
-
 ## 1.2.5
 
 ## 1.2.4

csharp/ql/campaigns/Solorigate/lib/change-notes/released/1.3.0.md

@@ -1,3 +0,0 @@
-## 1.3.0
-
-No user-facing changes.

csharp/ql/campaigns/Solorigate/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.3.0
+lastReleaseVersion: 1.2.5

csharp/ql/campaigns/Solorigate/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-all
-version: 1.3.0
+version: 1.3.0-dev
 groups:
     - csharp
     - solorigate

csharp/ql/campaigns/Solorigate/src/CHANGELOG.md

@@ -1,7 +1,3 @@
-## 1.3.0
-
-No user-facing changes.
-
 ## 1.2.5
 
 ## 1.2.4

csharp/ql/campaigns/Solorigate/src/change-notes/released/1.3.0.md

@@ -1,3 +0,0 @@
-## 1.3.0
-
-No user-facing changes.

csharp/ql/campaigns/Solorigate/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.3.0
+lastReleaseVersion: 1.2.5

csharp/ql/campaigns/Solorigate/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-queries
-version: 1.3.0
+version: 1.3.0-dev
 groups:
     - csharp
     - solorigate

csharp/ql/integration-tests/all-platforms/dotnet_run/Program.cs

@@ -1 +1 @@
-Console.WriteLine(args.Length > 0 ? args[0] + ", " + args[1] : "");
+Console.WriteLine(args.Length > 1 ? args[0] + ", " + args[1] : "Default reply");

csharp/ql/integration-tests/all-platforms/dotnet_run/dotnet_run.csproj

@@ -7,4 +7,8 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+ <Target Name="DeleteBinObjFolders" BeforeTargets="Clean">
+   <RemoveDir Directories=".\bin" />
+   <RemoveDir Directories=".\obj" />
+ </Target>
 </Project>

csharp/ql/integration-tests/all-platforms/dotnet_run/test.py

@@ -1,17 +1,43 @@
+import os
 from create_database_utils import *
 
+def run_codeql_database_create_stdout(args, dbname):
+    stdout = open(dbname + "file.txt", 'w+')
+    run_codeql_database_create(args, test_db=dbname, db=None, stdout=stdout, lang="csharp")
+    stdout.seek(0)
+    s = stdout.read()
+    stdout.close()
+    return s
+
+def check_build_out(msg, s):
+    if "[build-stdout] " + msg not in s:
+        raise Exception("The C# extractor did not interpret the 'dotnet run' command correctly")
+
 # no arguments
-run_codeql_database_create(['dotnet run'], test_db="test-db", db=None, lang="csharp")
+s = run_codeql_database_create_stdout(['dotnet run'], "test-db")
+check_build_out("Default reply", s)
 
 # no arguments, but `--`
-run_codeql_database_create(['dotnet clean', 'rm -rf test-db obj bin', 'dotnet run --'], test_db="test2-db", db=None, lang="csharp")
+s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test-db', 'dotnet run --'], "test2-db")
+check_build_out("Default reply", s)
+
+# one argument, no `--`
+s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test2-db', 'dotnet run hello'], "test3-db")
+check_build_out("Default reply", s)
+
+# one argument, but `--`
+s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test3-db', 'dotnet run -- hello'], "test4-db")
+check_build_out("Default reply", s)
 
 # two arguments, no `--`
-run_codeql_database_create(['dotnet clean', 'rm -rf test2-db obj bin', 'dotnet run hello world'], test_db="test3-db", db=None, lang="csharp")
+s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test4-db', 'dotnet run hello world'], "test5-db")
+check_build_out("hello, world", s)
 
 # two arguments, and `--`
-run_codeql_database_create(['dotnet clean', 'rm -rf test3-db obj bin', 'dotnet run -- hello world'], test_db="test4-db", db=None, lang="csharp")
+s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test5-db', 'dotnet run -- hello world'], "test6-db")
+check_build_out("hello, world", s)
 
 # shared compilation enabled; tracer should override by changing the command
 # to `dotnet run -p:UseSharedCompilation=true -p:UseSharedCompilation=false -- hello world`
-run_codeql_database_create(['dotnet clean', 'rm -rf test4-db obj bin', 'dotnet run -p:UseSharedCompilation=true -- hello world'], test_db="test5-db", db=None, lang="csharp")
+s = run_codeql_database_create_stdout(['dotnet clean', 'rm -rf test6-db', 'dotnet run -p:UseSharedCompilation=true -- hello world'], "test7-db")
+check_build_out("hello, world", s)

csharp/ql/lib/CHANGELOG.md

@@ -1,14 +1,3 @@
-## 0.4.0
-
-### Deprecated APIs
-
-* Some classes/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
-  The old name still exists as a deprecated alias.
-
-### Bug Fixes
-
-* Fixed an issue in the taint tracking analysis where implicit reads were not allowed by default in sinks or additional taint steps that used flow states.
-
 ## 0.3.5
 
 ## 0.3.4

csharp/ql/lib/change-notes/2022-09-08-implicit-read-flowstates.md

@@ -0,0 +1,4 @@
+---
+category: fix
+---
+* Fixed an issue in the taint tracking analysis where implicit reads were not allowed by default in sinks or additional taint steps that used flow states.

csharp/ql/lib/change-notes/2022-09-12-uppercase.md

@@ -0,0 +1,5 @@
+---
+category: deprecated
+---
+* Some classes/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
+  The old name still exists as a deprecated alias.

csharp/ql/lib/change-notes/released/0.4.0.md

@@ -1,10 +0,0 @@
-## 0.4.0
-
-### Deprecated APIs
-
-* Some classes/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
-  The old name still exists as a deprecated alias.
-
-### Bug Fixes
-
-* Fixed an issue in the taint tracking analysis where implicit reads were not allowed by default in sinks or additional taint steps that used flow states.

csharp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.4.0
+lastReleaseVersion: 0.3.5

csharp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-all
-version: 0.4.0
+version: 0.4.0-dev
 groups: csharp
 dbscheme: semmlecode.csharp.dbscheme
 extractor: csharp

csharp/ql/lib/semmle/code/csharp/dataflow/ExternalFlow.qll

@@ -504,16 +504,36 @@ class UnboundCallable extends Callable {
   }
 }
 
+pragma[nomagic]
+private predicate callableSpecInfo(Callable c, string namespace, string type, string name) {
+  c.getDeclaringType().hasQualifiedName(namespace, type) and
+  c.getName() = name
+}
+
+pragma[nomagic]
+private predicate subtypeSpecCandidate(string name, UnboundValueOrRefType t) {
+  exists(UnboundValueOrRefType t0 |
+    elementSpec(_, _, true, name, _, _, t0) and
+    t = t0.getASubTypeUnbound+()
+  )
+}
+
+pragma[nomagic]
+private predicate callableInfo(Callable c, string name, UnboundValueOrRefType decl) {
+  name = c.getName() and
+  decl = c.getDeclaringType()
+}
+
 private class InterpretedCallable extends Callable {
   InterpretedCallable() {
-    exists(UnboundValueOrRefType t, boolean subtypes, string name |
-      elementSpec(_, _, subtypes, name, _, _, t) and
-      this.hasName(name)
-    |
-      this.getDeclaringType() = t
-      or
-      subtypes = true and
-      this.getDeclaringType() = t.getASubTypeUnbound+()
+    exists(string namespace, string type, string name |
+      callableSpecInfo(this, namespace, type, name) and
+      elementSpec(namespace, type, _, name, _, _)
+    )
+    or
+    exists(string name, UnboundValueOrRefType t |
+      callableInfo(this, name, t) and
+      subtypeSpecCandidate(name, t)
     )
   }
 }

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowDispatch.qll

@@ -175,10 +175,19 @@ private module DispatchImpl {
    * restricted to those `call`s for which a context might make a difference.
    */
   DataFlowCallable viableImplInCallContext(NonDelegateDataFlowCall call, DataFlowCall ctx) {
-    result.getUnderlyingCallable() =
-      call.getDispatchCall()
-          .getADynamicTargetInCallContext(ctx.(NonDelegateDataFlowCall).getDispatchCall())
-          .getUnboundDeclaration()
+    exists(DispatchCall dc | dc = call.getDispatchCall() |
+      result.getUnderlyingCallable() =
+        getCallableForDataFlow(dc.getADynamicTargetInCallContext(ctx.(NonDelegateDataFlowCall)
+                .getDispatchCall()).getUnboundDeclaration())
+      or
+      exists(Callable c, DataFlowCallable encl |
+        result.asSummarizedCallable() = c and
+        mayBenefitFromCallContext(call, encl) and
+        encl = ctx.getARuntimeTarget() and
+        c = dc.getAStaticTarget().getUnboundDeclaration() and
+        not c instanceof RuntimeCallable
+      )
+    )
   }
 }
 

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll

@@ -709,7 +709,8 @@ private module Cached {
      */
     pragma[nomagic]
     private DataFlowCallable viableImplInCallContextExt(DataFlowCall call, DataFlowCall ctx) {
-      result = viableImplInCallContext(call, ctx)
+      result = viableImplInCallContext(call, ctx) and
+      result = viableCallable(call)
       or
       result = viableCallableLambda(call, TDataFlowCallSome(ctx))
       or

csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplConsistency.qll

@@ -38,6 +38,13 @@ module Consistency {
 
     /** Holds if `n` should be excluded from the consistency test `uniquePostUpdate`. */
     predicate uniquePostUpdateExclude(Node n) { none() }
+
+    /** Holds if `(call, ctx)` should be excluded from the consistency test `viableImplInCallContextTooLargeExclude`. */
+    predicate viableImplInCallContextTooLargeExclude(
+      DataFlowCall call, DataFlowCall ctx, DataFlowCallable callable
+    ) {
+      none()
+    }
   }
 
   private class RelevantNode extends Node {
@@ -217,4 +224,12 @@ module Consistency {
     not any(ConsistencyConfiguration c).postWithInFlowExclude(n) and
     msg = "PostUpdateNode should not be the target of local flow."
   }
+
+  query predicate viableImplInCallContextTooLarge(
+    DataFlowCall call, DataFlowCall ctx, DataFlowCallable callable
+  ) {
+    callable = viableImplInCallContext(call, ctx) and
+    not callable = viableCallable(call) and
+    not any(ConsistencyConfiguration c).viableImplInCallContextTooLargeExclude(call, ctx, callable)
+  }
 }

csharp/ql/src/CHANGELOG.md

@@ -1,10 +1,3 @@
-## 0.4.0
-
-### Minor Analysis Improvements
-
-* A new extractor option has been introduced for disabling CIL extraction. Either pass `-Ocil=false` to the `codeql` CLI or set the environment variable `CODEQL_EXTRACTOR_CSHARP_OPTION_CIL=false`.
-* The alert message of many queries have been changed to make the message consistent with other languages.
-
 ## 0.3.4
 
 ## 0.3.3

csharp/ql/src/change-notes/2022-08-23-alert-messages.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The alert message of many queries have been changed to make the message consistent with other languages.

csharp/ql/src/change-notes/2022-08-24-remove-cil-flag.md

@@ -1,6 +1,4 @@
-## 0.4.0
-
-### Minor Analysis Improvements
-
-* A new extractor option has been introduced for disabling CIL extraction. Either pass `-Ocil=false` to the `codeql` CLI or set the environment variable `CODEQL_EXTRACTOR_CSHARP_OPTION_CIL=false`.
-* The alert message of many queries have been changed to make the message consistent with other languages.
+---
+category: minorAnalysis
+---
+* A new extractor option has been introduced for disabling CIL extraction. Either pass `-Ocil=false` to the `codeql` CLI or set the environment variable `CODEQL_EXTRACTOR_CSHARP_OPTION_CIL=false`.

csharp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.4.0
+lastReleaseVersion: 0.3.4

csharp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-queries
-version: 0.4.0
+version: 0.4.0-dev
 groups: 
   - csharp
   - queries

docs/codeql/codeql-cli/creating-codeql-databases.rst

@@ -132,8 +132,7 @@ Python
 
 When creating databases for Python you must ensure:
 
-- You have Python 3 installed and available to the CodeQL extractor.
-- You have the version of Python used by your code installed.
+- You have the all of the required versions of Python installed.
 - You have access to the `pip <https://pypi.org/project/pip/>`__
   packaging management system and can install any
   packages that the codebase depends on.

docs/codeql/support/reusables/versions-compilers.rst

@@ -21,9 +21,9 @@
 
    Eclipse compiler for Java (ECJ) [5]_",``.java``
    JavaScript,ECMAScript 2022 or lower,Not applicable,"``.js``, ``.jsx``, ``.mjs``, ``.es``, ``.es6``, ``.htm``, ``.html``, ``.xhtm``, ``.xhtml``, ``.vue``, ``.hbs``, ``.ejs``, ``.njk``, ``.json``, ``.yaml``, ``.yml``, ``.raml``, ``.xml`` [6]_"
-   Python [7]_,"2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10",Not applicable,``.py``
-   Ruby [8]_,"up to 3.0.2",Not applicable,"``.rb``, ``.erb``, ``.gemspec``, ``Gemfile``"
-   TypeScript [9]_,"2.6-4.8",Standard TypeScript compiler,"``.ts``, ``.tsx``, ``.mts``, ``.cts``"
+   Python,"2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10",Not applicable,``.py``
+   Ruby [7]_,"up to 3.0.2",Not applicable,"``.rb``, ``.erb``, ``.gemspec``, ``Gemfile``"
+   TypeScript [8]_,"2.6-4.8",Standard TypeScript compiler,"``.ts``, ``.tsx``, ``.mts``, ``.cts``"
 
 .. container:: footnote-group
 
@@ -33,6 +33,5 @@
     .. [4] Builds that execute on Java 7 to 19 can be analyzed. The analysis understands Java 19 standard language features.
     .. [5] ECJ is supported when the build invokes it via the Maven Compiler plugin or the Takari Lifecycle plugin.
     .. [6] JSX and Flow code, YAML, JSON, HTML, and XML files may also be analyzed with JavaScript files.
-    .. [7] The extractor requires Python 3 to run. To analyze Python 2.7 you should install both versions of Python.
-    .. [8] Requires glibc 2.17.
-    .. [9] TypeScript analysis is performed by running the JavaScript extractor with TypeScript enabled. This is the default for LGTM.
+    .. [7] Requires glibc 2.17.
+    .. [8] TypeScript analysis is performed by running the JavaScript extractor with TypeScript enabled. This is the default for LGTM.

go/ql/lib/CHANGELOG.md

@@ -1,10 +1,3 @@
-## 0.3.0
-
-### Deprecated APIs
-
-* Some classes/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
-  The old name still exists as a deprecated alias.
-
 ## 0.2.5
 
 ## 0.2.4

go/ql/lib/change-notes/2022-09-12-uppercase.md

@@ -0,0 +1,5 @@
+---
+category: deprecated
+---
+* Some classes/modules with upper-case acronyms in their name have been renamed to follow our style-guide. 
+  The old name still exists as a deprecated alias.

go/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.3.0
+lastReleaseVersion: 0.2.5

go/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/go-all
-version: 0.3.0
+version: 0.3.0-dev
 groups: go
 dbscheme: go.dbscheme
 extractor: go

go/ql/src/CHANGELOG.md

@@ -1,13 +1,3 @@
-## 0.3.0
-
-### Query Metadata Changes
-
-* Added the `security-severity` tag and CWE tag to the `go/insecure-hostkeycallback` query.
-
-### Minor Analysis Improvements
-
-* The alert message of many queries have been changed to make the message consistent with other languages.
-
 ## 0.2.5
 
 ## 0.2.4

go/ql/src/change-notes/2022-08-23-alert-messages.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The alert message of many queries have been changed to make the message consistent with other languages.

go/ql/src/change-notes/2022-08-29-add-security-severity.md

@@ -0,0 +1,4 @@
+---
+category: queryMetadata
+---
+* Added the `security-severity` tag and CWE tag to the `go/insecure-hostkeycallback` query.

go/ql/src/change-notes/released/0.3.0.md

@@ -1,9 +0,0 @@
-## 0.3.0
-
-### Query Metadata Changes
-
-* Added the `security-severity` tag and CWE tag to the `go/insecure-hostkeycallback` query.
-
-### Minor Analysis Improvements
-
-* The alert message of many queries have been changed to make the message consistent with other languages.

go/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.3.0
+lastReleaseVersion: 0.2.5

go/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/go-queries
-version: 0.3.0
+version: 0.3.0-dev
 groups:
   - go
   - queries

java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt

@@ -574,8 +574,8 @@ open class KotlinFileExtractor(
 
                     break
                 } else if (parent is IrFile) {
-                    if (innerClass != null) {
-                        // We don't have to extract file class containers for classes
+                    if (innerClass != null && !innerClass.isLocal) {
+                        // We don't have to extract file class containers for classes except for local classes
                         break
                     }
                     if (this.filePath != parent.path) {

java/ql/lib/CHANGELOG.md

@@ -1,50 +1,3 @@
-## 0.4.0
-
-### Breaking Changes
-
-* The `Member.getQualifiedName()` predicate result now includes the qualified name of the declaring type.
-
-### Deprecated APIs
-
-* The predicate `Annotation.getAValue()` has been deprecated because it might lead to obtaining the value of the wrong annotation element by accident. `getValue(string)` (or one of the value type specific predicates) should be used to explicitly specify the name of the annotation element.
-* The predicate `Annotation.getAValue(string)` has been renamed to `getAnArrayValue(string)`.
-* The predicate `SuppressWarningsAnnotation.getASuppressedWarningLiteral()` has been deprecated because it unnecessarily restricts the result type; `getASuppressedWarning()` should be used instead.
-* The predicates `TargetAnnotation.getATargetExpression()` and `RetentionAnnotation.getRetentionPolicyExpression()` have been deprecated because getting the enum constant read expression is rarely useful, instead the corresponding predicates for getting the name of the referenced enum constants should be used.
-
-### New Features
-
-* Added a new predicate, `allowsBackup`, in the `AndroidApplicationXmlElement` class. This predicate detects if the application element does not disable the `android:allowBackup` attribute.
-* The predicates of the CodeQL class `Annotation` have been improved:
-  * Convenience value type specific predicates have been added, such as `getEnumConstantValue(string)` or `getStringValue(string)`.
-  * Convenience predicates for elements with array values have been added, such as `getAnEnumConstantArrayValue(string)`. While the behavior of the existing predicates has not changed, usage of them should be reviewed (or replaced with the newly added predicate) to make sure they work correctly for elements with array values.
-  * Some internal CodeQL usage of the `Annotation` predicates has been adjusted and corrected; this might affect the results of some queries.
-* New predicates have been added to the CodeQL class `Annotatable` to support getting declared and associated annotations. As part of that, `hasAnnotation()` has been changed to also consider inherited annotations, to be consistent with `hasAnnotation(string, string)` and `getAnAnnotation()`. The newly added predicate `hasDeclaredAnnotation()` can be used as replacement for the old functionality.
-* New predicates have been added to the CodeQL class `AnnotationType` to simplify getting information about usage of JDK meta-annotations, such as `@Retention`.
-
-### Major Analysis Improvements
-
-* The virtual dispatch relation used in data flow now favors summary models over source code for dispatch to interface methods from `java.util` unless there is evidence that a specific source implementation is reachable. This should provide increased precision for any projects that include, for example, custom `List` or `Map` implementations.
-
-### Minor Analysis Improvements
-
-* Added new sinks to the query `java/android/implict-pendingintents` to take into account the classes `androidx.core.app.NotificationManagerCompat` and `androidx.core.app.AlarmManagerCompat`.
-* Added new flow steps for `androidx.core.app.NotificationCompat` and its inner classes.
-* Added flow sinks, sources and summaries for the Kotlin standard library.
-* Added flow summary for `org.springframework.data.repository.CrudRepository.save()`.
-* Added new flow steps for the following Android classes:
-  * `android.content.ContentResolver`
-  * `android.content.ContentProviderClient`
-  * `android.content.ContentProviderOperation`
-  * `android.content.ContentProviderOperation$Builder`
-  * `android.content.ContentProviderResult`
-  * `android.database.Cursor`
-* Added taint flow models for the `java.lang.String.(charAt|getBytes)` methods.
-* Improved taint flow models for the `java.lang.String.(replace|replaceFirst|replaceAll)` methods. Additional results may be found where users do not properly sanitize their inputs.
-
-### Bug Fixes
-
-* Fixed an issue in the taint tracking analysis where implicit reads were not allowed by default in sinks or additional taint steps that used flow states.
-
 ## 0.3.5
 
 ## 0.3.4
@@ -68,7 +21,7 @@
 
 ### Minor Analysis Improvements
 
-* Added new flow steps for the classes `java.nio.file.Path` and `java.nio.file.Paths`.
+* Added new flow steps for the classes `java.io.Path` and `java.nio.Paths`.
 * The class `AndroidFragment` now also models the Android Jetpack version of the `Fragment` class (`androidx.fragment.app.Fragment`).
 * Java 19 builds can now be extracted. There are no non-preview new language features in this release, so the only user-visible change is that the CodeQL extractor will now correctly trace compilations using the JDK 19 release of `javac`.
 * Classes and methods that are seen with several different paths during the extraction process (for example, packaged into different JAR files) now report an arbitrarily selected location via their `getLocation` and `hasLocationInfo` predicates, rather than reporting all of them. This may lead to reduced alert duplication.

java/ql/lib/change-notes/2022-04-01-annotation-deprecations.md

@@ -0,0 +1,7 @@
+---
+category: deprecated
+---
+* The predicate `Annotation.getAValue()` has been deprecated because it might lead to obtaining the value of the wrong annotation element by accident. `getValue(string)` (or one of the value type specific predicates) should be used to explicitly specify the name of the annotation element.
+* The predicate `Annotation.getAValue(string)` has been renamed to `getAnArrayValue(string)`.
+* The predicate `SuppressWarningsAnnotation.getASuppressedWarningLiteral()` has been deprecated because it unnecessarily restricts the result type; `getASuppressedWarning()` should be used instead.
+* The predicates `TargetAnnotation.getATargetExpression()` and `RetentionAnnotation.getRetentionPolicyExpression()` have been deprecated because getting the enum constant read expression is rarely useful, instead the corresponding predicates for getting the name of the referenced enum constants should be used.

java/ql/lib/change-notes/2022-04-01-annotation-features.md

@@ -0,0 +1,9 @@
+---
+category: feature
+---
+* The predicates of the CodeQL class `Annotation` have been improved:
+  * Convenience value type specific predicates have been added, such as `getEnumConstantValue(string)` or `getStringValue(string)`.
+  * Convenience predicates for elements with array values have been added, such as `getAnEnumConstantArrayValue(string)`. While the behavior of the existing predicates has not changed, usage of them should be reviewed (or replaced with the newly added predicate) to make sure they work correctly for elements with array values.
+  * Some internal CodeQL usage of the `Annotation` predicates has been adjusted and corrected; this might affect the results of some queries.
+* New predicates have been added to the CodeQL class `Annotatable` to support getting declared and associated annotations. As part of that, `hasAnnotation()` has been changed to also consider inherited annotations, to be consistent with `hasAnnotation(string, string)` and `getAnAnnotation()`. The newly added predicate `hasDeclaredAnnotation()` can be used as replacement for the old functionality.
+* New predicates have been added to the CodeQL class `AnnotationType` to simplify getting information about usage of JDK meta-annotations, such as `@Retention`.

java/ql/lib/change-notes/2022-08-18-android-manifest-backup-predicate.md

@@ -0,0 +1,4 @@
+---
+category: feature
+---
+* Added a new predicate, `allowsBackup`, in the `AndroidApplicationXmlElement` class. This predicate detects if the application element does not disable the `android:allowBackup` attribute.

java/ql/lib/change-notes/2022-08-19-string-taint-models.md

@@ -0,0 +1,5 @@
+---
+category: minorAnalysis
+---
+* Added taint flow models for the `java.lang.String.(charAt|getBytes)` methods.
+* Improved taint flow models for the `java.lang.String.(replace|replaceFirst|replaceAll)` methods. Additional results may be found where users do not properly sanitize their inputs.

java/ql/lib/change-notes/2022-08-23-contentresolver-summaries.md

@@ -0,0 +1,10 @@
+---
+category: minorAnalysis
+---
+* Added new flow steps for the following Android classes:
+  * `android.content.ContentResolver`
+  * `android.content.ContentProviderClient`
+  * `android.content.ContentProviderOperation`
+  * `android.content.ContentProviderOperation$Builder`
+  * `android.content.ContentProviderResult`
+  * `android.database.Cursor`

java/ql/lib/change-notes/2022-08-25-taint-model-spring-crudrepository.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Added flow summary for `org.springframework.data.repository.CrudRepository.save()`.

java/ql/lib/change-notes/2022-08-31-kotlin-stdlib.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Added flow sinks, sources and summaries for the Kotlin standard library.

java/ql/lib/change-notes/2022-09-06-notificationcompat-summaries.md

@@ -0,0 +1,5 @@
+---
+category: minorAnalysis
+---
+* Added new flow steps for `androidx.core.app.NotificationCompat` and its inner classes.
+  

java/ql/lib/change-notes/2022-09-07-implicit-pendingintents-compat-sinks.md

@@ -0,0 +1,5 @@
+---
+category: minorAnalysis
+---
+* Added new sinks to the query `java/android/implict-pendingintents` to take into account the classes `androidx.core.app.NotificationManagerCompat` and `androidx.core.app.AlarmManagerCompat`.
+  

java/ql/lib/change-notes/2022-09-08-implicit-read-flowstates.md

@@ -0,0 +1,4 @@
+---
+category: fix
+---
+* Fixed an issue in the taint tracking analysis where implicit reads were not allowed by default in sinks or additional taint steps that used flow states.

java/ql/lib/change-notes/2022-09-13-Member-getQualifiedName.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* The `Member.getQualifiedName()` predicate result now includes the qualified name of the declaring type.

java/ql/lib/change-notes/2022-09-16-dispatch-confidence.md

@@ -0,0 +1,4 @@
+---
+category: majorAnalysis
+---
+* The virtual dispatch relation used in data flow now favors summary models over source code for dispatch to interface methods from `java.util` unless there is evidence that a specific source implementation is reachable. This should provide increased precision for any projects that include, for example, custom `List` or `Map` implementations.

java/ql/lib/change-notes/released/0.4.0.md

@@ -1,46 +0,0 @@
-## 0.4.0
-
-### Breaking Changes
-
-* The `Member.getQualifiedName()` predicate result now includes the qualified name of the declaring type.
-
-### Deprecated APIs
-
-* The predicate `Annotation.getAValue()` has been deprecated because it might lead to obtaining the value of the wrong annotation element by accident. `getValue(string)` (or one of the value type specific predicates) should be used to explicitly specify the name of the annotation element.
-* The predicate `Annotation.getAValue(string)` has been renamed to `getAnArrayValue(string)`.
-* The predicate `SuppressWarningsAnnotation.getASuppressedWarningLiteral()` has been deprecated because it unnecessarily restricts the result type; `getASuppressedWarning()` should be used instead.
-* The predicates `TargetAnnotation.getATargetExpression()` and `RetentionAnnotation.getRetentionPolicyExpression()` have been deprecated because getting the enum constant read expression is rarely useful, instead the corresponding predicates for getting the name of the referenced enum constants should be used.
-
-### New Features
-
-* Added a new predicate, `allowsBackup`, in the `AndroidApplicationXmlElement` class. This predicate detects if the application element does not disable the `android:allowBackup` attribute.
-* The predicates of the CodeQL class `Annotation` have been improved:
-  * Convenience value type specific predicates have been added, such as `getEnumConstantValue(string)` or `getStringValue(string)`.
-  * Convenience predicates for elements with array values have been added, such as `getAnEnumConstantArrayValue(string)`. While the behavior of the existing predicates has not changed, usage of them should be reviewed (or replaced with the newly added predicate) to make sure they work correctly for elements with array values.
-  * Some internal CodeQL usage of the `Annotation` predicates has been adjusted and corrected; this might affect the results of some queries.
-* New predicates have been added to the CodeQL class `Annotatable` to support getting declared and associated annotations. As part of that, `hasAnnotation()` has been changed to also consider inherited annotations, to be consistent with `hasAnnotation(string, string)` and `getAnAnnotation()`. The newly added predicate `hasDeclaredAnnotation()` can be used as replacement for the old functionality.
-* New predicates have been added to the CodeQL class `AnnotationType` to simplify getting information about usage of JDK meta-annotations, such as `@Retention`.
-
-### Major Analysis Improvements
-
-* The virtual dispatch relation used in data flow now favors summary models over source code for dispatch to interface methods from `java.util` unless there is evidence that a specific source implementation is reachable. This should provide increased precision for any projects that include, for example, custom `List` or `Map` implementations.
-
-### Minor Analysis Improvements
-
-* Added new sinks to the query `java/android/implict-pendingintents` to take into account the classes `androidx.core.app.NotificationManagerCompat` and `androidx.core.app.AlarmManagerCompat`.
-* Added new flow steps for `androidx.core.app.NotificationCompat` and its inner classes.
-* Added flow sinks, sources and summaries for the Kotlin standard library.
-* Added flow summary for `org.springframework.data.repository.CrudRepository.save()`.
-* Added new flow steps for the following Android classes:
-  * `android.content.ContentResolver`
-  * `android.content.ContentProviderClient`
-  * `android.content.ContentProviderOperation`
-  * `android.content.ContentProviderOperation$Builder`
-  * `android.content.ContentProviderResult`
-  * `android.database.Cursor`
-* Added taint flow models for the `java.lang.String.(charAt|getBytes)` methods.
-* Improved taint flow models for the `java.lang.String.(replace|replaceFirst|replaceAll)` methods. Additional results may be found where users do not properly sanitize their inputs.
-
-### Bug Fixes
-
-* Fixed an issue in the taint tracking analysis where implicit reads were not allowed by default in sinks or additional taint steps that used flow states.

java/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.4.0
+lastReleaseVersion: 0.3.5

java/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-all
-version: 0.4.0
+version: 0.4.0-dev
 groups: java
 dbscheme: config/semmlecode.dbscheme
 extractor: java

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll

@@ -709,7 +709,8 @@ private module Cached {
      */
     pragma[nomagic]
     private DataFlowCallable viableImplInCallContextExt(DataFlowCall call, DataFlowCall ctx) {
-      result = viableImplInCallContext(call, ctx)
+      result = viableImplInCallContext(call, ctx) and
+      result = viableCallable(call)
       or
       result = viableCallableLambda(call, TDataFlowCallSome(ctx))
       or

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll

@@ -38,6 +38,13 @@ module Consistency {
 
     /** Holds if `n` should be excluded from the consistency test `uniquePostUpdate`. */
     predicate uniquePostUpdateExclude(Node n) { none() }
+
+    /** Holds if `(call, ctx)` should be excluded from the consistency test `viableImplInCallContextTooLargeExclude`. */
+    predicate viableImplInCallContextTooLargeExclude(
+      DataFlowCall call, DataFlowCall ctx, DataFlowCallable callable
+    ) {
+      none()
+    }
   }
 
   private class RelevantNode extends Node {
@@ -217,4 +224,12 @@ module Consistency {
     not any(ConsistencyConfiguration c).postWithInFlowExclude(n) and
     msg = "PostUpdateNode should not be the target of local flow."
   }
+
+  query predicate viableImplInCallContextTooLarge(
+    DataFlowCall call, DataFlowCall ctx, DataFlowCallable callable
+  ) {
+    callable = viableImplInCallContext(call, ctx) and
+    not callable = viableCallable(call) and
+    not any(ConsistencyConfiguration c).viableImplInCallContextTooLargeExclude(call, ctx, callable)
+  }
 }

java/ql/src/CHANGELOG.md

@@ -1,21 +1,3 @@
-## 0.4.0
-
-### New Queries
-
-* The query "Server-side template injection" (`java/server-side-template-injection`) has been promoted from experimental to the main query pack. This query was originally [submitted as an experimental query by @porcupineyhairs](https://github.com/github/codeql/pull/5935).
-* Added a new query, `java/android/backup-enabled`, to detect if Android applications allow backups.
-
-### Query Metadata Changes
-
-* Removed the `@security-severity` tag from several queries not in the `Security/` folder that also had missing `security` tags.
-
-### Minor Analysis Improvements
-
-* The Java extractor now populates the `Method` relating to a `MethodAccess` consistently for calls using an explicit and implicit `this` qualifier. Previously if the method `foo` was inherited from a specialised generic type `ParentType<String>`, then an explicit call `this.foo()` would yield a `MethodAccess` whose `getMethod()` accessor returned the bound method `ParentType<String>.foo`, whereas an implicitly-qualified `foo()` `MethodAccess`'s `getMethod()` would return the unbound method `ParentType.foo`. Now both scenarios produce a bound method. This means that all data-flow queries may return more results where a relevant path transits a call to such an implicitly-qualified call to a member method with a bound generic type, while queries that inspect the result of `MethodAccess.getMethod()` may need to tolerate bound generic methods in more circumstances. The queries `java/iterator-remove-failure`, `java/non-static-nested-class`, `java/internal-representation-exposure`, `java/subtle-inherited-call` and `java/deprecated-call` have been amended to properly handle calls to bound generic methods, and in some instances may now produce more results in the explicit-`this` case as well.
-* Added taint model for arguments of `java.net.URI` constructors to the queries `java/path-injection` and `java/path-injection-local`.
-* Added new sinks related to Android's `AlarmManager` to the query `java/android/implicit-pendingintents`.
-* The alert message of many queries have been changed to make the message consistent with other languages.
-
 ## 0.3.4
 
 ## 0.3.3

java/ql/src/change-notes/2022-08-18-android-allowbackup-query.md

@@ -0,0 +1,4 @@
+---
+category: newQuery
+---
+* Added a new query, `java/android/backup-enabled`, to detect if Android applications allow backups.

java/ql/src/change-notes/2022-08-23-alert-messages.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The alert message of many queries have been changed to make the message consistent with other languages.

java/ql/src/change-notes/2022-08-29-remove-security-tag.md

@@ -0,0 +1,4 @@
+---
+category: queryMetadata
+---
+* Removed the `@security-severity` tag from several queries not in the `Security/` folder that also had missing `security` tags.

java/ql/src/change-notes/2022-09-01-implicit-pendingintents-new-sinks.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Added new sinks related to Android's `AlarmManager` to the query `java/android/implicit-pendingintents`.

java/ql/src/change-notes/2022-09-08-server-side-template-injection-query.md

@@ -0,0 +1,4 @@
+---
+category: newQuery
+---
+* The query "Server-side template injection" (`java/server-side-template-injection`) has been promoted from experimental to the main query pack. This query was originally [submitted as an experimental query by @porcupineyhairs](https://github.com/github/codeql/pull/5935).

java/ql/src/change-notes/2022-09-13-flow-model-for-uri-constructors.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Added taint model for arguments of `java.net.URI` constructors to the queries `java/path-injection` and `java/path-injection-local`.

java/ql/src/change-notes/2022-09-15-implicit-this-generic-method.md

@@ -1,17 +1,4 @@
-## 0.4.0
-
-### New Queries
-
-* The query "Server-side template injection" (`java/server-side-template-injection`) has been promoted from experimental to the main query pack. This query was originally [submitted as an experimental query by @porcupineyhairs](https://github.com/github/codeql/pull/5935).
-* Added a new query, `java/android/backup-enabled`, to detect if Android applications allow backups.
-
-### Query Metadata Changes
-
-* Removed the `@security-severity` tag from several queries not in the `Security/` folder that also had missing `security` tags.
-
-### Minor Analysis Improvements
-
+---
+category: minorAnalysis
+---
 * The Java extractor now populates the `Method` relating to a `MethodAccess` consistently for calls using an explicit and implicit `this` qualifier. Previously if the method `foo` was inherited from a specialised generic type `ParentType<String>`, then an explicit call `this.foo()` would yield a `MethodAccess` whose `getMethod()` accessor returned the bound method `ParentType<String>.foo`, whereas an implicitly-qualified `foo()` `MethodAccess`'s `getMethod()` would return the unbound method `ParentType.foo`. Now both scenarios produce a bound method. This means that all data-flow queries may return more results where a relevant path transits a call to such an implicitly-qualified call to a member method with a bound generic type, while queries that inspect the result of `MethodAccess.getMethod()` may need to tolerate bound generic methods in more circumstances. The queries `java/iterator-remove-failure`, `java/non-static-nested-class`, `java/internal-representation-exposure`, `java/subtle-inherited-call` and `java/deprecated-call` have been amended to properly handle calls to bound generic methods, and in some instances may now produce more results in the explicit-`this` case as well.
-* Added taint model for arguments of `java.net.URI` constructors to the queries `java/path-injection` and `java/path-injection-local`.
-* Added new sinks related to Android's `AlarmManager` to the query `java/android/implicit-pendingintents`.
-* The alert message of many queries have been changed to make the message consistent with other languages.

java/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.4.0
+lastReleaseVersion: 0.3.4

java/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-queries
-version: 0.4.0
+version: 0.4.0-dev
 groups: 
   - java
   - queries

java/ql/test/kotlin/library-tests/classes/PrintAst.expected

@@ -7,6 +7,30 @@ classes.kt:
 #   32|         0: [Parameter] s
 #   32|           0: [TypeAccess] String
 #   32|       5: [BlockStmt] { ... }
+#  158|     2: [Method] fn1
+#  158|       3: [TypeAccess] Unit
+#  158|       5: [BlockStmt] { ... }
+#  159|         0: [LocalTypeDeclStmt] class ...
+#  159|           0: [LocalClass] X
+#  159|             1: [Constructor] X
+#  159|               5: [BlockStmt] { ... }
+#  159|                 0: [SuperConstructorInvocationStmt] super(...)
+#  159|                 1: [BlockStmt] { ... }
+#  162|     3: [Method] fn2
+#  162|       3: [TypeAccess] Object
+#  162|       5: [BlockStmt] { ... }
+#  162|         0: [ReturnStmt] return ...
+#  162|           0: [StmtExpr] <Stmt>
+#  162|             0: [BlockStmt] { ... }
+#  162|               0: [LocalTypeDeclStmt] class ...
+#  162|                 0: [AnonymousClass,LocalClass] new Object(...) { ... }
+#  162|                   1: [Constructor] 
+#  162|                     5: [BlockStmt] { ... }
+#  162|                       0: [SuperConstructorInvocationStmt] super(...)
+#  162|                       1: [BlockStmt] { ... }
+#  162|               1: [ExprStmt] <Expr>;
+#  162|                 0: [ClassInstanceExpr] new (...)
+#  162|                   -3: [TypeAccess] Object
 #    2|   2: [Class] ClassOne
 #    2|     1: [Constructor] ClassOne
 #    2|       5: [BlockStmt] { ... }

java/ql/test/kotlin/library-tests/classes/anonymousClasses.expected

@@ -6,6 +6,7 @@
 | classes.kt:85:16:85:25 | new Object(...) { ... } | classes.kt:85:16:85:25 | new (...) |  | classes.kt:85:16:85:25 | Object | classes.kt:85:16:85:25 | class ... |
 | classes.kt:89:16:89:44 | new Interface3<Integer>(...) { ... } | classes.kt:89:16:89:44 | new (...) |  | classes.kt:89:16:89:44 | Interface3<Integer> | classes.kt:89:16:89:44 | class ... |
 | classes.kt:127:16:134:9 | new Object(...) { ... } | classes.kt:127:16:134:9 | new (...) |  | classes.kt:127:16:134:9 | Object | classes.kt:127:16:134:9 | class ... |
+| classes.kt:162:13:162:22 | new Object(...) { ... } | classes.kt:162:13:162:22 | new (...) |  | classes.kt:162:13:162:22 | Object | classes.kt:162:13:162:22 | class ... |
 | generic_anonymous.kt:3:19:5:3 | new Object(...) { ... } | generic_anonymous.kt:3:19:5:3 | new (...) |  | generic_anonymous.kt:3:19:5:3 | Object | generic_anonymous.kt:3:19:5:3 | class ... |
 | generic_anonymous.kt:26:13:26:37 | new Object(...) { ... } | generic_anonymous.kt:26:13:26:37 | new (...) |  | generic_anonymous.kt:26:13:26:37 | Object | generic_anonymous.kt:26:13:26:37 | class ... |
 | generic_anonymous.kt:27:13:27:37 | new Object(...) { ... } | generic_anonymous.kt:27:13:27:37 | new (...) |  | generic_anonymous.kt:27:13:27:37 | Object | generic_anonymous.kt:27:13:27:37 | class ... |

java/ql/test/kotlin/library-tests/classes/classes.expected

@@ -39,6 +39,8 @@
 | classes.kt:141:13:145:13 | Cl1 | Cl0$Cl1 | final, private |
 | classes.kt:150:1:156:1 | Cl00 | Cl00 | final, public |
 | classes.kt:151:5:155:5 | Cl01 | Cl00$Cl01 | final, public |
+| classes.kt:159:5:159:14 | X | ClassesKt$X | final, private |
+| classes.kt:162:13:162:22 | new Object(...) { ... } | <anonymous class> | final, private |
 | generic_anonymous.kt:0:0:0:0 | Generic_anonymousKt | Generic_anonymousKt | final, public |
 | generic_anonymous.kt:1:1:9:1 | Generic | Generic | final, private |
 | generic_anonymous.kt:3:19:5:3 | new Object(...) { ... } | <anonymous class> | final, private |

java/ql/test/kotlin/library-tests/classes/classes.kt

@@ -153,4 +153,10 @@ class Cl00<U0> {
             val x = Cl01<U1>()
         }
     }
-}
+}
+
+fun fn1() {
+    class X {}
+}
+
+fun fn2() = object { }

java/ql/test/kotlin/library-tests/classes/ctorCalls.expected

@@ -39,6 +39,8 @@ superCall
 | classes.kt:141:13:145:13 | super(...) |
 | classes.kt:150:1:156:1 | super(...) |
 | classes.kt:151:5:155:5 | super(...) |
+| classes.kt:159:5:159:14 | super(...) |
+| classes.kt:162:13:162:22 | super(...) |
 | generic_anonymous.kt:1:1:9:1 | super(...) |
 | generic_anonymous.kt:3:19:5:3 | super(...) |
 | generic_anonymous.kt:15:1:33:1 | super(...) |

java/ql/test/kotlin/library-tests/classes/localClass.expected

@@ -4,6 +4,7 @@
 | classes.kt:129:17:131:17 | class ... | classes.kt:129:17:131:17 | Local3 | classes.kt:128:13:133:13 | fn | classes.kt:127:16:134:9 | new Object(...) { ... } |
 | classes.kt:140:9:146:9 | class ... | classes.kt:140:9:146:9 |  | classes.kt:139:5:147:5 | func1 | classes.kt:138:1:148:1 | Cl0 |
 | classes.kt:141:13:145:13 | class ... | classes.kt:141:13:145:13 | Cl1 | classes.kt:140:9:146:9 | func2 | classes.kt:138:1:148:1 | Cl0 |
+| classes.kt:159:5:159:14 | class ... | classes.kt:159:5:159:14 | X | classes.kt:158:1:160:1 | fn1 | classes.kt:0:0:0:0 | ClassesKt |
 | generic_anonymous.kt:25:9:31:9 | class ... | generic_anonymous.kt:25:9:31:9 |  | generic_anonymous.kt:24:5:32:5 | func1 | generic_anonymous.kt:15:1:33:1 | Outer |
 | localClassField.kt:3:9:3:19 | class ... | localClassField.kt:3:9:3:19 | L | localClassField.kt:1:1:11:1 | A | localClassField.kt:1:1:11:1 | A |
 | localClassField.kt:8:9:8:19 | class ... | localClassField.kt:8:9:8:19 | L | localClassField.kt:1:1:11:1 | A | localClassField.kt:1:1:11:1 | A |

java/ql/test/kotlin/library-tests/classes/superTypes.expected

@@ -44,6 +44,8 @@
 | classes.kt:141:13:145:13 | Cl1 | file://<external>/Object.class:0:0:0:0 | Object |
 | classes.kt:150:1:156:1 | Cl00 | file://<external>/Object.class:0:0:0:0 | Object |
 | classes.kt:151:5:155:5 | Cl01 | file://<external>/Object.class:0:0:0:0 | Object |
+| classes.kt:159:5:159:14 | X | file://<external>/Object.class:0:0:0:0 | Object |
+| classes.kt:162:13:162:22 | new Object(...) { ... } | file://<external>/Object.class:0:0:0:0 | Object |
 | file://<external>/C1$<no name provided>$Local3.class:0:0:0:0 | Local3<Integer> | file://<external>/Object.class:0:0:0:0 | Object |
 | file://<external>/C1$Local1.class:0:0:0:0 | Local1<Integer> | file://<external>/Object.class:0:0:0:0 | Object |
 | file://<external>/C1$Local2.class:0:0:0:0 | Local2<Integer> | file://<external>/Object.class:0:0:0:0 | Object |

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/EndpointFeatures.qll

@@ -16,220 +16,8 @@ private import FunctionBodyFeatures as FunctionBodyFeatures
 private string getTokenFeature(DataFlow::Node endpoint, string featureName) {
   // Performance optimization: Restrict feature extraction to endpoints we've explicitly asked to featurize.
   endpoint = any(FeaturizationConfig cfg).getAnEndpointToFeaturize() and
-  (
-    // Features for endpoints that are contained within a function.
-    exists(Function function |
-      function = FunctionBodyFeatures::getRepresentativeFunctionForEndpoint(endpoint)
-    |
-      // The name of the function that encloses the endpoint.
-      featureName = "enclosingFunctionName" and result = FunctionNames::getNameToFeaturize(function)
-      or
-      // A feature containing natural language tokens from the function that encloses the endpoint in
-      // the order that they appear in the source code.
-      featureName = "enclosingFunctionBody" and
-      result = FunctionBodyFeatures::getBodyTokensFeature(function)
-    )
-    or
-    result =
-      strictconcat(DataFlow::CallNode call, string component |
-        component = getACallBasedTokenFeatureComponent(endpoint, call, featureName)
-      |
-        component, " "
-      )
-    or
-    // The access path of the function being called, both with and without structural info, if the
-    // function being called originates from an external API. For example, the endpoint here:
-    //
-    // ```js
-    // const mongoose = require('mongoose'),
-    //   User = mongoose.model('User', null);
-    // User.findOne(ENDPOINT);
-    // ```
-    //
-    // would have a callee access path with structural info of
-    // `mongoose member model instanceorreturn member findOne instanceorreturn`, and a callee access
-    // path without structural info of `mongoose model findOne`.
-    //
-    // These features indicate that the callee comes from (reading the access path backwards) an
-    // instance of the `findOne` member of an instance of the `model` member of the `mongoose`
-    // external library.
-    exists(AccessPaths::Boolean includeStructuralInfo |
-      featureName =
-        "calleeAccessPath" +
-          any(string x | if includeStructuralInfo = true then x = "WithStructuralInfo" else x = "") and
-      result =
-        concat(API::Node node, string accessPath |
-          node.getInducingNode().(DataFlow::CallNode).getAnArgument() = endpoint and
-          AccessPaths::accessPaths(node, includeStructuralInfo, accessPath, _)
-        |
-          accessPath, " "
-        )
-    )
-  )
-}
-
-/**
- * Gets a value of the function-call-related token-based feature named `featureName` associated
- * with the function call `call` and the endpoint `endpoint`.
- *
- * This may in general report multiple strings, each containing a space-separated list of tokens.
- *
- * **Technical details:** This predicate can have multiple values per endpoint and feature name. As
- * a result, the results from this predicate must be concatenated together.  However concatenating
- * other features like the function body tokens is expensive, so for performance reasons we separate
- * out this predicate from those other features.
- */
-private string getACallBasedTokenFeatureComponent(
-  DataFlow::Node endpoint, DataFlow::CallNode call, string featureName
-) {
-  // Performance optimization: Restrict feature extraction to endpoints we've explicitly asked to featurize.
-  endpoint = any(FeaturizationConfig cfg).getAnEndpointToFeaturize() and
-  // Features for endpoints that are an argument to a function call.
-  endpoint = call.getAnArgument() and
-  (
-    // The name of the function being called, e.g. in a call `Artist.findOne(...)`, this is `findOne`.
-    featureName = "calleeName" and result = call.getCalleeName()
-    or
-    // The name of the receiver of the call, e.g. in a call `Artist.findOne(...)`, this is `Artist`.
-    featureName = "receiverName" and result = call.getReceiver().asExpr().(VarRef).getName()
-    or
-    // The argument index of the endpoint, e.g. in `f(a, endpoint, b)`, this is 1.
-    featureName = "argumentIndex" and
-    result = any(int argIndex | call.getArgument(argIndex) = endpoint).toString()
-    or
-    // The name of the API that the function being called originates from, if the function being
-    // called originates from an external API. For example, the endpoint here:
-    //
-    // ```js
-    // const mongoose = require('mongoose'),
-    //   User = mongoose.model('User', null);
-    // User.findOne(ENDPOINT);
-    // ```
-    //
-    // would have a callee API name of `mongoose`.
-    featureName = "calleeApiName" and
-    exists(API::Node apiNode |
-      AccessPaths::accessPaths(apiNode, false, _, result) and call = apiNode.getInducingNode()
-    )
-  )
-}
-
-/**
- * This module provides functionality for getting a representation of the access path of nodes
- * within the program.
- *
- * For example, it gives the `User.find` callee here:
- *
- * ```js
- * const mongoose = require('mongoose'),
- * User = mongoose.model('User', null);
- * User.find({ 'isAdmin': true })
- * ```
- * the access path `mongoose member model instanceorreturn member find instanceorreturn`.
- *
- * This access path is based on the simplified access path that the untrusted data flowing to
- * external API query associates to each of its sinks, with modifications to optionally include
- * explicit structural information and to improve how well the path tokenizes.
- */
-private module AccessPaths {
-  bindingset[str]
-  private predicate isNumericString(string str) { exists(str.toInt()) }
-
-  /**
-   * Gets a parameter of `base` with name `name`, or a property named `name` of a destructuring parameter.
-   */
-  private API::Node getNamedParameter(API::Node base, string name) {
-    exists(API::Node param |
-      param = base.getAParameter() and
-      not param = base.getReceiver()
-    |
-      result = param and
-      name = param.asSource().(DataFlow::ParameterNode).getName()
-      or
-      param.asSource().asExpr() instanceof DestructuringPattern and
-      result = param.getMember(name)
-    )
-  }
-
-  /**
-   * A utility class that is equivalent to `boolean` but does not require type joining.
-   */
-  class Boolean extends boolean {
-    Boolean() { this = true or this = false }
-  }
-
-  /** Get the access path for the node. This includes structural information like `member`, `param`, and `functionalarg` if `includeStructuralInfo` is true. */
-  predicate accessPaths(
-    API::Node node, Boolean includeStructuralInfo, string accessPath, string apiName
-  ) {
-    //node = API::moduleImport(result)
-    node = API::moduleImport(apiName) and accessPath = apiName
-    or
-    exists(API::Node previousNode, string previousAccessPath |
-      previousNode.getDepth() < node.getDepth() and
-      accessPaths(previousNode, includeStructuralInfo, previousAccessPath, apiName)
-    |
-      // e.g. `new X`, `X()`
-      node = [previousNode.getInstance(), previousNode.getReturn()] and
-      if includeStructuralInfo = true
-      then accessPath = previousAccessPath + " instanceorreturn"
-      else accessPath = previousAccessPath
-      or
-      // e.g. `x.y`, `x[y]`, `const { y } = x`, where `y` is non-numeric and is known at analysis
-      // time.
-      exists(string member |
-        node = previousNode.getMember(member) and
-        not node = previousNode.getUnknownMember() and
-        not isNumericString(member) and
-        not (member = "default" and previousNode = API::moduleImport(_)) and
-        not member = "then" // use the 'promised' edges for .then callbacks
-      |
-        if includeStructuralInfo = true
-        then accessPath = previousAccessPath + " member " + member
-        else accessPath = previousAccessPath + " " + member
-      )
-      or
-      // e.g. `x.y`, `x[y]`, `const { y } = x`, where `y` is numeric or not known at analysis time.
-      (
-        node = previousNode.getUnknownMember() or
-        node = previousNode.getMember(any(string s | isNumericString(s)))
-      ) and
-      if includeStructuralInfo = true
-      then accessPath = previousAccessPath + " member"
-      else accessPath = previousAccessPath
-      or
-      // e.g. `x.then(y => ...)`
-      node = previousNode.getPromised() and
-      accessPath = previousAccessPath
-      or
-      // e.g. `x.y((a, b) => ...)`
-      // Name callback parameters after their name in the source code.
-      // For example, the `res` parameter in `express.get('/foo', (req, res) => {...})` will be
-      // named `express member get functionalarg param res`.
-      exists(string paramName |
-        node = getNamedParameter(previousNode.getAParameter(), paramName) and
-        (
-          if includeStructuralInfo = true
-          then accessPath = previousAccessPath + " functionalarg param " + paramName
-          else accessPath = previousAccessPath + " " + paramName
-        )
-        or
-        exists(string callbackName, int index |
-          node =
-            getNamedParameter(previousNode
-                  .getASuccessor(API::Label::parameter(index))
-                  .getMember(callbackName), paramName) and
-          index != -1 and // ignore receiver
-          if includeStructuralInfo = true
-          then
-            accessPath =
-              previousAccessPath + " functionalarg " + index + " " + callbackName + " param " +
-                paramName
-          else accessPath = previousAccessPath + " " + index + " " + callbackName + " " + paramName
-        )
-      )
-    )
-  }
+  exists(EndpointFeature f | f.getName() = featureName and result = f.getValue(endpoint)) and
+  featureName = getASupportedFeatureName()
 }
 
 private module FunctionNames {
@@ -284,13 +72,7 @@ private module FunctionNames {
 }
 
 /** Get a name of a supported generic token-based feature. */
-string getASupportedFeatureName() {
-  result =
-    [
-      "enclosingFunctionName", "calleeName", "receiverName", "argumentIndex", "calleeApiName",
-      "calleeAccessPath", "calleeAccessPathWithStructuralInfo", "enclosingFunctionBody"
-    ]
-}
+string getASupportedFeatureName() { result = any(EndpointFeature f).getName() }
 
 /**
  * Generic token-based features for ATM.
@@ -303,3 +85,591 @@ predicate tokenFeatures(DataFlow::Node endpoint, string featureName, string feat
   endpoint = any(FeaturizationConfig cfg).getAnEndpointToFeaturize() and
   featureValue = getTokenFeature(endpoint, featureName)
 }
+
+/**
+ * See EndpointFeature
+ */
+private newtype TEndpointFeature =
+  TEnclosingFunctionName() or
+  TReceiverName() or
+  TEnclosingFunctionBody() or
+  TFileImports() or
+  TCalleeImports() or
+  TCalleeFlexibleAccessPath() or
+  TInputAccessPathFromCallee() or
+  TInputArgumentIndex() or
+  TContextFunctionInterfaces() or
+  TContextSurroundingFunctionParameters() or
+  TAssignedToPropName() or
+  TStringConcatenatedWith()
+
+/**
+ * An implementation of an endpoint feature: defines feature-name/value tuples for use in ML.
+ */
+abstract class EndpointFeature extends TEndpointFeature {
+  /**
+   * Gets the name of the feature. Used by the ML model.
+   * Names are coupled to models: changing the name of a feature requires retraining the model.
+   */
+  abstract string getName();
+
+  /**
+   * Gets the value of the feature. Used by the ML model.
+   * Models are trained based on feature values, so changing the value of a feature requires retraining the model.
+   */
+  abstract string getValue(DataFlow::Node endpoint);
+
+  string toString() { result = this.getName() }
+}
+
+/**
+ * The feature for the name of the function that encloses the endpoint.
+ */
+class EnclosingFunctionName extends EndpointFeature, TEnclosingFunctionName {
+  override string getName() { result = "enclosingFunctionName" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      FunctionNames::getNameToFeaturize(FunctionBodyFeatures::getRepresentativeFunctionForEndpoint(endpoint))
+  }
+}
+
+/**
+ * The feature for the name of the receiver of the call, e.g. in a call `Artist.findOne(...)`, this is `Artist`.
+ */
+class ReceiverName extends EndpointFeature, TReceiverName {
+  override string getName() { result = "receiverName" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      strictconcat(DataFlow::CallNode call, string component |
+        endpoint = call.getAnArgument() and
+        component = call.getReceiver().asExpr().(VarRef).getName()
+      |
+        component, " "
+      )
+  }
+}
+
+/**
+ * The feature for the natural language tokens from the function that encloses the endpoint in
+ * the order that they appear in the source code.
+ */
+class EnclosingFunctionBody extends EndpointFeature, TEnclosingFunctionBody {
+  override string getName() { result = "enclosingFunctionBody" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    endpoint = any(FeaturizationConfig cfg).getAnEndpointToFeaturize() and
+    result =
+      FunctionBodyFeatures::getBodyTokensFeature(FunctionBodyFeatures::getRepresentativeFunctionForEndpoint(endpoint))
+  }
+}
+
+/**
+ * The feature for the imports defined in the file containing an endpoint.
+ *
+ * ### Example
+ *
+ * ```javascript
+ * import { findOne } from 'mongoose';
+ * import * as _ from 'lodash';
+ * const pg = require('pg');
+ *
+ * // ...
+ * ```
+ *
+ * In this file, all endpoints will have the value `lodash mongoose pg` for the feature `fileImports`.
+ */
+class FileImports extends EndpointFeature, TFileImports {
+  override string getName() { result = "fileImports" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result = SyntacticUtilities::getImportPathsForFile(endpoint.getFile())
+  }
+}
+
+/**
+ * The feature for the function parameters of the functions that enclose an endpoint.
+ *
+ * ### Example
+ * ```javascript
+ * function f(a, b) {
+ *   // ...
+ *   const g = (c, d) => x.foo(endpoint);
+ * //                          ^^^^^^^^
+ * }
+ * ```
+ * In the above example, the feature for the marked endpoint has value '(a, b)\n(c, d)'.
+ * The line breaks act as a separator between the parameters of different functions but
+ * will be treated by tokenization as if they were spaces.
+ */
+class ContextSurroundingFunctionParameters extends EndpointFeature,
+  TContextSurroundingFunctionParameters {
+  override string getName() { result = "contextSurroundingFunctionParameters" }
+
+  Function getRelevantFunction(DataFlow::Node endpoint) {
+    result = endpoint.asExpr().getEnclosingFunction*()
+  }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result =
+      concat(string functionParameterLine, Function f |
+        f = this.getRelevantFunction(endpoint) and
+        functionParameterLine = SyntacticUtilities::getFunctionParametersFeatureComponent(f)
+      |
+        functionParameterLine, "\n"
+        order by
+          f.getLocation().getStartLine(), f.getLocation().getStartColumn()
+      )
+  }
+}
+
+/**
+ * The feature that gives the name of any properties an endpoint is assigned to (if any).
+ *
+ * ### Example
+ * ```javascript
+ * const div = document.createElement('div');
+ * div.innerHTML = endpoint; // feature value is 'innerHTML'
+ *
+ * foo({x: endpoint}); // feature value is 'x'
+ * ```
+ */
+class AssignedToPropName extends EndpointFeature, TAssignedToPropName {
+  override string getName() { result = "assignedToPropName" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    exists(DataFlow::PropWrite w | w.getRhs().asExpr().getUnderlyingValue().flow() = endpoint |
+      result = w.getPropertyName()
+    )
+  }
+}
+
+/**
+ * The feature that shows the text an endpoint is being concatenated with.
+ *
+ * ### Example
+ *
+ * ```javascript
+ * const x = 'foo' + endpoint + 'bar'; // feature value is `'foo' -endpoint- 'bar'
+ * ```
+ */
+class StringConcatenatedWith extends EndpointFeature, TStringConcatenatedWith {
+  override string getName() { result = "stringConcatenatedWith" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    exists(StringOps::ConcatenationRoot root |
+      root.getALeaf() = endpoint and
+      result =
+        concat(StringOps::ConcatenationLeaf p |
+            p.getRoot() = root and
+            (
+              p.getStartLine() < endpoint.getStartLine()
+              or
+              p.getStartLine() = endpoint.getStartLine() and
+              p.getStartColumn() < endpoint.getStartColumn()
+            )
+          |
+            SyntacticUtilities::renderStringConcatOperand(p), " + "
+            order by
+              p.getStartLine(), p.getStartColumn()
+          ) + " -endpoint- " +
+          concat(StringOps::ConcatenationLeaf p |
+            p.getRoot() = root and
+            (
+              p.getStartLine() > endpoint.getStartLine()
+              or
+              p.getStartLine() = endpoint.getStartLine() and
+              p.getStartColumn() > endpoint.getStartColumn()
+            )
+          |
+            SyntacticUtilities::renderStringConcatOperand(p), " + "
+            order by
+              p.getStartLine(), p.getStartColumn()
+          )
+    )
+  }
+}
+
+/**
+ * The feature for the imports used in the callee of an invocation.
+ *
+ * ### Example
+ *
+ * ```javascript
+ * import * as _ from 'lodash';
+ *
+ * // ...
+ * _.deepClone(someObject);
+ * //          ^^^^^^^^^^ will have the value `lodash` for the feature `calleeImports`.
+ * ```
+ */
+class CalleeImports extends EndpointFeature, TCalleeImports {
+  override string getName() { result = "calleeImports" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    not result = SyntacticUtilities::getUnknownSymbol() and
+    exists(DataFlow::InvokeNode invk |
+      (
+        invk.getAnArgument() = endpoint or
+        SyntacticUtilities::getANestedInitializerValue(invk.getAnArgument()
+              .asExpr()
+              .getUnderlyingValue()).flow() = endpoint
+      ) and
+      result =
+        concat(string importPath |
+          importPath = SyntacticUtilities::getCalleeImportPath(invk.getCalleeNode())
+        |
+          importPath, " " order by importPath
+        )
+    )
+  }
+}
+
+/**
+ * The feature for the interfaces of all named functions in the same file as the endpoint.
+ *
+ * ### Example
+ * ```javascript
+ * // Will return: "f(a, b, c)\ng(x, y, z)\nh(u, v)" for this file.
+ * function f(a, b, c) { ... }
+ *
+ * function g(x, y, z) {
+ *   function h(u, v) { ... }
+ *   ...
+ * }
+ * ```
+ */
+class ContextFunctionInterfaces extends EndpointFeature, TContextFunctionInterfaces {
+  override string getName() { result = "contextFunctionInterfaces" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    result = SyntacticUtilities::getFunctionInterfacesForFile(endpoint.getFile())
+  }
+}
+
+/**
+ * Syntactic utilities for feature value computation.
+ */
+private module SyntacticUtilities {
+  /**
+   * Renders an operand in a string concatenation by surrounding a constant in quotes, and
+   * by using `getSimpleAccessPath` for everything else.
+   */
+  string renderStringConcatOperand(DataFlow::Node operand) {
+    if exists(unique(string v | operand.mayHaveStringValue(v)))
+    then result = "'" + any(string v | operand.mayHaveStringValue(v)) + "'"
+    else result = getSimpleAccessPath(operand)
+  }
+
+  /** Gets all the imports defined in the file containing the endpoint. */
+  string getImportPathsForFile(File file) {
+    result =
+      concat(string importPath |
+        importPath = SyntacticUtilities::getImportPathForFile(file)
+      |
+        importPath, " " order by importPath
+      )
+  }
+
+  /** Gets an import located in `file`. */
+  string getImportPathForFile(File file) {
+    result = any(Import imp | imp.getFile() = file).getImportedPath().getValue()
+  }
+
+  /**
+   * Gets the feature component for the parameters of a function.
+   *
+   * ```javascript
+   * function f(a, b, c) { // will return "(a, b, c)" for this function
+   *  return a + b + c;
+   * }
+   *
+   * async function g(a) { // will return "(a)" for this function
+   *   return 2*a
+   * };
+   *
+   * const h = (b) => 3*b; // will return "(b)" for this function
+   * ```
+   */
+  string getFunctionParametersFeatureComponent(Function f) {
+    result =
+      "(" +
+        concat(string parameter, int i |
+          parameter = getParameterNameOrUnknown(f.getParameter(i))
+        |
+          parameter, ", " order by i
+        ) + ")"
+  }
+
+  /**
+   * Gets the function interfaces of all named functions in a file, concatenated together.
+   *
+   * ```javascript
+   * // Will return: "f(a, b, c)\ng(x, y, z)\nh(u, v)" for this file.
+   * function f(a, b, c) { ... }
+   *
+   * function g(x, y, z) {
+   *   function h(u, v) { ... }
+   *   ...
+   * }
+   */
+  string getFunctionInterfacesForFile(File file) {
+    result =
+      concat(Function func, string line |
+        func.getFile() = file and
+        line = func.getName() + getFunctionParametersFeatureComponent(func)
+      |
+        line, "\n" order by line
+      )
+  }
+
+  /**
+   * Gets a property initializer value in an object literal or one of its nested object literals.
+   */
+  Expr getANestedInitializerValue(ObjectExpr o) {
+    exists(Expr init | init = o.getAProperty().getInit().getUnderlyingValue() |
+      result = [init, getANestedInitializerValue(init)]
+    )
+  }
+
+  /**
+   * Computes a simple access path for how a callee can refer to a value that appears in an argument to a call.
+   *
+   * Supports:
+   * - direct arguments
+   * - properties of (nested) objects that are arguments
+   *
+   * Unknown cases and property names result in `?`.
+   */
+  string getSimpleParameterAccessPath(DataFlow::Node node) {
+    if exists(DataFlow::CallNode call | node = call.getArgument(_))
+    then exists(DataFlow::CallNode call, int i | node = call.getArgument(i) | result = i + "")
+    else result = getSimplePropertyAccessPath(node)
+  }
+
+  /**
+   * Computes a simple access path for how a user can refer to a value that appears in an (nested) object.
+   *
+   * Supports:
+   * - properties of (nested) objects
+   *
+   * Unknown cases and property names result in `?`.
+   */
+  string getSimplePropertyAccessPath(DataFlow::Node node) {
+    if exists(ObjectExpr o | o.getAProperty().getInit().getUnderlyingValue() = node.asExpr())
+    then
+      exists(DataFlow::PropWrite w |
+        w.getRhs() = node and
+        result = getSimpleParameterAccessPath(w.getBase()) + "." + getPropertyNameOrUnknown(w)
+      )
+    else result = getUnknownSymbol()
+  }
+
+  /**
+   * Gets the imported package path that this node depends on, if any.
+   *
+   * Otherwise, returns '?'.
+   *
+   * XXX Be careful with using this in your features, as it might teach the model
+   * a fixed list of "dangerous" libraries that could lead to bad generalization.
+   */
+  string getCalleeImportPath(DataFlow::Node node) {
+    exists(DataFlow::Node src | src = node.getALocalSource() |
+      if src instanceof DataFlow::ModuleImportNode
+      then result = src.(DataFlow::ModuleImportNode).getPath()
+      else
+        if src instanceof DataFlow::PropRead
+        then result = getCalleeImportPath(src.(DataFlow::PropRead).getBase())
+        else
+          if src instanceof DataFlow::InvokeNode
+          then result = getCalleeImportPath(src.(DataFlow::InvokeNode).getCalleeNode())
+          else
+            if src.asExpr() instanceof AwaitExpr
+            then result = getCalleeImportPath(src.asExpr().(AwaitExpr).getOperand().flow())
+            else result = getUnknownSymbol()
+    )
+  }
+
+  /**
+   * Computes a simple access path for a node.
+   *
+   * Supports:
+   * - variable reads (including `this` and `super`)
+   * - imports
+   * - await
+   * - property reads
+   * - invocations
+   *
+   * Unknown cases and property names results in `?`.
+   *
+   * # Examples
+   *
+   *  - The node `x.foo` will have the simple access path `x.foo`.
+   *  - In the following file, the simple access path will be `import("./foo").bar.baz`:
+   *
+   * ```javascript
+   * import * as lib from "./foo"
+   * console.log(lib.bar.baz());
+   * //          ^^^^^^^^^^^ node
+   */
+  string getSimpleAccessPath(DataFlow::Node node) {
+    exists(Expr e | e = node.asExpr().getUnderlyingValue() |
+      if
+        e instanceof SuperAccess or
+        e instanceof ThisAccess or
+        e instanceof VarAccess or
+        e instanceof Import or
+        e instanceof AwaitExpr or
+        node instanceof DataFlow::PropRead or
+        node instanceof DataFlow::InvokeNode
+      then
+        e instanceof SuperAccess and result = "super"
+        or
+        e instanceof ThisAccess and result = "this"
+        or
+        e instanceof VarAccess and result = e.(VarAccess).getName()
+        or
+        e instanceof Import and result = "import(" + getSimpleImportPath(e) + ")"
+        or
+        e instanceof AwaitExpr and
+        result = "(await " + getSimpleAccessPath(e.(AwaitExpr).getOperand().flow()) + ")"
+        or
+        node instanceof DataFlow::PropRead and
+        result =
+          getSimpleAccessPath(node.(DataFlow::PropRead).getBase()) + "." +
+            getPropertyNameOrUnknown(node)
+        or
+        (node instanceof DataFlow::InvokeNode and not e instanceof Import) and
+        result = getSimpleAccessPath(node.(DataFlow::InvokeNode).getCalleeNode()) + "()"
+      else result = getUnknownSymbol()
+    )
+  }
+
+  string getUnknownSymbol() { result = "?" }
+
+  /**
+   * Gets the imported path.
+   *
+   * XXX To avoid teaching the ML model about npm packages, only relative paths are supported
+   *
+   * Unknown paths result in `?`.
+   */
+  string getSimpleImportPath(Import i) {
+    if exists(i.getImportedPath().getValue())
+    then
+      exists(string p | p = i.getImportedPath().getValue() |
+        // Hide absolute imports from ML training data.
+        // ============================================
+        // There is the hypothesis that exposing absolute imports to the model
+        // might lead to bad generalization. For example, the model might learn
+        // to strongly associate a specific database client with sinks and no
+        // longer be able to flag sinks when data flow is broken.
+        // Placing this logic so deeply within the feature extraction code is
+        // perhaps a bit of a hack and it is a use case to consider when refactoring
+        // endpoint filters/data extraction.
+        if p.matches(".%") then result = "\"p\"" else result = "!"
+      )
+    else result = getUnknownSymbol()
+  }
+
+  /**
+   * Gets the property name of a property reference or `?` if it is unknown.
+   */
+  string getPropertyNameOrUnknown(DataFlow::PropRef ref) {
+    if exists(ref.getPropertyName())
+    then result = ref.getPropertyName()
+    else result = getUnknownSymbol()
+  }
+
+  /**
+   * Gets the parameter name if it exists, or `?` if it is unknown.
+   */
+  string getParameterNameOrUnknown(Parameter p) {
+    if exists(p.getName()) then result = p.getName() else result = getUnknownSymbol()
+  }
+}
+
+/**
+ * The feature for the access path of the callee node of a call that has an argument that "contains" the endpoint.
+ *
+ * "Containment" is syntactic, and currently means that the endpoint is an argument to the call, or that the endpoint is a (nested) property value of an argument.
+ *
+ * Examples:
+ * ```
+ * foo(endpoint); // -> foo
+ * foo.bar(endpoint); // -> foo.bar
+ * foo.bar({ baz: endpoint }); // -> foo.bar
+ * this.foo.bar(endpoint); // -> this.foo.bar
+ * foo[complex()].bar(endpoint); // -> foo.?.bar
+ * ```
+ */
+class CalleeFlexibleAccessPath extends EndpointFeature, TCalleeFlexibleAccessPath {
+  override string getName() { result = "CalleeFlexibleAccessPath" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    exists(DataFlow::InvokeNode invk |
+      result = SyntacticUtilities::getSimpleAccessPath(invk.getCalleeNode()) and
+      // ignore the unknown path
+      not result = SyntacticUtilities::getUnknownSymbol() and
+      (
+        invk.getAnArgument() = endpoint or
+        SyntacticUtilities::getANestedInitializerValue(invk.getAnArgument()
+              .asExpr()
+              .getUnderlyingValue()).flow() = endpoint
+      )
+    )
+  }
+}
+
+/**
+ * The feature for how a callee can refer to a the endpoint that is "contained" in some argument to a call
+ *
+ * "Containment" is syntactic, and currently means that the endpoint is an argument to the call, or that the endpoint is a (nested) property value of an argument.
+ *
+ * Examples:
+ * ```
+ * foo({ bar: endpoint }); // -> bar
+ * foo(x, { bar: { baz: endpoint } }); // -> bar.baz
+ * ```
+ */
+class InputAccessPathFromCallee extends EndpointFeature, TInputAccessPathFromCallee {
+  override string getName() { result = "InputAccessPathFromCallee" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    exists(DataFlow::InvokeNode invk |
+      result = SyntacticUtilities::getSimpleParameterAccessPath(endpoint) and
+      SyntacticUtilities::getANestedInitializerValue(invk.getAnArgument()
+            .asExpr()
+            .getUnderlyingValue()).flow() = endpoint
+    )
+  }
+}
+
+/**
+ * The feature for how the index of an argument that "contains" and endpoint.
+ *
+ * "Containment" is syntactic, and currently means that the endpoint is an argument to the call, or that the endpoint is a (nested) property value of an argument.
+ *
+ * Examples:
+ * ```
+ * foo(endpoint); // -> 0
+ * foo({ bar: endpoint }); // -> 0
+ * foo(x, { bar: { baz: endpoint } }); // -> 1
+ * ```
+ */
+class InputArgumentIndex extends EndpointFeature, TInputArgumentIndex {
+  override string getName() { result = "InputArgumentIndex" }
+
+  override string getValue(DataFlow::Node endpoint) {
+    exists(DataFlow::InvokeNode invk, DataFlow::Node arg, int i | arg = invk.getArgument(i) |
+      result = i + "" and
+      (
+        invk.getArgument(i) = endpoint
+        or
+        SyntacticUtilities::getANestedInitializerValue(arg.asExpr().getUnderlyingValue()).flow() =
+          endpoint
+      )
+    )
+  }
+}

javascript/ql/experimental/adaptivethreatmodeling/src/codeql-pack.lock.yml

@@ -1,6 +1,6 @@
 ---
 dependencies:
-  codeql/javascript-experimental-atm-model:
-    version: 0.2.0
+  dsp-testing/javascript-experimental-atm-model:
+    version: 0.2.1-2022-10-07-10h42m06s.nifty-feijoa-zg0q5y7b.e9244d8dec3fa9b20143187d29d5cf1dd625c4c66973088de08b82a8b193b660
 compiled: false
 lockVersion: 1.0.0