From 732e2c9f39228ffffb56a04883dc7efaf6fee5c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Apr 2026 08:09:17 +0000 Subject: [PATCH 01/90] Bump nlohmann_json from 3.11.3 to 3.12.0.bcr.1 Bumps [nlohmann_json](https://github.com/nlohmann/json) from 3.11.3 to 3.12.0.bcr.1. - [Release notes](https://github.com/nlohmann/json/releases) - [Changelog](https://github.com/nlohmann/json/blob/develop/ChangeLog.md) - [Commits](https://github.com/nlohmann/json/commits) --- updated-dependencies: - dependency-name: nlohmann_json dependency-version: 3.12.0.bcr.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 16b4a4691f8..6d93889d4a8 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -24,7 +24,7 @@ bazel_dep(name = "rules_python", version = "1.9.0") bazel_dep(name = "rules_shell", version = "0.7.1") bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "abseil-cpp", version = "20260107.1", repo_name = "absl") -bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json") +bazel_dep(name = "nlohmann_json", version = "3.12.0.bcr.1", repo_name = "json") bazel_dep(name = "fmt", version = "12.1.0-codeql.1") bazel_dep(name = "rules_kotlin", version = "2.2.2-codeql.1") bazel_dep(name = "gazelle", version = "0.50.0") From 872c08148ea75805897592414592ea4fe31d0503 Mon Sep 17 00:00:00 2001 From: yoff Date: Tue, 2 Jun 2026 14:09:28 +0000 Subject: [PATCH 02/90] Python: add shared-CFG AstSig adapter (AstNodeImpl) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Preparatory refactor for the shared-CFG dataflow migration. Adds the adapter that mediates between the Python AST and the shared codeql.controlflow.ControlFlowGraph signature, plus the test suites that validate the new CFG directly against this adapter. The public facade is added in the following commit. Library additions: - semmle.python.controlflow.internal.AstNodeImpl — wraps Python's Stmt/Expr/Scope/Pattern and adds two synthetic kinds of node (BlockStmt for body slots, intermediate nodes for multi-operand boolean expressions) to satisfy the shared CFG signature. - lib/ide-contextual-queries/printCfg.ql — the IDE "Print CFG" query, retargeted to the new CFG. - consistency-queries/CfgConsistency.ql — consistency query running the shared CFG's standard checks against Python. Test additions (all driven directly off AstNodeImpl): - ControlFlow/bindings/* — annotation-driven SSA-binding tests (annassign, compound, comprehension, decorated, except_handler, imports, match_pattern, parameters, simple, type_params, walrus_starred, with_stmt, dead_under_no_raise). - ControlFlow/evaluation-order/NewCfg*.ql — mirrors of the existing OldCfg evaluation-order self-validation suite, run against the new CFG via NewCfgImpl.qll. - Minor extensions to existing test_if.py / test_boolean.py + cosmetic .expected churn on a handful of OldCfg tests. No dataflow, SSA, or production query is migrated yet. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../ql/consistency-queries/CfgConsistency.ql | 2 + .../ql/lib/ide-contextual-queries/printCfg.ql | 42 + .../controlflow/internal/AstNodeImpl.qll | 1628 +++++++++++++++++ .../CONSISTENCY/CfgConsistency.expected | 4 + .../bindings/BindingsTest.expected | 0 .../ControlFlow/bindings/BindingsTest.ql | 32 + .../ControlFlow/bindings/annassign.py | 13 + .../ControlFlow/bindings/compound.py | 14 + .../ControlFlow/bindings/comprehension.py | 21 + .../bindings/dead_under_no_raise.py | 52 + .../ControlFlow/bindings/decorated.py | 30 + .../ControlFlow/bindings/except_handler.py | 19 + .../ControlFlow/bindings/imports.py | 14 + .../ControlFlow/bindings/match_pattern.py | 24 + .../ControlFlow/bindings/parameters.py | 42 + .../ControlFlow/bindings/simple.py | 14 + .../ControlFlow/bindings/type_params.py | 21 + .../ControlFlow/bindings/walrus_starred.py | 14 + .../ControlFlow/bindings/with_stmt.py | 21 + .../NewCfgAllLiveReachable.expected | 0 .../NewCfgAllLiveReachable.ql | 14 + .../NewCfgAnnotationHasCfgNode.expected | 1 + .../NewCfgAnnotationHasCfgNode.ql | 18 + .../NewCfgBasicBlockAnnotationGap.expected | 0 .../NewCfgBasicBlockAnnotationGap.ql | 26 + .../NewCfgBasicBlockOrdering.expected | 0 .../NewCfgBasicBlockOrdering.ql | 21 + .../NewCfgBranchTimestamps.expected | 0 .../NewCfgBranchTimestamps.ql | 80 + ...gConsecutivePredecessorTimestamps.expected | 1 + .../NewCfgConsecutivePredecessorTimestamps.ql | 22 + .../NewCfgConsecutiveTimestamps.expected | 0 .../NewCfgConsecutiveTimestamps.ql | 29 + .../evaluation-order/NewCfgImpl.qll | 120 ++ .../NewCfgNeverReachable.expected | 0 .../evaluation-order/NewCfgNeverReachable.ql | 21 + .../NewCfgNoBackwardFlow.expected | 0 .../evaluation-order/NewCfgNoBackwardFlow.ql | 22 + .../NewCfgNoBasicBlock.expected | 1 + .../evaluation-order/NewCfgNoBasicBlock.ql | 18 + .../NewCfgNoSharedReachable.expected | 0 .../NewCfgNoSharedReachable.ql | 21 + .../NewCfgStrictForward.expected | 0 .../evaluation-order/NewCfgStrictForward.ql | 22 + .../evaluation-order/OldCfgImpl.qll | 8 +- .../ControlFlow/evaluation-order/test_if.py | 2 +- 46 files changed, 2449 insertions(+), 5 deletions(-) create mode 100644 python/ql/consistency-queries/CfgConsistency.ql create mode 100644 python/ql/lib/ide-contextual-queries/printCfg.ql create mode 100644 python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll create mode 100644 python/ql/test/extractor-tests/syntax_error/CONSISTENCY/CfgConsistency.expected create mode 100644 python/ql/test/library-tests/ControlFlow/bindings/BindingsTest.expected create mode 100644 python/ql/test/library-tests/ControlFlow/bindings/BindingsTest.ql create mode 100644 python/ql/test/library-tests/ControlFlow/bindings/annassign.py create mode 100644 python/ql/test/library-tests/ControlFlow/bindings/compound.py create mode 100644 python/ql/test/library-tests/ControlFlow/bindings/comprehension.py create mode 100644 python/ql/test/library-tests/ControlFlow/bindings/dead_under_no_raise.py create mode 100644 python/ql/test/library-tests/ControlFlow/bindings/decorated.py create mode 100644 python/ql/test/library-tests/ControlFlow/bindings/except_handler.py create mode 100644 python/ql/test/library-tests/ControlFlow/bindings/imports.py create mode 100644 python/ql/test/library-tests/ControlFlow/bindings/match_pattern.py create mode 100644 python/ql/test/library-tests/ControlFlow/bindings/parameters.py create mode 100644 python/ql/test/library-tests/ControlFlow/bindings/simple.py create mode 100644 python/ql/test/library-tests/ControlFlow/bindings/type_params.py create mode 100644 python/ql/test/library-tests/ControlFlow/bindings/walrus_starred.py create mode 100644 python/ql/test/library-tests/ControlFlow/bindings/with_stmt.py create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAllLiveReachable.expected create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAllLiveReachable.ql create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAnnotationHasCfgNode.expected create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAnnotationHasCfgNode.ql create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.expected create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.ql create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.expected create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.ql create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBranchTimestamps.expected create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBranchTimestamps.ql create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.expected create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.ql create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.expected create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.ql create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgImpl.qll create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.expected create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.ql create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.expected create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.ql create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBasicBlock.expected create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBasicBlock.ql create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.expected create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.ql create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.expected create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.ql diff --git a/python/ql/consistency-queries/CfgConsistency.ql b/python/ql/consistency-queries/CfgConsistency.ql new file mode 100644 index 00000000000..ab13eddf190 --- /dev/null +++ b/python/ql/consistency-queries/CfgConsistency.ql @@ -0,0 +1,2 @@ +import semmle.python.controlflow.internal.AstNodeImpl +import ControlFlow::Consistency diff --git a/python/ql/lib/ide-contextual-queries/printCfg.ql b/python/ql/lib/ide-contextual-queries/printCfg.ql new file mode 100644 index 00000000000..6e325e84bb7 --- /dev/null +++ b/python/ql/lib/ide-contextual-queries/printCfg.ql @@ -0,0 +1,42 @@ +/** + * @name Print CFG + * @description Produces a representation of a file's Control Flow Graph. + * This query is used by the VS Code extension. + * @id py/print-cfg + * @kind graph + * @tags ide-contextual-queries/print-cfg + */ + +import semmle.python.Files as Files +// import semmle.python.Scope +import semmle.python.controlflow.internal.AstNodeImpl + +external string selectedSourceFile(); + +private predicate selectedSourceFileAlias = selectedSourceFile/0; + +external int selectedSourceLine(); + +private predicate selectedSourceLineAlias = selectedSourceLine/0; + +external int selectedSourceColumn(); + +private predicate selectedSourceColumnAlias = selectedSourceColumn/0; + +module ViewCfgQueryInput implements ControlFlow::ViewCfgQueryInputSig { + predicate selectedSourceFile = selectedSourceFileAlias/0; + + predicate selectedSourceLine = selectedSourceLineAlias/0; + + predicate selectedSourceColumn = selectedSourceColumnAlias/0; + + predicate cfgScopeSpan( + Ast::Callable scope, Files::File file, int startLine, int startColumn, int endLine, + int endColumn + ) { + file = scope.getLocation().getFile() and + scope.getLocation().hasLocationInfo(_, startLine, startColumn, endLine, endColumn) + } +} + +import ControlFlow::ViewCfgQuery diff --git a/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll b/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll new file mode 100644 index 00000000000..5d87b16f351 --- /dev/null +++ b/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll @@ -0,0 +1,1628 @@ +/** + * Provides classes for the shared control-flow library, mediating between + * the Python AST and `AstSig`. + * + * The `Ast` module wraps Python's `Stmt`, `Expr`, `Scope`, and `Pattern`, + * and adds two synthetic kinds of node: + * - `BlockStmt`, identifying a body slot of a parent AST node (e.g. an + * `if`'s then or else branch). `Py::StmtList` itself is not directly + * wrapped. + * - Intermediate nodes for multi-operand boolean expressions. + */ +overlay[local?] +module; + +private import python as Py +private import codeql.controlflow.ControlFlowGraph +private import codeql.controlflow.SuccessorType +private import codeql.util.Void + +/** + * Gets the bound `Name` of a PEP 695 type parameter (`TypeVar`, + * `ParamSpec`, or `TypeVarTuple`). The base `TypeParameter` class does + * not expose `getName()`; this helper dispatches over the subtypes. + */ +private Py::Name typeParameterName(Py::TypeParameter tp) { + result = tp.(Py::TypeVar).getName() + or + result = tp.(Py::ParamSpec).getName() + or + result = tp.(Py::TypeVarTuple).getName() +} + +/** Provides the Python implementation of the shared CFG `AstSig`. */ +module Ast implements AstSig { + private newtype TAstNode = + TPyStmt(Py::Stmt s) or + TPyExpr(Py::Expr e) { not e instanceof Py::BoolExpr } or + TScope(Py::Scope sc) or + TPattern(Py::Pattern p) or + /** + * A synthetic node representing an operand pair of an `and`/`or` + * expression. For `a and b and c` (operands 0, 1, 2) we model the + * operation as a right-nested tree: pair 0 represents the whole + * expression with left=a and right=pair 1; pair 1 represents + * `b and c` with left=b and right=c. Each Python `Py::BoolExpr` + * with `n` operands has `n - 1` such pairs (indices `0 .. n - 2`). + */ + TBoolExprPair(Py::BoolExpr be, int index) { index = [0 .. count(be.getAValue()) - 2] } or + /** + * A synthetic block statement, wrapping a `Py::StmtList`. Each list of + * statements that represents an imperative block (a function/class/module + * body, an `if`/`while`/`for` branch, a `try`/`except`/`finally` body, + * etc.) becomes one `BlockStmt` node in the CFG. `Py::StmtList`s used + * in other roles - `Try.getHandlers()` (iterated via `getCatch`) and + * `MatchStmt.getCases()` (iterated via `getCase`) - are excluded, as + * the shared library's `Try`/`Switch` logic walks their items + * individually. + */ + TBlockStmt(Py::StmtList sl) { + not sl = any(Py::Try t).getHandlers() and + not sl = any(Py::MatchStmt m).getCases() + } + + /** + * The union of `TPyStmt` (wrapping `Py::Stmt`) and `TBlockStmt` (wrapping + * `Py::StmtList`). Both represent the kinds of node that can appear in + * a `Stmt` position in the CFG. + */ + private class TStmt = TPyStmt or TBlockStmt; + + /** + * The union of `TPyExpr` (wrapping non-boolean `Py::Expr`) and + * `TBoolExprPair` (synthetic operand pairs of `and`/`or` expressions). + * Both represent the kinds of node that can appear in an `Expr` + * position in the CFG. + */ + private class TExpr = TPyExpr or TBoolExprPair; + + /** + * An AST node visible to the shared CFG. + * + * This is the abstract implementation class. It enforces that each + * concrete subclass provides `toString`, `getLocation`, and + * `getEnclosingCallable` (one subclass per `TAstNode` newtype branch). + * The public alias `AstNode` is what users (and the `AstSig` signature) + * see; subclasses inside this module extend `AstNodeImpl` directly. + */ + abstract private class AstNodeImpl extends TAstNode { + /** Gets a textual representation of this AST node. */ + abstract string toString(); + + /** Gets the location of this AST node. */ + abstract Py::Location getLocation(); + + /** Gets the enclosing callable that contains this node, if any. */ + abstract Callable getEnclosingCallable(); + + /** Gets the underlying Python `Stmt`, if this node wraps one. */ + Py::Stmt asStmt() { this = TPyStmt(result) } + + /** + * Gets the underlying Python `Expr`, if this node wraps one. Boolean + * expressions are represented by `TBoolExprPair(_, 0)`; this + * predicate also recovers the underlying `Py::BoolExpr` from such a + * representation. + */ + Py::Expr asExpr() { + this = TPyExpr(result) + or + this = TBoolExprPair(result, 0) + } + + /** Gets the underlying Python `Scope`, if this node wraps one. */ + Py::Scope asScope() { this = TScope(result) } + + /** Gets the underlying Python `Pattern`, if this node wraps one. */ + Py::Pattern asPattern() { this = TPattern(result) } + + /** Gets the underlying Python `StmtList`, if this node is a `BlockStmt`. */ + Py::StmtList asStmtList() { this = TBlockStmt(result) } + + /** + * Gets the child of this AST node at the specified (zero-based) + * index, in evaluation order. Subclasses with children override + * this method. + */ + AstNode getChild(int index) { none() } + } + + /** An AST node visible to the shared CFG. */ + final class AstNode = AstNodeImpl; + + /** Gets the immediately enclosing callable that contains `node`. */ + Callable getEnclosingCallable(AstNode node) { result = node.getEnclosingCallable() } + + /** + * A callable: a function, class, or module scope. + * + * In Python, all three are executable scopes with statement bodies. + */ + class Callable extends AstNodeImpl, TScope { + private Py::Scope sc; + + Callable() { this = TScope(sc) } + + override string toString() { result = sc.toString() } + + override Py::Location getLocation() { result = sc.getLocation() } + + override Callable getEnclosingCallable() { result.asScope() = sc.getEnclosingScope() } + } + + /** Gets the body of callable `c`. */ + AstNode callableGetBody(Callable c) { result.asStmtList() = c.asScope().getBody() } + + /** + * A parameter of a callable. + * + * Modelled per the C# template (`csharp/.../ControlFlowGraph.qll:147-156`): + * each Python parameter (the `Py::Parameter` AST node, which is a `Name` + * or — Python 2 only — a `Tuple` in store context) becomes a CFG node + * at a stable position in the enclosing callable's entry sequence. + * + * Default-value expressions for positional and keyword-only parameters + * are wired separately on the `FunctionDefExpr` / `LambdaExpr` wrappers + * (they evaluate at function-definition time, not at call time). + * `Parameter::getDefaultValue()` returns `none()` here, signalling to + * the shared library that the parameter never falls back to a default + * during call binding. This mirrors C# for non-optional parameters. + */ + class Parameter extends Expr { + private Py::Parameter param; + + Parameter() { this = TPyExpr(param) } + + /** Gets the underlying Python parameter. */ + Py::Parameter asParameter() { result = param } + + /** + * Gets the default-value expression of this parameter, if any. + * + * Returns `none()`: defaults evaluate at function-definition time and + * are wired into the CFG via `FunctionDefExpr.getDefault` / + * `LambdaExpr.getDefault`. The shared library calls this predicate + * to model the "missing argument → evaluate default" fallback during + * call binding, which Python does not model at the CFG level. + */ + Expr getDefaultValue() { none() } + + /** + * Gets the pattern for this parameter. In Python, there is no destructuring + * pattern syntax for parameters, so the pattern is the parameter itself. + */ + AstNode getPattern() { result = this } + } + + /** + * Gets the `index`th parameter of callable `c`, ordered as Python binds + * them at call time: positional, then vararg (`*args`), then + * keyword-only, then kwarg (`**kwargs`). + */ + Parameter callableGetParameter(Callable c, int index) { + exists(Py::Function f | f = c.asScope() | + result.asParameter() = + rank[index + 1](Py::Parameter p, int subOrder, int subIndex | + // positional parameters first + p = f.getArg(subIndex) and subOrder = 0 + or + // then *args + p = f.getVararg() and subOrder = 1 and subIndex = 0 + or + // then keyword-only parameters + p = f.getKeywordOnlyArg(subIndex) and subOrder = 2 + or + // finally **kwargs + p = f.getKwarg() and subOrder = 3 and subIndex = 0 + | + p order by subOrder, subIndex + ) + ) + } + + /** A statement. */ + class Stmt extends AstNodeImpl, TStmt { + // For `TPyStmt` instances, delegate to the wrapped Python statement. + // `BlockStmt` (the only `TBlockStmt` subclass) provides its own overrides. + override string toString() { result = this.asStmt().toString() } + + override Py::Location getLocation() { result = this.asStmt().getLocation() } + + override Callable getEnclosingCallable() { result.asScope() = this.asStmt().getScope() } + } + + /** An expression. */ + class Expr extends AstNodeImpl, TExpr { + // For `TPyExpr` instances, delegate to the wrapped Python expression. + // `BinaryExpr` (the only `TBoolExprPair` subclass) provides its own overrides. + override string toString() { result = this.asExpr().toString() } + + override Py::Location getLocation() { result = this.asExpr().getLocation() } + + override Callable getEnclosingCallable() { result.asScope() = this.asExpr().getScope() } + } + + /** A pattern in a `match` statement. */ + additional class Pattern extends AstNodeImpl, TPattern { + private Py::Pattern p; + + Pattern() { this = TPattern(p) } + + override string toString() { result = p.toString() } + + override Py::Location getLocation() { result = p.getLocation() } + + override Callable getEnclosingCallable() { result.asScope() = p.getScope() } + } + + /** + * A `case x` pattern that binds `x` to the matched value. + */ + additional class MatchCapturePattern extends Pattern { + private Py::MatchCapturePattern cap; + + MatchCapturePattern() { this = TPattern(cap) } + + /** Gets the bound Name expression. */ + Expr getVariable() { result.asExpr() = cap.getVariable() } + + override AstNode getChild(int index) { index = 0 and result = this.getVariable() } + } + + /** + * A `case pattern as name` pattern. + */ + additional class MatchAsPattern extends Pattern { + private Py::MatchAsPattern asp; + + MatchAsPattern() { this = TPattern(asp) } + + /** Gets the inner pattern. */ + AstNode getPattern() { result.asPattern() = asp.getPattern() } + + /** Gets the bound Name expression. */ + Expr getAlias() { result.asExpr() = asp.getAlias() } + + override AstNode getChild(int index) { + index = 0 and result = this.getPattern() + or + index = 1 and result = this.getAlias() + } + } + + /** + * A `case [a, b, *rest]` star pattern. Binds `rest` to the remaining + * elements of the sequence. + */ + additional class MatchStarPattern extends Pattern { + private Py::MatchStarPattern starp; + + MatchStarPattern() { this = TPattern(starp) } + + /** Gets the target Pattern (a `MatchCapturePattern` if `*rest`). */ + AstNode getTarget() { result.asPattern() = starp.getTarget() } + + override AstNode getChild(int index) { index = 0 and result = this.getTarget() } + } + + /** + * A `case [a, b, ...]` sequence pattern. Recurses into the sub-patterns. + */ + additional class MatchSequencePattern extends Pattern { + private Py::MatchSequencePattern seqp; + + MatchSequencePattern() { this = TPattern(seqp) } + + /** Gets the `n`th sub-pattern. */ + AstNode getPattern(int n) { result.asPattern() = seqp.getPattern(n) } + + override AstNode getChild(int index) { result = this.getPattern(index) } + } + + /** + * A `case Cls(a, b, x=y)` class pattern. + */ + additional class MatchClassPattern extends Pattern { + private Py::MatchClassPattern clsp; + + MatchClassPattern() { this = TPattern(clsp) } + + /** Gets the class expression of this class pattern. */ + Expr getClass() { result.asExpr() = clsp.getClass() } + + /** Gets the `n`th positional sub-pattern. */ + AstNode getPositional(int n) { result.asPattern() = clsp.getPositional(n) } + + /** Gets the `n`th keyword sub-pattern. */ + AstNode getKeyword(int n) { result.asPattern() = clsp.getKeyword(n) } + + private int numPositional() { result = count(int i | exists(clsp.getPositional(i))) } + + override AstNode getChild(int index) { + index = 0 and result = this.getClass() + or + result = this.getPositional(index - 1) and index >= 1 + or + result = this.getKeyword(index - 1 - this.numPositional()) and + index >= 1 + this.numPositional() + } + } + + /** + * A `case {k: v}` mapping pattern. + */ + additional class MatchMappingPattern extends Pattern { + private Py::MatchMappingPattern mapp; + + MatchMappingPattern() { this = TPattern(mapp) } + + AstNode getMapping(int n) { result.asPattern() = mapp.getMapping(n) } + + override AstNode getChild(int index) { result = this.getMapping(index) } + } + + /** + * A key-value pair inside a `case {k: v}` mapping pattern. + */ + additional class MatchKeyValuePattern extends Pattern { + private Py::MatchKeyValuePattern kvp; + + MatchKeyValuePattern() { this = TPattern(kvp) } + + AstNode getKey() { result.asPattern() = kvp.getKey() } + + AstNode getValue() { result.asPattern() = kvp.getValue() } + + override AstNode getChild(int index) { + index = 0 and result = this.getKey() + or + index = 1 and result = this.getValue() + } + } + + /** + * A `case Cls(name=value)` keyword sub-pattern. + */ + additional class MatchKeywordPattern extends Pattern { + private Py::MatchKeywordPattern kwp; + + MatchKeywordPattern() { this = TPattern(kwp) } + + Expr getAttribute() { result.asExpr() = kwp.getAttribute() } + + AstNode getValue() { result.asPattern() = kwp.getValue() } + + override AstNode getChild(int index) { + index = 0 and result = this.getAttribute() + or + index = 1 and result = this.getValue() + } + } + + /** A `case **rest` double-star mapping sub-pattern. */ + additional class MatchDoubleStarPattern extends Pattern { + private Py::MatchDoubleStarPattern dsp; + + MatchDoubleStarPattern() { this = TPattern(dsp) } + + AstNode getTarget() { result.asPattern() = dsp.getTarget() } + + override AstNode getChild(int index) { index = 0 and result = this.getTarget() } + } + + /** A `case p1 | p2 | …` or-pattern. */ + additional class MatchOrPattern extends Pattern { + private Py::MatchOrPattern orp; + + MatchOrPattern() { this = TPattern(orp) } + + AstNode getPattern(int n) { result.asPattern() = orp.getPattern(n) } + + override AstNode getChild(int index) { result = this.getPattern(index) } + } + + /** A `case 1` literal pattern. */ + additional class MatchLiteralPattern extends Pattern { + private Py::MatchLiteralPattern litp; + + MatchLiteralPattern() { this = TPattern(litp) } + + Expr getLiteral() { result.asExpr() = litp.getLiteral() } + + override AstNode getChild(int index) { index = 0 and result = this.getLiteral() } + } + + /** A `case Cls.NAME` value pattern. */ + additional class MatchValuePattern extends Pattern { + private Py::MatchValuePattern vp; + + MatchValuePattern() { this = TPattern(vp) } + + Expr getValue() { result.asExpr() = vp.getValue() } + + override AstNode getChild(int index) { index = 0 and result = this.getValue() } + } + + /** + * A block statement, modeling the body of a parent AST node as a + * sequence of statements. + */ + class BlockStmt extends Stmt, TBlockStmt { + private Py::StmtList sl; + + BlockStmt() { this = TBlockStmt(sl) } + + /** Gets the `n`th (zero-based) statement in this block. */ + Stmt getStmt(int n) { result.asStmt() = sl.getItem(n) } + + /** Gets the last statement in this block. */ + Stmt getLastStmt() { result.asStmt() = sl.getLastItem() } + + override string toString() { result = sl.toString() } + + // `Py::StmtList` has no native location; approximate with the first + // item's location. + override Py::Location getLocation() { result = sl.getItem(0).getLocation() } + + override Callable getEnclosingCallable() { + result.asScope() = sl.getParent().(Py::Scope) + or + result.asScope() = sl.getParent().(Py::Stmt).getScope() + } + + override AstNode getChild(int index) { result = this.getStmt(index) } + } + + /** An expression statement. */ + class ExprStmt extends Stmt { + private Py::ExprStmt exprStmt; + + ExprStmt() { this = TPyStmt(exprStmt) } + + /** Gets the expression in this expression statement. */ + Expr getExpr() { result.asExpr() = exprStmt.getValue() } + + override AstNode getChild(int index) { index = 0 and result = this.getExpr() } + } + + /** An assignment statement (`x = y = expr`). */ + additional class AssignStmt extends Stmt { + private Py::Assign assign; + + AssignStmt() { this = TPyStmt(assign) } + + Expr getValue() { result.asExpr() = assign.getValue() } + + Expr getTarget(int n) { result.asExpr() = assign.getTarget(n) } + + int getNumberOfTargets() { result = count(assign.getATarget()) } + + override AstNode getChild(int index) { + index = 0 and result = this.getValue() + or + result = this.getTarget(index - 1) and index >= 1 + } + } + + /** An augmented assignment statement (`x += expr`). */ + additional class AugAssignStmt extends Stmt { + private Py::AugAssign augAssign; + + AugAssignStmt() { this = TPyStmt(augAssign) } + + Expr getOperation() { result.asExpr() = augAssign.getOperation() } + + override AstNode getChild(int index) { index = 0 and result = this.getOperation() } + } + + /** + * An annotated assignment statement (`x: T = expr`, or `x: T` without + * value). The evaluation order follows CPython: annotation first, then + * the optional value, then the target binding. + */ + additional class AnnAssignStmt extends Stmt { + private Py::AnnAssign annAssign; + + AnnAssignStmt() { this = TPyStmt(annAssign) } + + Expr getAnnotation() { result.asExpr() = annAssign.getAnnotation() } + + Expr getValue() { result.asExpr() = annAssign.getValue() } + + Expr getTarget() { result.asExpr() = annAssign.getTarget() } + + override AstNode getChild(int index) { + index = 0 and result = this.getAnnotation() + or + index = 1 and result = this.getValue() + or + index = 2 and result = this.getTarget() + } + } + + /** An assignment expression / walrus operator (`x := expr`). */ + additional class NamedExpr extends Expr { + private Py::AssignExpr assignExpr; + + NamedExpr() { this = TPyExpr(assignExpr) } + + Expr getValue() { result.asExpr() = assignExpr.getValue() } + + Expr getTarget() { result.asExpr() = assignExpr.getTarget() } + + override AstNode getChild(int index) { + index = 0 and result = this.getValue() + or + index = 1 and result = this.getTarget() + } + } + + /** + * An `if` statement. + * + * Python's `elif` chains are represented as nested `If` nodes in the + * else branch's `StmtList`. The shared CFG library handles this + * naturally: `getElse()` returns the `BlockStmt` wrapping the else + * branch, and if that block contains a single `If`, the result is + * a chained conditional. + */ + class IfStmt extends Stmt { + private Py::If ifStmt; + + IfStmt() { this = TPyStmt(ifStmt) } + + /** Gets the underlying Python `If` statement. */ + Py::If asIf() { result = ifStmt } + + /** Gets the condition of this `if` statement. */ + Expr getCondition() { result.asExpr() = ifStmt.getTest() } + + /** Gets the `then` (true) branch of this `if` statement. */ + Stmt getThen() { result.asStmtList() = ifStmt.getBody() } + + /** Gets the `else` (false) branch, if any. */ + Stmt getElse() { result.asStmtList() = ifStmt.getOrelse() } + + override AstNode getChild(int index) { + index = 0 and result = this.getCondition() + or + index = 1 and result = this.getThen() + or + index = 2 and result = this.getElse() + } + } + + /** A loop statement. */ + class LoopStmt extends Stmt { + LoopStmt() { + this = TPyStmt(any(Py::While w)) + or + this = TPyStmt(any(Py::For f)) + } + + /** Gets the body of this loop statement. */ + Stmt getBody() { none() } + } + + /** A `while` loop statement. */ + class WhileStmt extends LoopStmt { + private Py::While whileStmt; + + WhileStmt() { this = TPyStmt(whileStmt) } + + /** Gets the boolean condition of this `while` loop. */ + Expr getCondition() { result.asExpr() = whileStmt.getTest() } + + override Stmt getBody() { result.asStmtList() = whileStmt.getBody() } + + /** Gets the `else` branch of this `while` loop, if any. */ + Stmt getElse() { result.asStmtList() = whileStmt.getOrelse() } + + override AstNode getChild(int index) { + index = 0 and result = this.getCondition() + or + index = 1 and result = this.getBody() + or + index = 2 and result = this.getElse() + } + } + + /** + * A `do-while` loop statement. Python has no do-while construct. + */ + class DoStmt extends LoopStmt { + DoStmt() { none() } + + Expr getCondition() { none() } + } + + /** An `until` loop. Python has no `until` loop. */ + class UntilStmt extends LoopStmt { + UntilStmt() { none() } + + Expr getCondition() { none() } + } + + /** A C-style `for` loop. Python has no C-style for loop. */ + class ForStmt extends LoopStmt { + ForStmt() { none() } + + AstNode getInit(int index) { none() } + + Expr getCondition() { none() } + + AstNode getUpdate(int index) { none() } + } + + /** A for-each loop (`for x in iterable:`). */ + class ForeachStmt extends LoopStmt { + private Py::For forStmt; + + ForeachStmt() { this = TPyStmt(forStmt) } + + /** Gets the loop variable. */ + Expr getVariable() { result.asExpr() = forStmt.getTarget() } + + /** Gets the collection being iterated. */ + Expr getCollection() { result.asExpr() = forStmt.getIter() } + + override Stmt getBody() { result.asStmtList() = forStmt.getBody() } + + /** Gets the `else` branch of this `for` loop, if any. */ + Stmt getElse() { result.asStmtList() = forStmt.getOrelse() } + + override AstNode getChild(int index) { + index = 0 and result = this.getCollection() + or + index = 1 and result = this.getVariable() + or + index = 2 and result = this.getBody() + or + index = 3 and result = this.getElse() + } + } + + /** A `break` statement. */ + class BreakStmt extends Stmt { + BreakStmt() { this = TPyStmt(any(Py::Break b)) } + } + + /** A `continue` statement. */ + class ContinueStmt extends Stmt { + ContinueStmt() { this = TPyStmt(any(Py::Continue c)) } + } + + /** A `goto` statement. Python has no goto. */ + class GotoStmt extends Stmt { + GotoStmt() { none() } + } + + /** A `return` statement. */ + class ReturnStmt extends Stmt { + private Py::Return ret; + + ReturnStmt() { this = TPyStmt(ret) } + + /** Gets the expression being returned, if any. */ + Expr getExpr() { result.asExpr() = ret.getValue() } + + override AstNode getChild(int index) { index = 0 and result = this.getExpr() } + } + + /** A `raise` statement (mapped to `Throw`). */ + class Throw extends Stmt { + private Py::Raise raise; + + Throw() { this = TPyStmt(raise) } + + /** Gets the expression being raised. */ + Expr getExpr() { result.asExpr() = raise.getException() } + + /** Gets the cause of this `raise`, if any. */ + Expr getCause() { result.asExpr() = raise.getCause() } + + override AstNode getChild(int index) { + index = 0 and result = this.getExpr() + or + index = 1 and result = this.getCause() + } + } + + /** + * An `import` statement (`import a, b` or `from m import a, b`). + * + * Each alias contributes two children in evaluation order: first the + * value expression (which performs the import side-effect), then the + * bound `asname` Name (the in-scope binding). This makes both reachable + * from the CFG and allows `Name.defines(v)` for `asname` Names to have + * corresponding CFG nodes — which is essential for SSA to see import + * bindings. + */ + additional class ImportStmt extends Stmt { + private Py::Import imp; + + ImportStmt() { this = TPyStmt(imp) } + + /** Gets the value (module/member expression) of the `n`th alias. */ + Expr getValue(int n) { result.asExpr() = imp.getName(n).getValue() } + + /** Gets the bound `asname` of the `n`th alias. */ + Expr getAsname(int n) { result.asExpr() = imp.getName(n).getAsname() } + + /** Gets the number of aliases in this import statement. */ + int getNumberOfAliases() { result = count(int i | exists(imp.getName(i))) } + + override AstNode getChild(int index) { + exists(int i | + index = 2 * i and result = this.getValue(i) + or + index = 2 * i + 1 and result = this.getAsname(i) + ) + } + } + + /** + * A `from m import *` statement. Evaluates the module expression but + * binds no name (the bindings happen by side-effect at runtime, which + * is not modelled at the CFG level). + */ + additional class ImportStarStmt extends Stmt { + private Py::ImportStar imp; + + ImportStarStmt() { this = TPyStmt(imp) } + + Expr getModule() { result.asExpr() = imp.getModule() } + + override AstNode getChild(int index) { index = 0 and result = this.getModule() } + } + + /** A `with` statement. */ + additional class WithStmt extends Stmt { + private Py::With withStmt; + + WithStmt() { this = TPyStmt(withStmt) } + + Expr getContextExpr() { result.asExpr() = withStmt.getContextExpr() } + + Expr getOptionalVars() { result.asExpr() = withStmt.getOptionalVars() } + + Stmt getBody() { result.asStmtList() = withStmt.getBody() } + + override AstNode getChild(int index) { + index = 0 and result = this.getContextExpr() + or + index = 1 and result = this.getOptionalVars() + or + index = 2 and result = this.getBody() + } + } + + /** An `assert` statement. */ + additional class AssertStmt extends Stmt { + private Py::Assert assertStmt; + + AssertStmt() { this = TPyStmt(assertStmt) } + + Expr getTest() { result.asExpr() = assertStmt.getTest() } + + Expr getMsg() { result.asExpr() = assertStmt.getMsg() } + + override AstNode getChild(int index) { + index = 0 and result = this.getTest() + or + index = 1 and result = this.getMsg() + } + } + + /** A `delete` statement. */ + additional class DeleteStmt extends Stmt { + private Py::Delete del; + + DeleteStmt() { this = TPyStmt(del) } + + Expr getTarget(int n) { result.asExpr() = del.getTarget(n) } + + override AstNode getChild(int index) { result = this.getTarget(index) } + } + + /** + * A PEP 695 `type` statement (`type Alias[T1, T2] = value`). + * + * The type parameters bind at statement-evaluation time. The value + * expression is captured for lazy evaluation but the alias `Name` + * itself binds the resulting `TypeAliasType` object — so the CFG must + * visit at minimum the type-parameter names and the alias name. + */ + additional class TypeAliasStmt extends Stmt { + private Py::TypeAlias ta; + + TypeAliasStmt() { this = TPyStmt(ta) } + + /** Gets the alias `Name` bound by this statement. */ + Expr getName() { result.asExpr() = ta.getName() } + + /** + * Gets the `n`th PEP 695 type-parameter name (a `Name` in store + * context), in declaration order. + */ + Expr getTypeParamName(int n) { result.asExpr() = typeParameterName(ta.getTypeParameter(n)) } + + int getNumberOfTypeParams() { result = count(ta.getATypeParameter()) } + + override AstNode getChild(int index) { + result = this.getTypeParamName(index) + or + index = this.getNumberOfTypeParams() and result = this.getName() + } + } + + /** A `try` statement. */ + class TryStmt extends Stmt { + private Py::Try tryStmt; + + TryStmt() { this = TPyStmt(tryStmt) } + + AstNode getBody(int index) { index = 0 and result.asStmtList() = tryStmt.getBody() } + + /** Gets the `else` branch of this `try` statement, if any. */ + Stmt getElse() { result.asStmtList() = tryStmt.getOrelse() } + + Stmt getFinally() { result.asStmtList() = tryStmt.getFinalbody() } + + CatchClause getCatch(int index) { result.asStmt() = tryStmt.getHandler(index) } + + override AstNode getChild(int index) { + index = 0 and result = this.getBody(0) + or + result = this.getCatch(index - 1) and index >= 1 + or + index = -1 and result = this.getFinally() + or + index = -2 and result = this.getElse() + } + } + + /** + * Gets the `else` branch of `try` statement `try`, if any. + */ + AstNode getTryElse(TryStmt try) { result = try.getElse() } + + /** + * Gets the `else` branch of loop `loop`, if any. + * + * Python's `while`/`for` loops may have an `else` block that runs when the + * loop completes without `break`. + */ + AstNode getLoopElse(LoopStmt loop) { + result = loop.(WhileStmt).getElse() + or + result = loop.(ForeachStmt).getElse() + } + + /** An exception handler (`except` or `except*`). */ + class CatchClause extends Stmt { + private Py::ExceptionHandler handler; + + CatchClause() { this = TPyStmt(handler) } + + /** + * Gets the type-test pattern of this exception handler, if any. + * + * This is the single syntactic type expression (`except TypeError:` → + * `TypeError`; `except (A, B):` → the `(A, B)` tuple). A bare `except:` + * has no pattern and is therefore a catch-all that matches any + * exception. The type test and the variable binding are independent + * child nodes in Python, matching the shared CFG model. + * + * We read the raw type child directly (extractor relation `py_exprs` + * at child index 1) rather than the public `getType()` accessor: the + * latter flattens a tuple of exception types (`except (A, B):`) into + * its individual elements, which would yield several patterns for one + * handler and violate the shared CFG's single-pattern-per-catch + * contract. The raw child is the one tuple node, modelling the + * runtime's single `isinstance(exc, (A, B))` test. + */ + AstNode getPattern() { + exists(Py::Expr rawType | py_exprs(rawType, _, handler, 1) | result.asExpr() = rawType) + } + + /** Gets the variable name of this exception handler, if any. */ + AstNode getVariable() { result.asExpr() = handler.getName() } + + /** Holds: catch clauses do not have a `Condition` in Python's model. */ + Expr getCondition() { none() } + + /** Gets the body of this exception handler. */ + Stmt getBody() { + result.asStmtList() = handler.(Py::ExceptStmt).getBody() + or + result.asStmtList() = handler.(Py::ExceptGroupStmt).getBody() + } + + override AstNode getChild(int index) { + index = 0 and result = this.getPattern() + or + index = 1 and result = this.getVariable() + or + index = 2 and result = this.getBody() + } + } + + /** A `match` statement, mapped to the shared CFG's `Switch`. */ + class Switch extends Stmt { + private Py::MatchStmt matchStmt; + + Switch() { this = TPyStmt(matchStmt) } + + Expr getExpr() { result.asExpr() = matchStmt.getSubject() } + + Case getCase(int index) { result.asStmt() = matchStmt.getCase(index) } + + Stmt getStmt(int index) { none() } + + override AstNode getChild(int index) { + index = 0 and result = this.getExpr() + or + result = this.getCase(index - 1) and index >= 1 + } + } + + /** A `case` clause in a match statement. */ + class Case extends Stmt { + private Py::Case caseStmt; + + Case() { this = TPyStmt(caseStmt) } + + AstNode getPattern(int index) { index = 0 and result.asPattern() = caseStmt.getPattern() } + + Expr getGuard() { result.asExpr() = caseStmt.getGuard().(Py::Guard).getTest() } + + AstNode getBody() { result.asStmtList() = caseStmt.getBody() } + + /** Holds if this case is a wildcard pattern (`case _:`). */ + predicate isWildcard() { caseStmt.getPattern() instanceof Py::MatchWildcardPattern } + + override AstNode getChild(int index) { + index = 0 and result = this.getPattern(0) + or + index = 1 and result = this.getGuard() + or + index = 2 and result = this.getBody() + } + } + + /** A wildcard case (`case _:`). */ + class DefaultCase extends Case { + DefaultCase() { this.isWildcard() } + } + + /** A conditional expression (`x if cond else y`). */ + class ConditionalExpr extends Expr { + private Py::IfExp ifExp; + + ConditionalExpr() { this = TPyExpr(ifExp) } + + /** Gets the condition of this expression. */ + Expr getCondition() { result.asExpr() = ifExp.getTest() } + + /** Gets the true branch of this expression. */ + Expr getThen() { result.asExpr() = ifExp.getBody() } + + /** Gets the false branch of this expression. */ + Expr getElse() { result.asExpr() = ifExp.getOrelse() } + + override AstNode getChild(int index) { + index = 0 and result = this.getCondition() + or + index = 1 and result = this.getThen() + or + index = 2 and result = this.getElse() + } + } + + /** + * A binary expression for the shared CFG. In Python, this covers all + * `and`/`or` expression operand pairs. + */ + class BinaryExpr extends Expr, TBoolExprPair { + private Py::BoolExpr be; + private int index; + + BinaryExpr() { this = TBoolExprPair(be, index) } + + /** Gets the underlying Python `BoolExpr`. */ + Py::BoolExpr getBoolExpr() { result = be } + + /** Gets the (zero-based) index of this pair within its `BoolExpr`. */ + int getIndex() { result = index } + + override string toString() { result = be.getOperator() } + + override Py::Location getLocation() { result = be.getValue(index).getLocation() } + + override Callable getEnclosingCallable() { result.asScope() = be.getScope() } + + /** Gets the left operand of this binary expression. */ + Expr getLeftOperand() { result.asExpr() = be.getValue(index) } + + /** Gets the right operand of this binary expression. */ + Expr getRightOperand() { + // Last pair: right operand is the final value. + index = count(be.getAValue()) - 2 and result.asExpr() = be.getValue(index + 1) + or + // Non-last pair: right operand is the next synthetic pair. + index < count(be.getAValue()) - 2 and + exists(BinaryExpr next | + next.getBoolExpr() = be and next.getIndex() = index + 1 and result = next + ) + } + + override AstNode getChild(int childIndex) { + childIndex = 0 and result = this.getLeftOperand() + or + childIndex = 1 and result = this.getRightOperand() + } + } + + /** A short-circuiting logical `and` expression. */ + class LogicalAndExpr extends BinaryExpr { + LogicalAndExpr() { this.getBoolExpr().getOp() instanceof Py::And } + } + + /** A short-circuiting logical `or` expression. */ + class LogicalOrExpr extends BinaryExpr { + LogicalOrExpr() { this.getBoolExpr().getOp() instanceof Py::Or } + } + + /** A null-coalescing expression. Python has no null-coalescing operator. */ + class NullCoalescingExpr extends BinaryExpr { + NullCoalescingExpr() { none() } + } + + /** + * A unary expression. Currently only used for the `not` subclass. + */ + class UnaryExpr extends Expr { + UnaryExpr() { exists(Py::UnaryExpr u | this = TPyExpr(u) and u.getOp() instanceof Py::Not) } + + /** Gets the operand of this unary expression. */ + Expr getOperand() { result.asExpr() = this.asExpr().(Py::UnaryExpr).getOperand() } + + override AstNode getChild(int index) { index = 0 and result = this.getOperand() } + } + + /** A logical `not` expression. */ + class LogicalNotExpr extends UnaryExpr { } + + /** + * An assignment expression. + * + * Empty in Python: `x = y` and `x += y` are statements (`AssignStmt` and + * `AugAssignStmt`), not expressions, and the walrus `x := y` is modeled + * separately as `NamedExpr`. The shared library's `Assignment` extends + * `BinaryExpr`, so it cannot share instances with our `Stmt`-based + * assignment forms. + */ + class Assignment extends BinaryExpr { + Assignment() { none() } + } + + /** A simple assignment expression. Empty in Python (see `Assignment`). */ + class AssignExpr extends Assignment { } + + /** A compound assignment expression. Empty in Python (see `Assignment`). */ + class CompoundAssignment extends Assignment { } + + /** + * A short-circuiting logical AND compound assignment expression (`&&=`). + * Python has no such operator. + */ + class AssignLogicalAndExpr extends CompoundAssignment { } + + /** + * A short-circuiting logical OR compound assignment expression (`||=`). + * Python has no such operator. + */ + class AssignLogicalOrExpr extends CompoundAssignment { } + + /** + * A short-circuiting null-coalescing compound assignment expression + * (`??=`). Python has no such operator. + */ + class AssignNullCoalescingExpr extends CompoundAssignment { } + + /** A boolean literal expression (`True` or `False`). */ + class BooleanLiteral extends Expr { + BooleanLiteral() { this = TPyExpr(any(Py::True t)) or this = TPyExpr(any(Py::False f)) } + + /** Gets the boolean value of this literal. */ + boolean getValue() { + this.asExpr() instanceof Py::True and result = true + or + this.asExpr() instanceof Py::False and result = false + } + } + + /** A pattern match expression. Python has no `instanceof`-style pattern match expression. */ + class PatternMatchExpr extends Expr { + PatternMatchExpr() { none() } + + Expr getExpr() { none() } + + AstNode getPattern() { none() } + } + + // ===== Python-specific expression classes (used by `getChild`) ===== + /** A Python binary expression (arithmetic, bitwise, matmul, etc.). */ + additional class ArithBinaryExpr extends Expr { + private Py::BinaryExpr binExpr; + + ArithBinaryExpr() { this = TPyExpr(binExpr) } + + Expr getLeft() { result.asExpr() = binExpr.getLeft() } + + Expr getRight() { result.asExpr() = binExpr.getRight() } + + override AstNode getChild(int index) { + index = 0 and result = this.getLeft() + or + index = 1 and result = this.getRight() + } + } + + /** A call expression (`func(args...)`). */ + additional class CallExpr extends Expr { + private Py::Call call; + + CallExpr() { this = TPyExpr(call) } + + Expr getFunc() { result.asExpr() = call.getFunc() } + + Expr getPositionalArg(int n) { result.asExpr() = call.getPositionalArg(n) } + + int getNumberOfPositionalArgs() { result = count(call.getAPositionalArg()) } + + Expr getKeywordValue(int n) { + result.asExpr() = call.getNamedArg(n).(Py::Keyword).getValue() + or + result.asExpr() = call.getNamedArg(n).(Py::DictUnpacking).getValue() + } + + int getNumberOfNamedArgs() { result = count(call.getANamedArg()) } + + override AstNode getChild(int index) { + index = 0 and result = this.getFunc() + or + result = this.getPositionalArg(index - 1) and index >= 1 + or + result = this.getKeywordValue(index - 1 - this.getNumberOfPositionalArgs()) and + index >= 1 + this.getNumberOfPositionalArgs() + } + } + + /** A subscript expression (`obj[index]`). */ + additional class SubscriptExpr extends Expr { + private Py::Subscript sub; + + SubscriptExpr() { this = TPyExpr(sub) } + + Expr getObject() { result.asExpr() = sub.getObject() } + + Expr getIndex() { result.asExpr() = sub.getIndex() } + + override AstNode getChild(int index) { + index = 0 and result = this.getObject() + or + index = 1 and result = this.getIndex() + } + } + + /** An attribute access (`obj.name`). */ + additional class AttributeExpr extends Expr { + private Py::Attribute attr; + + AttributeExpr() { this = TPyExpr(attr) } + + Expr getObject() { result.asExpr() = attr.getObject() } + + override AstNode getChild(int index) { index = 0 and result = this.getObject() } + } + + /** + * An `import x.y` module expression. Modelled as a leaf — the dotted + * name is just a string. + */ + additional class ImportExpression extends Expr { + ImportExpression() { this.asExpr() instanceof Py::ImportExpr } + } + + /** + * A `from m import x` member access. The module sub-expression is a + * child so that the CFG visits both the module load and this + * attribute selection. + */ + additional class ImportMemberExpr extends Expr { + private Py::ImportMember im; + + ImportMemberExpr() { this = TPyExpr(im) } + + /** Gets the module expression `m` in `from m import x`. */ + Expr getModule() { result.asExpr() = im.getModule() } + + override AstNode getChild(int index) { index = 0 and result = this.getModule() } + } + + /** A tuple literal. */ + additional class TupleExpr extends Expr { + private Py::Tuple tuple; + + TupleExpr() { this = TPyExpr(tuple) } + + Expr getElt(int n) { result.asExpr() = tuple.getElt(n) } + + override AstNode getChild(int index) { result = this.getElt(index) } + } + + /** A list literal. */ + additional class ListExpr extends Expr { + private Py::List list; + + ListExpr() { this = TPyExpr(list) } + + Expr getElt(int n) { result.asExpr() = list.getElt(n) } + + override AstNode getChild(int index) { result = this.getElt(index) } + } + + /** A set literal. */ + additional class SetExpr extends Expr { + private Py::Set set; + + SetExpr() { this = TPyExpr(set) } + + Expr getElt(int n) { result.asExpr() = set.getElt(n) } + + override AstNode getChild(int index) { result = this.getElt(index) } + } + + /** A dict literal. */ + additional class DictExpr extends Expr { + private Py::Dict dict; + + DictExpr() { this = TPyExpr(dict) } + + /** + * Gets the key of the `n`th item (at child index `2*n`); the value is + * at child index `2*n + 1`. + */ + Expr getKey(int n) { result.asExpr() = dict.getItem(n).(Py::KeyValuePair).getKey() } + + Expr getValue(int n) { result.asExpr() = dict.getItem(n).(Py::KeyValuePair).getValue() } + + int getNumberOfItems() { result = count(dict.getAnItem()) } + + override AstNode getChild(int index) { + exists(int item | + index = 2 * item and result = this.getKey(item) + or + index = 2 * item + 1 and result = this.getValue(item) + ) + } + } + + /** A unary expression other than `not` (e.g., `-x`, `+x`, `~x`). */ + additional class ArithUnaryExpr extends Expr { + private Py::UnaryExpr unaryExpr; + + ArithUnaryExpr() { this = TPyExpr(unaryExpr) and not unaryExpr.getOp() instanceof Py::Not } + + Expr getOperand() { result.asExpr() = unaryExpr.getOperand() } + + override AstNode getChild(int index) { index = 0 and result = this.getOperand() } + } + + /** + * A comprehension or generator expression. The iterable is evaluated in + * the enclosing scope; the body runs in a nested synthetic function + * scope handled by its own CFG. + */ + additional class Comprehension extends Expr { + private Py::Expr iterable; + + Comprehension() { + exists(Py::Expr c | this = TPyExpr(c) | + iterable = c.(Py::ListComp).getIterable() + or + iterable = c.(Py::SetComp).getIterable() + or + iterable = c.(Py::DictComp).getIterable() + or + iterable = c.(Py::GeneratorExp).getIterable() + ) + } + + Expr getIterable() { result.asExpr() = iterable } + + override AstNode getChild(int index) { index = 0 and result = this.getIterable() } + } + + /** A comparison expression (`a < b`, `a < b < c`, etc.). */ + additional class CompareExpr extends Expr { + private Py::Compare cmp; + + CompareExpr() { this = TPyExpr(cmp) } + + Expr getLeft() { result.asExpr() = cmp.getLeft() } + + Expr getComparator(int n) { result.asExpr() = cmp.getComparator(n) } + + override AstNode getChild(int index) { + index = 0 and result = this.getLeft() + or + result = this.getComparator(index - 1) and index >= 1 + } + } + + /** A slice expression (`start:stop:step`). */ + additional class SliceExpr extends Expr { + private Py::Slice slice; + + SliceExpr() { this = TPyExpr(slice) } + + Expr getStart() { result.asExpr() = slice.getStart() } + + Expr getStop() { result.asExpr() = slice.getStop() } + + Expr getStep() { result.asExpr() = slice.getStep() } + + override AstNode getChild(int index) { + index = 0 and result = this.getStart() + or + index = 1 and result = this.getStop() + or + index = 2 and result = this.getStep() + } + } + + /** A starred expression (`*x`). */ + additional class StarredExpr extends Expr { + private Py::Starred starred; + + StarredExpr() { this = TPyExpr(starred) } + + Expr getValue() { result.asExpr() = starred.getValue() } + + override AstNode getChild(int index) { index = 0 and result = this.getValue() } + } + + /** A formatted string literal (`f"...{expr}..."`). */ + additional class FstringExpr extends Expr { + private Py::Fstring fstring; + + FstringExpr() { this = TPyExpr(fstring) } + + Expr getValue(int n) { result.asExpr() = fstring.getValue(n) } + + override AstNode getChild(int index) { result = this.getValue(index) } + } + + /** A formatted value inside an f-string (`{expr}` or `{expr:spec}`). */ + additional class FormattedValueExpr extends Expr { + private Py::FormattedValue fv; + + FormattedValueExpr() { this = TPyExpr(fv) } + + Expr getValue() { result.asExpr() = fv.getValue() } + + Expr getFormatSpec() { result.asExpr() = fv.getFormatSpec() } + + override AstNode getChild(int index) { + index = 0 and result = this.getValue() + or + index = 1 and result = this.getFormatSpec() + } + } + + /** A `yield` expression. */ + additional class YieldExpr extends Expr { + private Py::Yield yield; + + YieldExpr() { this = TPyExpr(yield) } + + Expr getValue() { result.asExpr() = yield.getValue() } + + override AstNode getChild(int index) { index = 0 and result = this.getValue() } + } + + /** A `yield from` expression. */ + additional class YieldFromExpr extends Expr { + private Py::YieldFrom yieldFrom; + + YieldFromExpr() { this = TPyExpr(yieldFrom) } + + Expr getValue() { result.asExpr() = yieldFrom.getValue() } + + override AstNode getChild(int index) { index = 0 and result = this.getValue() } + } + + /** An `await` expression. */ + additional class AwaitExpr extends Expr { + private Py::Await await; + + AwaitExpr() { this = TPyExpr(await) } + + Expr getValue() { result.asExpr() = await.getValue() } + + override AstNode getChild(int index) { index = 0 and result = this.getValue() } + } + + /** + * A class definition expression (visits bases, but NOT PEP 695 type + * parameters — those bind in an annotation scope that nests the class + * body, so they belong to the inner scope's CFG, not the enclosing + * scope's; the legacy CFG also omitted them). + */ + additional class ClassDefExpr extends Expr { + private Py::ClassExpr classExpr; + + ClassDefExpr() { this = TPyExpr(classExpr) } + + Expr getBase(int n) { result.asExpr() = classExpr.getBase(n) } + + override AstNode getChild(int index) { result = this.getBase(index) } + } + + /** + * A function definition expression (visits positional and keyword + * defaults, but NOT PEP 695 type parameters — those bind in an + * annotation scope that nests the function body, so they belong to + * the inner scope's CFG, not the enclosing scope's; the legacy CFG + * also omitted them). + */ + additional class FunctionDefExpr extends Expr { + private Py::FunctionExpr funcExpr; + + FunctionDefExpr() { this = TPyExpr(funcExpr) } + + /** + * Gets the `n`th default for a positional argument, in evaluation + * order. Note that `Args.getDefault(int)` is indexed by argument + * position (with gaps for arguments without defaults), so we must + * renumber here to obtain contiguous indices. + */ + Expr getDefault(int n) { + result.asExpr() = + rank[n + 1](Py::Expr d, int i | d = funcExpr.getArgs().getDefault(i) | d order by i) + } + + /** Gets the `n`th default for a keyword-only argument, in evaluation order. */ + Expr getKwDefault(int n) { + result.asExpr() = + rank[n + 1](Py::Expr d, int i | d = funcExpr.getArgs().getKwDefault(i) | d order by i) + } + + int getNumberOfDefaults() { result = count(funcExpr.getArgs().getADefault()) } + + override AstNode getChild(int index) { + result = this.getDefault(index) + or + result = this.getKwDefault(index - this.getNumberOfDefaults()) + } + } + + /** A lambda expression (has default args evaluated at definition time). */ + additional class LambdaExpr extends Expr { + private Py::Lambda lambda; + + LambdaExpr() { this = TPyExpr(lambda) } + + /** Gets the `n`th default for a positional argument, in evaluation order. */ + Expr getDefault(int n) { + result.asExpr() = + rank[n + 1](Py::Expr d, int i | d = lambda.getArgs().getDefault(i) | d order by i) + } + + /** Gets the `n`th default for a keyword-only argument, in evaluation order. */ + Expr getKwDefault(int n) { + result.asExpr() = + rank[n + 1](Py::Expr d, int i | d = lambda.getArgs().getKwDefault(i) | d order by i) + } + + int getNumberOfDefaults() { result = count(lambda.getArgs().getADefault()) } + + override AstNode getChild(int index) { + result = this.getDefault(index) + or + result = this.getKwDefault(index - this.getNumberOfDefaults()) + } + } + + /** Gets the child of `n` at the specified (zero-based) index. */ + AstNode getChild(AstNode n, int index) { result = n.getChild(index) } +} + +private module Cfg0 = Make0; + +private import Cfg0 + +private module Cfg1 = Make1; + +private import Cfg1 + +private module Cfg2 = Make2; + +private import Cfg2 + +private module Input implements InputSig1, InputSig2 { + predicate cfgCachedStageRef() { CfgCachedStage::ref() } + + private newtype TLabel = TNone() + + class Label extends TLabel { + string toString() { result = "label" } + } + + class CallableContext = Void; + + predicate inConditionalContext(Ast::AstNode n, ConditionKind kind) { + kind.isBoolean() and + n = any(Ast::AssertStmt a).getTest() + } + + private string assertThrowTag() { result = "[assert-throw]" } + + predicate additionalNode(Ast::AstNode n, string tag, NormalSuccessor t) { + n instanceof Ast::AssertStmt and tag = assertThrowTag() and t instanceof DirectSuccessor + } + + predicate beginAbruptCompletion( + Ast::AstNode ast, PreControlFlowNode n, AbruptCompletion c, boolean always + ) { + ast instanceof Ast::AssertStmt and + n.isAdditional(ast, assertThrowTag()) and + c.asSimpleAbruptCompletion() instanceof ExceptionSuccessor and + always = true + } + + predicate endAbruptCompletion(Ast::AstNode ast, PreControlFlowNode n, AbruptCompletion c) { + none() + } + + predicate step(PreControlFlowNode n1, PreControlFlowNode n2) { + exists(Ast::AssertStmt assertStmt | + n1.isBefore(assertStmt) and + n2.isBefore(assertStmt.getTest()) + or + n1.isAfterTrue(assertStmt.getTest()) and + n2.isAfter(assertStmt) + or + n1.isAfterFalse(assertStmt.getTest()) and + ( + n2.isBefore(assertStmt.getMsg()) + or + not exists(assertStmt.getMsg()) and + n2.isAdditional(assertStmt, assertThrowTag()) + ) + or + n1.isAfter(assertStmt.getMsg()) and + n2.isAdditional(assertStmt, assertThrowTag()) + ) + } +} + +import CfgCachedStage +import Public + +/** + * Maps a CFG AST wrapper node to the corresponding Python AST node, if any. + * Entry, exit, and synthetic nodes have no corresponding Python AST node. + */ +Py::AstNode astNodeToPyNode(Ast::AstNode n) { + result = n.asExpr() + or + result = n.asStmt() + or + result = n.asScope() + or + result = n.asPattern() +} diff --git a/python/ql/test/extractor-tests/syntax_error/CONSISTENCY/CfgConsistency.expected b/python/ql/test/extractor-tests/syntax_error/CONSISTENCY/CfgConsistency.expected new file mode 100644 index 00000000000..91a01a3a3d9 --- /dev/null +++ b/python/ql/test/extractor-tests/syntax_error/CONSISTENCY/CfgConsistency.expected @@ -0,0 +1,4 @@ +consistencyOverview +| deadEnd | 1 | +deadEnd +| without_loop.py:7:5:7:9 | Break | diff --git a/python/ql/test/library-tests/ControlFlow/bindings/BindingsTest.expected b/python/ql/test/library-tests/ControlFlow/bindings/BindingsTest.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/bindings/BindingsTest.ql b/python/ql/test/library-tests/ControlFlow/bindings/BindingsTest.ql new file mode 100644 index 00000000000..a507878911b --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/BindingsTest.ql @@ -0,0 +1,32 @@ +/** + * Phase -1 of the dataflow CFG migration: verifies that every variable + * binding visible to the AST (`Name.defines(v)`) corresponds to a CFG node + * in the new CFG (`semmle.python.controlflow.internal.AstNodeImpl`). + * + * The expected tag is `cfgdefines=`. Each binding annotation in the + * test sources looks like `# $ cfgdefines=x` for a binding currently + * covered by the new CFG, or `# $ MISSING: cfgdefines=x` for a binding + * that is known to be uncovered (a "red" test case that should be + * green-flipped once the corresponding `cfg-ext-*` extension lands). + */ + +import python +import semmle.python.controlflow.internal.AstNodeImpl as CfgImpl +import utils.test.InlineExpectationsTest + +module CfgBindingsTest implements TestSig { + string getARelevantTag() { result = "cfgdefines" } + + predicate hasActualResult(Location location, string element, string tag, string value) { + exists(Name n, Variable v, CfgImpl::ControlFlowNode cfg | + n.defines(v) and + cfg.getAstNode().asExpr() = n and + location = n.getLocation() and + element = n.toString() and + tag = "cfgdefines" and + value = v.getId() + ) + } +} + +import MakeTest diff --git a/python/ql/test/library-tests/ControlFlow/bindings/annassign.py b/python/ql/test/library-tests/ControlFlow/bindings/annassign.py new file mode 100644 index 00000000000..7a9ae3ab6c7 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/annassign.py @@ -0,0 +1,13 @@ +# Annotated assignment (PEP 526). Both with and without an initializer. + +a: int = 1 # $ cfgdefines=a +b: str = "hi" # $ cfgdefines=b + +# Annotation without value: the AST records `c` as defined, +# and the new CFG now visits it via the AnnAssignStmt wrapper. +c: int # $ cfgdefines=c + +class K: # $ cfgdefines=K + field: int = 0 # $ cfgdefines=field + + diff --git a/python/ql/test/library-tests/ControlFlow/bindings/compound.py b/python/ql/test/library-tests/ControlFlow/bindings/compound.py new file mode 100644 index 00000000000..cb2f36f12ff --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/compound.py @@ -0,0 +1,14 @@ +# Compound (tuple/list) assignment targets — actually wired in the new CFG. + +a, b = (1, 2) # $ cfgdefines=a cfgdefines=b +[c, d] = [3, 4] # $ cfgdefines=c cfgdefines=d + +# Nested unpacking. +(e, (f, g)) = (1, (2, 3)) # $ cfgdefines=e cfgdefines=f cfgdefines=g + +# Star unpacking. +h, *i = [1, 2, 3] # $ cfgdefines=h cfgdefines=i + +# Chained assignment with compound target. +j = k, l = (5, 6) # $ cfgdefines=j cfgdefines=k cfgdefines=l + diff --git a/python/ql/test/library-tests/ControlFlow/bindings/comprehension.py b/python/ql/test/library-tests/ControlFlow/bindings/comprehension.py new file mode 100644 index 00000000000..6b5f722c1f7 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/comprehension.py @@ -0,0 +1,21 @@ +# Comprehension and `for` loop targets — wired in the new CFG. +# Comprehensions are nested function scopes with a synthetic `.0` parameter +# bound to the iterable. + +# Bare-name `for` target. +for i in range(3): # $ cfgdefines=i + pass + +# Compound `for` target. +for k, v in [(1, 2)]: # $ cfgdefines=k cfgdefines=v + pass + +# Comprehension targets. +_ = [x for x in range(3)] # $ cfgdefines=_ cfgdefines=x cfgdefines=.0 +_ = {y: z for y, z in []} # $ cfgdefines=_ cfgdefines=y cfgdefines=z cfgdefines=.0 +_ = (a for a in []) # $ cfgdefines=_ cfgdefines=a cfgdefines=.0 + +# Nested comprehensions. +_ = [b for c in [] for b in c] # $ cfgdefines=_ cfgdefines=c cfgdefines=b cfgdefines=.0 + + diff --git a/python/ql/test/library-tests/ControlFlow/bindings/dead_under_no_raise.py b/python/ql/test/library-tests/ControlFlow/bindings/dead_under_no_raise.py new file mode 100644 index 00000000000..dbfb857b536 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/dead_under_no_raise.py @@ -0,0 +1,52 @@ +# Dead bindings under the "no expressions raise" CFG abstraction. +# +# The new CFG does not currently model raise edges from arbitrary +# expressions. As a consequence, code that is only reachable through +# exception flow is (correctly) classified as dead and has no CFG node. +# Variable bindings in dead code do not need CFG nodes - SSA / dataflow +# over dead code is moot. +# +# These tests act as a regression guard: the bindings below intentionally +# have no `cfgdefines=` annotations. If raise modelling is later added, +# the BindingsTest infrastructure will surface the new CFG nodes as +# unexpected results, and this file will need to be revisited. + + +def f(obj): # $ cfgdefines=f cfgdefines=obj + try: + return len(obj) + except TypeError: + pass + + # The first try's body always returns; its except handler does not + # raise or otherwise transfer control, so under "no expressions + # raise" the only paths out of the try-statement are dead. Everything + # below is unreachable. + try: + hint = type(obj).__length_hint__ + except AttributeError: + return None + return hint + + +def g(): # $ cfgdefines=g + try: + raise Exception("inner") + except: + raise Exception("outer") + else: + # Unreachable: the inner try body always raises, so the `else:` + # clause never runs. + hit_inner_else = True + + +def h(cache, key): # $ cfgdefines=h cfgdefines=cache cfgdefines=key + try: + return cache[key] + except KeyError: + pass + + # Same pattern as `f`: dead under "no expressions raise". + value = compute(key) + cache[key] = value + return value diff --git a/python/ql/test/library-tests/ControlFlow/bindings/decorated.py b/python/ql/test/library-tests/ControlFlow/bindings/decorated.py new file mode 100644 index 00000000000..9b93c166ace --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/decorated.py @@ -0,0 +1,30 @@ +# Decorated `def`/`class` — wired in the new CFG. + + +def deco(f): # $ cfgdefines=deco cfgdefines=f + return f + + +@deco +def decorated_func(): # $ cfgdefines=decorated_func + pass + + +@deco +class DecoratedClass: # $ cfgdefines=DecoratedClass + pass + + +# Stacked decorators. +@deco +@deco +def doubly(): # $ cfgdefines=doubly + pass + + +# Inside a class body. +class Outer: # $ cfgdefines=Outer + @staticmethod + def inner(): # $ cfgdefines=inner + pass + diff --git a/python/ql/test/library-tests/ControlFlow/bindings/except_handler.py b/python/ql/test/library-tests/ControlFlow/bindings/except_handler.py new file mode 100644 index 00000000000..57b6c99fe9b --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/except_handler.py @@ -0,0 +1,19 @@ +# Exception-handler name bindings. These are already wired in the new +# CFG provided the try body can raise; `raise` statements are reliably +# treated as exception sources. + +try: + raise ValueError("oops") +except ValueError as e: # $ cfgdefines=e + pass + +try: + raise TypeError("oops") +except (TypeError, KeyError) as err: # $ cfgdefines=err + pass + +# Exception groups (Python 3.11+). +try: + raise ValueError("oops") +except* ValueError as eg: # $ cfgdefines=eg + pass diff --git a/python/ql/test/library-tests/ControlFlow/bindings/imports.py b/python/ql/test/library-tests/ControlFlow/bindings/imports.py new file mode 100644 index 00000000000..c8834b5332a --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/imports.py @@ -0,0 +1,14 @@ +# Import aliases — all bound names below are now reachable via the new +# CFG's `ImportStmt` wrapper. + +import os # $ cfgdefines=os +import os.path # $ cfgdefines=os +import os as o # $ cfgdefines=o +from os import path # $ cfgdefines=path +from os import path as p # $ cfgdefines=p +from os import sep, linesep # $ cfgdefines=sep cfgdefines=linesep +from os import ( + getcwd, # $ cfgdefines=getcwd + getcwdb, # $ cfgdefines=getcwdb +) + diff --git a/python/ql/test/library-tests/ControlFlow/bindings/match_pattern.py b/python/ql/test/library-tests/ControlFlow/bindings/match_pattern.py new file mode 100644 index 00000000000..0868a2680d0 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/match_pattern.py @@ -0,0 +1,24 @@ +# Match-statement pattern bindings — wired in the new CFG. + +def f(subject): # $ cfgdefines=f cfgdefines=subject + match subject: + case x: # $ cfgdefines=x + pass + case [a, b]: # $ cfgdefines=a cfgdefines=b + pass + case {"k": v}: # $ cfgdefines=v + pass + case Point(p, q): # $ cfgdefines=p cfgdefines=q + pass + case [_, *rest]: # $ cfgdefines=rest + pass + case (1 | 2) as n: # $ cfgdefines=n + pass + + +class Point: # $ cfgdefines=Point + __match_args__ = ("x", "y") # $ cfgdefines=__match_args__ + x: int # $ cfgdefines=x + y: int # $ cfgdefines=y + + diff --git a/python/ql/test/library-tests/ControlFlow/bindings/parameters.py b/python/ql/test/library-tests/ControlFlow/bindings/parameters.py new file mode 100644 index 00000000000..7fe5e01e4c4 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/parameters.py @@ -0,0 +1,42 @@ +# Function parameters. + +def positional(a, b): # $ cfgdefines=positional cfgdefines=a cfgdefines=b + pass + + +def with_default(x=1, y=2): # $ cfgdefines=with_default cfgdefines=x cfgdefines=y + pass + + +def with_vararg(*args): # $ cfgdefines=with_vararg cfgdefines=args + pass + + +def with_kwarg(**kwargs): # $ cfgdefines=with_kwarg cfgdefines=kwargs + pass + + +def with_kwonly(*, k1, k2=5): # $ cfgdefines=with_kwonly cfgdefines=k1 cfgdefines=k2 + pass + + +def kitchen_sink(a, b=2, *args, k1, k2=5, **kw): # $ cfgdefines=kitchen_sink cfgdefines=a cfgdefines=b cfgdefines=args cfgdefines=k1 cfgdefines=k2 cfgdefines=kw + pass + + +# Methods get `self` / `cls`. +class C: # $ cfgdefines=C + def method(self, x): # $ cfgdefines=method cfgdefines=self cfgdefines=x + pass + + @classmethod + def cmethod(cls, x): # $ cfgdefines=cmethod cfgdefines=cls cfgdefines=x + pass + + +# Lambda parameter. +_ = lambda p: p + 1 # $ cfgdefines=_ cfgdefines=p + +# PEP 570 positional-only. +def pos_only(a, b, /, c): # $ cfgdefines=pos_only cfgdefines=a cfgdefines=b cfgdefines=c + pass diff --git a/python/ql/test/library-tests/ControlFlow/bindings/simple.py b/python/ql/test/library-tests/ControlFlow/bindings/simple.py new file mode 100644 index 00000000000..51cb7d828c9 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/simple.py @@ -0,0 +1,14 @@ +# Simple bindings that should already work in the new CFG. +# No MISSING annotations expected. + +x = 1 # $ cfgdefines=x +y = x + 1 # $ cfgdefines=y + +def f(): # $ cfgdefines=f + pass + +class C: # $ cfgdefines=C + pass + +# Re-assignment. +x = 2 # $ cfgdefines=x diff --git a/python/ql/test/library-tests/ControlFlow/bindings/type_params.py b/python/ql/test/library-tests/ControlFlow/bindings/type_params.py new file mode 100644 index 00000000000..2bd34dc3f0e --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/type_params.py @@ -0,0 +1,21 @@ +# PEP 695 type parameters (Python 3.12+). + +# PEP 695 type-param names on `def`/`class` bind in an annotation scope +# that nests the function/class body — they have no CFG node in the +# enclosing scope (matching the legacy CFG). +def func[T](x: T) -> T: # $ cfgdefines=func cfgdefines=x + return x + + +class Box[T]: # $ cfgdefines=Box + item: T # $ cfgdefines=item + + +# Multi-parameter, with bound and variadics. +def multi[T: int, *Ts, **P](x: T, *args: *Ts, **kwargs: P.kwargs) -> T: # $ cfgdefines=multi cfgdefines=x cfgdefines=args cfgdefines=kwargs + return x + + +# `type` statement (PEP 695). +type Alias[T] = list[T] # $ cfgdefines=Alias cfgdefines=T + diff --git a/python/ql/test/library-tests/ControlFlow/bindings/walrus_starred.py b/python/ql/test/library-tests/ControlFlow/bindings/walrus_starred.py new file mode 100644 index 00000000000..5c0c1bd8319 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/walrus_starred.py @@ -0,0 +1,14 @@ +# Walrus and starred-target edge cases — wired in the new CFG. + +# Walrus in expression context. +if (y := 5) > 0: # $ cfgdefines=y + pass + +# Walrus in a comprehension. The comprehension introduces a synthetic +# `.0` parameter bound to the iterable. +_ = [w for _ in range(3) if (w := 1)] # $ cfgdefines=_ cfgdefines=w cfgdefines=.0 + +# Starred target in a Tuple LHS. +*head, tail = [1, 2, 3] # $ cfgdefines=head cfgdefines=tail + + diff --git a/python/ql/test/library-tests/ControlFlow/bindings/with_stmt.py b/python/ql/test/library-tests/ControlFlow/bindings/with_stmt.py new file mode 100644 index 00000000000..5fffe46c5d4 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/with_stmt.py @@ -0,0 +1,21 @@ +# `with cm() as x:` bindings — wired in the new CFG. + +class CM: # $ cfgdefines=CM + def __enter__(self): return self # $ cfgdefines=__enter__ cfgdefines=self + def __exit__(self, *a): pass # $ cfgdefines=__exit__ cfgdefines=self cfgdefines=a + +with CM() as x: # $ cfgdefines=x + pass + +# Multiple items. +with CM() as a, CM() as b: # $ cfgdefines=a cfgdefines=b + pass + +# Parenthesised form (Python 3.10+). +with (CM() as p, CM() as q): # $ cfgdefines=p cfgdefines=q + pass + +# Compound target in `with`. +with CM() as (m, n): # $ cfgdefines=m cfgdefines=n + pass + diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAllLiveReachable.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAllLiveReachable.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAllLiveReachable.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAllLiveReachable.ql new file mode 100644 index 00000000000..75f02d14a9c --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAllLiveReachable.ql @@ -0,0 +1,14 @@ +/** New-CFG version of AllLiveReachable. */ + +import python +import TimerUtils +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from TimerCfgNode a, TestFunction f +where allLiveReachable(a, f) +select a, "Unreachable live annotation; entry of $@ does not reach this node", f, f.getName() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAnnotationHasCfgNode.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAnnotationHasCfgNode.expected new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAnnotationHasCfgNode.expected @@ -0,0 +1 @@ + diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAnnotationHasCfgNode.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAnnotationHasCfgNode.ql new file mode 100644 index 00000000000..4b1d82e27e6 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAnnotationHasCfgNode.ql @@ -0,0 +1,18 @@ +/** + * New-CFG version of AnnotationHasCfgNode. + * + * Checks that every timer annotation has a corresponding CFG node. + */ + +import python +import TimerUtils +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils::CfgTests + +from TimerAnnotation ann +where annotationWithoutCfgNode(ann) +select ann, "Annotation in $@ has no CFG node", ann.getTestFunction(), + ann.getTestFunction().getName() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.ql new file mode 100644 index 00000000000..80dd759a365 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.ql @@ -0,0 +1,26 @@ +/** + * New-CFG version of BasicBlockAnnotationGap. + * + * Original: + * Checks that within a basic block, if a node is annotated then its + * successor is also annotated (or excluded). A gap in annotations + * within a basic block indicates a missing annotation, since there + * are no branches to justify the gap. + * + * Nodes with exceptional successors are excluded, as the exception + * edge leaves the basic block and the normal successor may be dead. + */ + +import python +import TimerUtils +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from TimerCfgNode a, CfgNode succ +where basicBlockAnnotationGap(a, succ) +select a, "Annotated node followed by unannotated $@ in the same basic block", succ, + succ.getNode().toString() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.ql new file mode 100644 index 00000000000..f06d08d937e --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.ql @@ -0,0 +1,21 @@ +/** + * New-CFG version of BasicBlockOrdering. + * + * Original: + * Checks that within a single basic block, annotations appear in + * increasing minimum-timestamp order. + */ + +import python +import TimerUtils +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from TimerCfgNode a, TimerCfgNode b, int minA, int minB +where basicBlockOrdering(a, b, minA, minB) +select a, "Basic block ordering: $@ appears before $@", a.getTimestampExpr(minA), + "timestamp " + minA, b.getTimestampExpr(minB), "timestamp " + minB diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBranchTimestamps.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBranchTimestamps.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBranchTimestamps.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBranchTimestamps.ql new file mode 100644 index 00000000000..cfd8ffb4e4b --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBranchTimestamps.ql @@ -0,0 +1,80 @@ +/** + * New-CFG version of BranchTimestamps. + * + * Checks that when a node has both a true and false successor, the + * live timestamps on one branch are marked as dead on the other. + * This ensures that boolean branches are fully annotated with dead() + * markers for the paths not taken. + * + * Limitation: the `@ t[ts, ...]` / `dead(ts)` annotation scheme can only + * model branch-dead-ness for plain boolean control flow that reconverges + * linearly after the split — i.e. `if`-with-else and `if`-expression. + * It cannot model: + * + * * loops (`while` / `for`): body timestamps repeat across iterations, + * so the loop-exit annotation can't list them as dead; + * * `match` statements: each `case` body is a syntactically distinct + * sub-tree, and the branches don't reconverge through a common + * annotation point in the timeline; + * * `try` / `with` and `raise` / `assert`: exception edges are modelled + * as true/false but flow to syntactically distinct handlers, with no + * reconvergence in the linear annotation order; + * * short-circuit `and` / `or` (`BoolExpr`): the branches reconverge at + * the BoolExpr's after-node, so timestamps on one branch are live + * downstream of the other rather than dead; + * * `if` without an `else` clause, and `if`/`elif` chains: the false + * branch reconverges with the true branch at the post-if statement + * (no-else) or fans out across multiple elif-test annotations, + * neither of which fit the binary annotation scheme. + * + * Branch nodes inside those constructs are therefore whitelisted out + * below. The check still fires (and is useful) for plain `if`/`else` + * and conditional-expression branching. + */ + +import python +import TimerUtils +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +/** + * Holds if `f` contains a construct whose branches the linear-timestamp + * annotation scheme cannot describe (see file-level comment). + */ +private predicate hasUnmodellableBranching(Function f) { + exists(AstNode bad | + bad.getScope() = f and + ( + bad instanceof While + or + bad instanceof For + or + bad instanceof MatchStmt + or + bad instanceof Try + or + bad instanceof With + or + bad instanceof Raise + or + bad instanceof Assert + or + bad instanceof BoolExpr + or + bad instanceof If and + (not exists(bad.(If).getAnOrelse()) or bad.(If).isElif()) + ) + ) +} + +from TimerCfgNode node, int ts, string branch +where + missingBranchTimestamp(node, ts, branch) and + not hasUnmodellableBranching(node.getTestFunction()) +select node, + "Timestamp " + ts + " on true/false branch is missing a dead() annotation on the " + branch + + " successor in $@", node.getTestFunction(), node.getTestFunction().getName() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.expected new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.expected @@ -0,0 +1 @@ + diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.ql new file mode 100644 index 00000000000..3feacae264e --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.ql @@ -0,0 +1,22 @@ +/** + * New-CFG version of ConsecutivePredecessorTimestamps. + * + * Checks that each annotated node (except the minimum timestamp) has + * a predecessor annotation with timestamp `a - 1`. This is the reverse + * of ConsecutiveTimestamps: it catches nodes that are reachable but + * arrived at from the wrong place (skipping an intermediate node). + */ + +import python +import TimerUtils +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from TimerAnnotation ann, int a +where consecutivePredecessorTimestamps(ann, a) +select ann, "$@ in $@ has no consecutive predecessor (expected " + (a - 1) + ")", + ann.getTimestampExpr(a), "Timestamp " + a, ann.getTestFunction(), ann.getTestFunction().getName() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.ql new file mode 100644 index 00000000000..8e52663d6ea --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.ql @@ -0,0 +1,29 @@ +/** + * New-CFG version of ConsecutiveTimestamps. + * + * Original: + * Checks that consecutive annotated nodes have consecutive timestamps: + * for each annotation with timestamp `a`, some CFG node for that annotation + * must have a next annotation containing `a + 1`. + * + * Handles CFG splitting (e.g., finally blocks duplicated for normal/exceptional + * flow) by checking that at least one split has the required successor. + * + * Only applies to functions where all annotations are in the function's + * own scope (excludes tests with generators, async, comprehensions, or + * lambdas that have annotations in nested scopes). + */ + +import python +import TimerUtils +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from TimerAnnotation ann, int a +where consecutiveTimestamps(ann, a) +select ann, "$@ in $@ has no consecutive successor (expected " + (a + 1) + ")", + ann.getTimestampExpr(a), "Timestamp " + a, ann.getTestFunction(), ann.getTestFunction().getName() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgImpl.qll b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgImpl.qll new file mode 100644 index 00000000000..cbecb2da19d --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgImpl.qll @@ -0,0 +1,120 @@ +/** + * Implementation of the evaluation-order CFG signature using the new + * shared control flow graph from AstNodeImpl. + */ + +private import python as Py +import TimerUtils +private import semmle.python.controlflow.internal.AstNodeImpl as CfgImpl +private import codeql.controlflow.SuccessorType + +private class NewControlFlowNode = CfgImpl::ControlFlowNode; + +private class NewBasicBlock = CfgImpl::BasicBlock; + +/** New (shared) CFG implementation of the evaluation-order signature. */ +module NewCfg implements EvalOrderCfgSig { + class CfgNode instanceof NewControlFlowNode { + // We must pick a *unique* representative CFG node for each AST node. The + // shared CFG has several nodes per AST node (before / in-post-order / after + // / after-value splits), but the timer test framework keys annotations on + // `getNode()` and assumes one CFG node per annotated AST node. Without a + // filter, an annotated `f()` would map to both `f()` and `After f()`, which + // breaks two framework invariants: (1) the "no shared reachable" check + // requires that two distinct nodes sharing a timestamp be mutually + // unreachable (true/false branches of a condition), but `Before f()`, + // `f()` and `After f()` share the annotation's timestamp *and* lie on one + // linear path; and (2) the annotation walk (`nextTimerAnnotation`) halts at + // the first reachable representative, so a second node for the same AST + // node would stall the walk on the same timestamp instead of advancing to + // the next evaluation event. + // + // We use the "after" node (`isAfter`) rather than the canonical `injects` + // node, because `injects` represents short-circuit / conditional + // expressions (`and`/`or`/`not`/ternary) by their *before* node, placing + // them ahead of their operands — wrong for evaluation order. `isAfter` + // instead picks the post-evaluation node: the merged before/after node for + // simple leaves, the `TAfterNode` for post-order expressions, and the + // `AfterValueNode`(s) for pre-order conditionals, all positioned after the + // operands. The two value-split nodes of a conditional are genuinely + // distinct evaluation outcomes (handled by `getATrueSuccessor` / + // `getAFalseSuccessor`), so they do not violate the uniqueness assumption. + CfgNode() { NewControlFlowNode.super.isAfter(_) } + + string toString() { result = NewControlFlowNode.super.toString() } + + Py::Location getLocation() { result = NewControlFlowNode.super.getLocation() } + + Py::AstNode getNode() { + result = CfgImpl::astNodeToPyNode(NewControlFlowNode.super.getAstNode()) + } + + CfgNode getASuccessor() { nextCfgNode(this, result) } + + CfgNode getATrueSuccessor() { + NewControlFlowNode.super.isAfterTrue(_) and + // Only where there's also a false branch (true boolean split) + exists(NewControlFlowNode other | other.isAfterFalse(NewControlFlowNode.super.getAstNode())) and + nextCfgNodeFrom(this, result) + } + + CfgNode getAFalseSuccessor() { + NewControlFlowNode.super.isAfterFalse(_) and + // Only where there's also a true branch (true boolean split) + exists(NewControlFlowNode other | other.isAfterTrue(NewControlFlowNode.super.getAstNode())) and + nextCfgNodeFrom(this, result) + } + + CfgNode getAnExceptionalSuccessor() { + exists(NewControlFlowNode mid | + mid = NewControlFlowNode.super.getAnExceptionSuccessor() and + nextCfgNodeFrom(mid, result) + ) + } + + Py::Scope getScope() { result = NewControlFlowNode.super.getEnclosingCallable().asScope() } + + BasicBlock getBasicBlock() { + exists(NewBasicBlock bb, int i | bb.getNode(i) = this and result = bb) + } + } + + /** + * Holds if `next` is the nearest CfgNode reachable from `n` via + * one or more raw CFG successor edges, skipping non-CfgNode intermediaries. + */ + private predicate nextCfgNodeFrom(NewControlFlowNode n, CfgNode next) { + next = n.getASuccessor() + or + exists(NewControlFlowNode mid | + mid = n.getASuccessor() and + not mid instanceof CfgNode and + nextCfgNodeFrom(mid, next) + ) + } + + /** + * Holds if `next` is the nearest CfgNode successor of `n`, + * skipping synthetic intermediate nodes. + */ + private predicate nextCfgNode(CfgNode n, CfgNode next) { nextCfgNodeFrom(n, next) } + + class BasicBlock instanceof NewBasicBlock { + string toString() { result = NewBasicBlock.super.toString() } + + CfgNode getNode(int n) { result = NewBasicBlock.super.getNode(n) } + + predicate reaches(BasicBlock bb) { this = bb or this.strictlyReaches(bb) } + + predicate strictlyReaches(BasicBlock bb) { NewBasicBlock.super.getASuccessor+() = bb } + + predicate strictlyDominates(BasicBlock bb) { NewBasicBlock.super.strictlyDominates(bb) } + } + + CfgNode scopeGetEntryNode(Py::Scope s) { + exists(CfgImpl::ControlFlow::EntryNode entry | + entry.getEnclosingCallable().asScope() = s and + nextCfgNodeFrom(entry, result) + ) + } +} diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.ql new file mode 100644 index 00000000000..6949b2cc6e9 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.ql @@ -0,0 +1,21 @@ +/** + * New-CFG version of NeverReachable. + * + * Original: + * Checks that expressions annotated with `t.never` either have no CFG + * node, or if they do, that the node is not reachable from its scope's + * entry (including within the same basic block). + */ + +import python +import TimerUtils +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils::CfgTests + +from TimerAnnotation ann +where neverReachable(ann) +select ann, "Node annotated with t.never is reachable in $@", ann.getTestFunction(), + ann.getTestFunction().getName() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.ql new file mode 100644 index 00000000000..442ca5f5456 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.ql @@ -0,0 +1,22 @@ +/** + * New-CFG version of NoBackwardFlow. + * + * Original: + * Checks that time never flows backward between consecutive timer annotations + * in the CFG. For each pair of consecutive annotated nodes (A -> B), there must + * exist timestamps a in A and b in B with a < b. + */ + +import python +import TimerUtils +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from TimerCfgNode a, TimerCfgNode b, int minA, int maxB +where noBackwardFlow(a, b, minA, maxB) +select a, "Backward flow: $@ flows to $@ (max timestamp $@)", a.getTimestampExpr(minA), + minA.toString(), b, b.getNode().toString(), b.getTimestampExpr(maxB), maxB.toString() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBasicBlock.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBasicBlock.expected new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBasicBlock.expected @@ -0,0 +1 @@ + diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBasicBlock.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBasicBlock.ql new file mode 100644 index 00000000000..e07890f7250 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBasicBlock.ql @@ -0,0 +1,18 @@ +/** + * New-CFG version of NoBasicBlock. + * + * Checks that every annotated CFG node belongs to a basic block. + */ + +import python +import TimerUtils +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from CfgNode n, TestFunction f +where noBasicBlock(n, f) +select n, "CFG node in $@ does not belong to any basic block", f, f.getName() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.ql new file mode 100644 index 00000000000..5a1a1aba2a7 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.ql @@ -0,0 +1,21 @@ +/** + * New-CFG version of NoSharedReachable. + * + * Original: + * Checks that two annotations sharing a timestamp value are on + * mutually exclusive CFG paths (neither can reach the other). + */ + +import python +import TimerUtils +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from TimerCfgNode a, TimerCfgNode b, int ts +where noSharedReachable(a, b, ts) +select a, "Shared timestamp $@ but this node reaches $@", a.getTimestampExpr(ts), ts.toString(), b, + b.getNode().toString() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.ql new file mode 100644 index 00000000000..ebbc60346db --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.ql @@ -0,0 +1,22 @@ +/** + * New-CFG version of StrictForward. + * + * Original: + * Stronger version of NoBackwardFlow: for consecutive annotated nodes + * A -> B that both have a single timestamp (non-loop code) and B does + * NOT dominate A (forward edge), requires max(A) < min(B). + */ + +import python +import TimerUtils +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from TimerCfgNode a, TimerCfgNode b, int maxA, int minB +where strictForward(a, b, maxA, minB) +select a, "Strict forward violation: $@ flows to $@", a.getTimestampExpr(maxA), "timestamp " + maxA, + b.getTimestampExpr(minB), "timestamp " + minB diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/OldCfgImpl.qll b/python/ql/test/library-tests/ControlFlow/evaluation-order/OldCfgImpl.qll index cb7bbb495b8..fc52c8dd3ed 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/OldCfgImpl.qll +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/OldCfgImpl.qll @@ -3,14 +3,14 @@ * Python control flow graph. */ -private import python as PY +private import python as Py import TimerUtils /** Existing Python CFG implementation of the evaluation-order signature. */ module OldCfg implements EvalOrderCfgSig { - class CfgNode = PY::ControlFlowNode; + class CfgNode = Py::ControlFlowNode; - class BasicBlock = PY::BasicBlock; + class BasicBlock = Py::BasicBlock; - CfgNode scopeGetEntryNode(PY::Scope s) { result = s.getEntryNode() } + CfgNode scopeGetEntryNode(Py::Scope s) { result = s.getEntryNode() } } diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/test_if.py b/python/ql/test/library-tests/ControlFlow/evaluation-order/test_if.py index 8880aaaef34..a6eb6c7d5ca 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/test_if.py +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/test_if.py @@ -85,7 +85,7 @@ def test_nested_if_else(t): else: z = 2 @ t[dead(4)] else: - z = 3 @ t[dead(4)] + z = 3 @ t[dead(3), dead(4)] w = 0 @ t[5] From fbfbbd342a0120beb0db6e4280f3d2e7d624df4f Mon Sep 17 00:00:00 2001 From: yoff Date: Tue, 2 Jun 2026 14:09:28 +0000 Subject: [PATCH 03/90] Python: add new shared-CFG-backed control flow graph facade (Cfg) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the public facade on top of the AstNodeImpl adapter from the previous commit. Re-exposes the same API surface as semmle/python/Flow.qll (ControlFlowNode, CallNode, BasicBlock, NameNode, DefinitionNode, CompareNode, ...), backed by the shared codeql.controlflow.ControlFlowGraph library. - semmle.python.controlflow.internal.Cfg — public facade. - ControlFlow/store-load/* — basic store/load coverage via the facade. The new CFG library is added additively: it has zero callers in lib/ and src/, and the legacy CFG in semmle/python/Flow.qll remains the default. Dataflow, SSA, and production query migration land in follow-up PRs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../change-notes/2026-05-19-add-shared-cfg.md | 4 + .../python/controlflow/internal/Cfg.qll | 1025 +++++++++++++++++ .../store-load/StoreLoadTest.expected | 0 .../ControlFlow/store-load/StoreLoadTest.ql | 41 + .../ControlFlow/store-load/test.py | 56 + 5 files changed, 1126 insertions(+) create mode 100644 python/ql/lib/change-notes/2026-05-19-add-shared-cfg.md create mode 100644 python/ql/lib/semmle/python/controlflow/internal/Cfg.qll create mode 100644 python/ql/test/library-tests/ControlFlow/store-load/StoreLoadTest.expected create mode 100644 python/ql/test/library-tests/ControlFlow/store-load/StoreLoadTest.ql create mode 100644 python/ql/test/library-tests/ControlFlow/store-load/test.py diff --git a/python/ql/lib/change-notes/2026-05-19-add-shared-cfg.md b/python/ql/lib/change-notes/2026-05-19-add-shared-cfg.md new file mode 100644 index 00000000000..913f95320d8 --- /dev/null +++ b/python/ql/lib/change-notes/2026-05-19-add-shared-cfg.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* A new Python control flow graph implementation has been added under `semmle.python.controlflow.internal.Cfg` (backed by `AstNodeImpl.qll`), built on the shared `codeql.controlflow.ControlFlowGraph` library. It is not yet used by the dataflow library or any production query; the legacy CFG in `semmle/python/Flow.qll` remains the default. The new library is exposed for tests and for upcoming migrations. diff --git a/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll b/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll new file mode 100644 index 00000000000..2d39ae8450e --- /dev/null +++ b/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll @@ -0,0 +1,1025 @@ +/** + * Provides a Python control flow graph facade backed by the shared + * `codeql.controlflow.ControlFlowGraph` library (via `AstNodeImpl.qll`). + * + * This module re-exposes the same API surface as `semmle/python/Flow.qll` + * (the legacy CFG), but is implemented on the new shared CFG. It is + * intended as a drop-in replacement for use by the Python dataflow library + * and other downstream code. + * + * Layering follows the Java pattern (`java/ql/lib/semmle/code/java/Expr.qll` + * and `SsaImpl.qll`): variable identity and similar AST-level semantics + * live on the Python AST classes (`Name.defines(v)`, `Name.uses(v)`, ...); + * the CFG layer is purely positional, with `toAst` / `getNode` bridging + * back to the AST. The shared SSA library can then be parameterized on + * (`BasicBlock`, `int`) directly, with no CFG-level variable predicates. + */ +overlay[local?] +module; + +private import python as Py +private import semmle.python.controlflow.internal.AstNodeImpl as CfgImpl +private import codeql.controlflow.SuccessorType +private import codeql.controlflow.BasicBlock as BB + +/** + * A nested sub-module that explicitly implements `BB::CfgSig`, so this + * `Cfg` facade can be passed to parameterised shared modules such as + * `codeql.dataflow.VariableCapture::Flow`. The sub-module + * exposes the *raw* shared-CFG types from `AstNodeImpl.qll` (where the + * signature is satisfied natively), not the facade's wrapped types. + */ +module CfgForBb implements BB::CfgSig { + class ControlFlowNode = CfgImpl::ControlFlowNode; + + class BasicBlock = CfgImpl::BasicBlock; + + class EntryBasicBlock = CfgImpl::Cfg::EntryBasicBlock; + + predicate dominatingEdge = CfgImpl::Cfg::dominatingEdge/2; +} + +/** + * Gets the Python AST node corresponding to CFG node `n`, if any. + * + * Multiple CFG nodes may map to the same AST node (e.g. `TBeforeNode(Call)` + * and `TAstNode(Call)` both map to `Py::Call`). This is a pure translation; + * uniqueness constraints are enforced at the dataflow layer where needed. + */ +private Py::AstNode toAst(CfgImpl::ControlFlowNode n) { + result = CfgImpl::astNodeToPyNode(n.getAstNode()) +} + +/** + * A control flow node. + * + * This is the full set of CFG nodes from the shared library — it includes + * before-nodes, in-order/post-order nodes, after-value-split nodes, and + * entry/exit nodes. This enables full control-flow-level reasoning and + * compatibility with the shared control-flow reachability library. + * + * AST-level semantics (`getNode()`, `isLoad()`, typed wrappers, etc.) + * are available only on the `injects` (canonical) node for each AST node. + * Non-injects nodes are purely positional CFG nodes with no AST mapping. + */ +class ControlFlowNode extends CfgImpl::ControlFlowNode { + /** Gets the syntactic element corresponding to this flow node, if any. */ + Py::AstNode getNode() { result = toAst(this) } + + /** Gets a predecessor of this flow node. */ + ControlFlowNode getAPredecessor() { this = result.getASuccessor() } + + /** Gets a successor of this flow node. */ + ControlFlowNode getASuccessor() { result = super.getASuccessor() } + + /** Gets a successor for this node if the relevant condition is True. */ + ControlFlowNode getATrueSuccessor() { + result = super.getASuccessor(any(BooleanSuccessor t | t.getValue() = true)) + } + + /** Gets a successor for this node if the relevant condition is False. */ + ControlFlowNode getAFalseSuccessor() { + result = super.getASuccessor(any(BooleanSuccessor t | t.getValue() = false)) + } + + /** Gets a successor for this node if an exception is raised. */ + ControlFlowNode getAnExceptionalSuccessor() { result = super.getAnExceptionSuccessor() } + + /** Gets a successor for this node if no exception is raised. */ + ControlFlowNode getANormalSuccessor() { result = super.getANormalSuccessor() } + + /** Gets the basic block containing this flow node. */ + BasicBlock getBasicBlock() { result = super.getBasicBlock() } + + /** Gets the scope containing this flow node. */ + Py::Scope getScope() { result = super.getEnclosingCallable().asScope() } + + /** Gets the enclosing module. */ + Py::Module getEnclosingModule() { result = this.getScope().getEnclosingModule() } + + /** Gets the immediate dominator of this flow node. */ + ControlFlowNode getImmediateDominator() { + // Defined positionally via the basic-block dominance tree. + exists(BasicBlock bb, int i | bb.getNode(i) = this | + // Predecessor within the same basic block. + i > 0 and result = bb.getNode(i - 1) + or + // First node of `bb`: dominator is the last node of the immediate dominator block. + i = 0 and result = bb.getImmediateDominator().getLastNode() + ) + } + + /** Holds if this strictly dominates `other`. */ + pragma[inline] + predicate strictlyDominates(ControlFlowNode other) { super.strictlyDominates(other) } + + /** Holds if this dominates `other` (reflexively). */ + pragma[inline] + predicate dominates(ControlFlowNode other) { super.dominates(other) } + + /** Holds if this is the first node in its enclosing scope. */ + predicate isEntryNode() { this instanceof CfgImpl::ControlFlow::EntryNode } + + /** Holds if this is the first node of a module. */ + predicate isModuleEntry() { + this.isEntryNode() and super.getAstNode().asScope() instanceof Py::Module + } + + /** Holds if this node may exit its scope by raising an exception. */ + predicate isExceptionalExit(Py::Scope s) { + this instanceof CfgImpl::ControlFlow::ExceptionalExitNode and + super.getEnclosingCallable().asScope() = s + } + + /** Holds if this node is a normal (non-exceptional) exit. */ + predicate isNormalExit() { this instanceof CfgImpl::ControlFlow::NormalExitNode } + + // ===== AST-shape predicates (bridges to the wrapped Python AST) ===== + /** + * Holds if this flow node is a load (including those in augmented + * assignments). + * + * Note: an augmented-assignment target (`x[i]` in `x[i] += 1`) is + * both a load and a store — `isLoad` and `isStore` both hold on the + * canonical CFG node. This mirrors Java's `VarAccess.isVarRead`, + * which holds on the destination of compound and unary assignments + * even though the destination is also a write. + */ + predicate isLoad() { exists(Py::Expr e | e = toAst(this) | py_expr_contexts(_, 3, e)) } + + /** Holds if this flow node is a store (including those in augmented assignments). */ + predicate isStore() { + exists(Py::Expr e | e = toAst(this) | py_expr_contexts(_, 5, e) or augstore(_, this)) + } + + /** Holds if this flow node is a delete. */ + predicate isDelete() { exists(Py::Expr e | e = toAst(this) | py_expr_contexts(_, 2, e)) } + + /** Holds if this flow node is a parameter. */ + predicate isParameter() { exists(Py::Expr e | e = toAst(this) | py_expr_contexts(_, 4, e)) } + + /** Holds if this flow node is a store in an augmented assignment. */ + predicate isAugStore() { augstore(_, this) } + + /** Holds if this flow node is a load in an augmented assignment. */ + predicate isAugLoad() { augstore(this, _) } + + /** Holds if this flow node corresponds to a literal. */ + predicate isLiteral() { + toAst(this) instanceof Py::Bytes or + toAst(this) instanceof Py::Dict or + toAst(this) instanceof Py::DictComp or + toAst(this) instanceof Py::Set or + toAst(this) instanceof Py::SetComp or + toAst(this) instanceof Py::Ellipsis or + toAst(this) instanceof Py::GeneratorExp or + toAst(this) instanceof Py::Lambda or + toAst(this) instanceof Py::ListComp or + toAst(this) instanceof Py::List or + toAst(this) instanceof Py::Num or + toAst(this) instanceof Py::Tuple or + toAst(this) instanceof Py::Unicode or + toAst(this) instanceof Py::NameConstant + } + + /** Holds if this flow node corresponds to an attribute expression. */ + predicate isAttribute() { toAst(this) instanceof Py::Attribute } + + /** Holds if this flow node corresponds to a subscript expression. */ + predicate isSubscript() { toAst(this) instanceof Py::Subscript } + + /** Holds if this flow node corresponds to an import member. */ + predicate isImportMember() { toAst(this) instanceof Py::ImportMember } + + /** Holds if this flow node corresponds to a call. */ + predicate isCall() { toAst(this) instanceof Py::Call } + + /** Holds if this flow node corresponds to an import. */ + predicate isImport() { toAst(this) instanceof Py::ImportExpr } + + /** Holds if this flow node corresponds to a conditional expression. */ + predicate isIfExp() { toAst(this) instanceof Py::IfExp } + + /** Holds if this flow node corresponds to a function definition expression. */ + predicate isFunction() { toAst(this) instanceof Py::FunctionExpr } + + /** Holds if this flow node corresponds to a class definition expression. */ + predicate isClass() { toAst(this) instanceof Py::ClassExpr } + + /** + * Holds if this flow node is a branch (i.e. has both a true and a + * false successor). + */ + predicate isBranch() { exists(this.getATrueSuccessor()) or exists(this.getAFalseSuccessor()) } + + /** + * Gets a CFG child of this node, defined as a CFG node whose AST node + * is a child of this CFG node's AST node, restricted to nodes that + * dominate this one (so the child has been evaluated by the time we + * reach this node). + * + * Mirrors `Flow.qll`'s `getAChild`. UnaryExprNode is excluded because + * its operand is its CFG predecessor (handled separately). + */ + pragma[nomagic] + ControlFlowNode getAChild() { + toAst(this).(Py::Expr).getAChildNode() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) and + not this instanceof UnaryExprNode + } + + /** Holds if this flow node strictly reaches `other`. */ + predicate strictlyReaches(ControlFlowNode other) { this.getASuccessor+() = other } +} + +/** + * Holds if `load` is the load half of an augmented-assignment target, + * and `store` is the corresponding store half. + * + * In the legacy CFG (`Flow.qll`) the same Python `Name` had two + * distinct CFG nodes — a load node (context 3) earlier in the BB, and + * a store node (context 5) later. The legacy `augstore` related the + * pair via dominance. + * + * In the new (shared) CFG, the canonical node for an AST expression is + * unique, so `load` and `store` collapse onto the same CFG node. The + * predicate is therefore reflexive on the augmented-assignment + * target's canonical node. + */ +private predicate augstore(ControlFlowNode load, ControlFlowNode store) { + exists(Py::AugAssign aa | aa.getTarget() = toAst(load)) and + load = store +} + +/** + * A basic block — a maximal-length sequence of control flow nodes such + * that no node except the first has a predecessor outside the sequence, + * and no node except the last has a successor outside the sequence. + */ +class BasicBlock extends CfgImpl::BasicBlock { + /** Gets the `n`th node in this basic block. */ + ControlFlowNode getNode(int n) { result = super.getNode(n) } + + /** Gets a node in this basic block. */ + ControlFlowNode getANode() { result = super.getNode(_) } + + /** Gets the first node in this basic block. */ + ControlFlowNode firstNode() { result = this.getNode(0) } + + /** Gets the last node in this basic block. */ + ControlFlowNode getLastNode() { result = super.getLastNode() } + + /** Holds if this basic block contains `node`. */ + predicate contains(ControlFlowNode node) { node = this.getANode() } + + // Inherited from the shared library's `BasicBlock`: + // getASuccessor(), getASuccessor(SuccessorType), getAPredecessor(), + // strictlyDominates(), dominates(), getImmediateDominator(), + // length(), inLoop(). + // We shadow `getNode(int)` etc. to return `ControlFlowNode` (this + // facade's type) and add Python-style helpers below. + /** Gets a true successor to this basic block. */ + BasicBlock getATrueSuccessor() { + result = super.getASuccessor(any(BooleanSuccessor t | t.getValue() = true)) + } + + /** Gets a false successor to this basic block. */ + BasicBlock getAFalseSuccessor() { + result = super.getASuccessor(any(BooleanSuccessor t | t.getValue() = false)) + } + + /** Gets an unconditional successor to this basic block. */ + BasicBlock getAnUnconditionalSuccessor() { + result = super.getASuccessor() and + not result = this.getATrueSuccessor() and + not result = this.getAFalseSuccessor() + } + + /** Gets an exceptional successor to this basic block. */ + BasicBlock getAnExceptionalSuccessor() { result = super.getASuccessor(any(ExceptionSuccessor t)) } + + /** + * Holds if this basic block is in the dominance frontier of `df`. + * + * Note: implemented locally rather than via the shared lib, which + * doesn't currently expose a `dominanceFrontier` predicate at this + * level. + */ + predicate inDominanceFrontier(BasicBlock df) { + this = df.getAPredecessor() and not this = df.getImmediateDominator() + or + exists(BasicBlock prev | prev.inDominanceFrontier(df) | + this = prev.getImmediateDominator() and + not this = df.getImmediateDominator() + ) + } + + /** Holds if this basic block strictly reaches `other`. */ + predicate strictlyReaches(BasicBlock other) { super.getASuccessor+() = other } + + /** Holds if this basic block reaches `other` (reflexively). */ + predicate reaches(BasicBlock other) { this = other or this.strictlyReaches(other) } + + /** Holds if flow from this basic block reaches a normal exit from its scope. */ + predicate reachesExit() { + this.getANode() instanceof CfgImpl::ControlFlow::NormalExitNode + or + exists(BasicBlock succ | succ = super.getASuccessor() and succ.reachesExit()) + } + + /** Gets the scope of this basic block. */ + Py::Scope getScope() { exists(ControlFlowNode n | n = this.getANode() | result = n.getScope()) } + + /** Holds if flow from this BasicBlock always reaches `succ`. */ + predicate alwaysReaches(BasicBlock succ) { + succ = this + or + strictcount(BasicBlock s | s = super.getASuccessor()) = 1 and + succ = super.getASuccessor() + or + forex(BasicBlock immsucc | immsucc = super.getASuccessor() | immsucc.alwaysReaches(succ)) + } + + /** + * Holds if this basic block ends in a node that branches on a boolean + * outcome, and `other` is dominated by the corresponding successor + * for `branch` while not being reachable from the other branch + * without going through this BB. + * + * In other words: any execution that reaches `other` must have just + * evaluated the last node of this BB and taken the `branch` outcome. + * This mirrors the legacy `ConditionBlock.controls(BB, branch)`. + */ + predicate controls(BasicBlock other, boolean branch) { + exists(BasicBlock succ | + branch = true and succ = this.getATrueSuccessor() + or + branch = false and succ = this.getAFalseSuccessor() + | + succ.dominates(other) and + // The other branch must not also reach `other` — otherwise + // `other` is not actually controlled by `branch`. + not exists(BasicBlock otherSucc | + branch = true and otherSucc = this.getAFalseSuccessor() + or + branch = false and otherSucc = this.getATrueSuccessor() + | + otherSucc.reaches(other) + ) + ) + } +} + +// =========================================================================== +// Re-exports for SSA / dominance consumers +// +// The shared `BB::CfgSig` requires `EntryBasicBlock` and `dominatingEdge` in +// addition to the BasicBlock class we already expose. They are provided by +// the shared CFG library on the `BB::Make` instantiation produced by +// `AstNodeImpl.qll`. +// =========================================================================== +/** An entry basic block, that is, a basic block whose first node is an entry node. */ +class EntryBasicBlock = CfgImpl::Cfg::EntryBasicBlock; + +/** + * Holds if `bb1` has `bb2` as a direct successor and the edge between `bb1` + * and `bb2` is a dominating edge. + */ +predicate dominatingEdge = CfgImpl::Cfg::dominatingEdge/2; + +// =========================================================================== +// AST-shape subclasses of ControlFlowNode +// +// Each class is a thin wrapper around the canonical CFG node for a given +// kind of Python AST node. Methods that take/return CFG nodes look up +// related CFG nodes by AST identity (via `getNode()`), and the dominance +// constraint from the old CFG (`result.getBasicBlock().dominates(this.getBasicBlock())`) +// is preserved. +// =========================================================================== +/** Gets the canonical `ControlFlowNode` for AST expression `e`. */ +ControlFlowNode astExprToCfg(Py::Expr e) { result.getNode() = e } + +/** A control flow node corresponding to a `Name` or `PlaceHolder` expression. */ +class NameNode extends ControlFlowNode { + NameNode() { + toAst(this) instanceof Py::Name + or + toAst(this) instanceof Py::PlaceHolder + } + + /** + * Holds if this flow node defines the variable `v`. + * + * This includes augmented-assignment targets — `n += 1` is both a + * read and a write of `n`, so `defines(n)` and `uses(n)` both hold + * on the same canonical CFG node. Mirrors Java's `VariableUpdate` + * semantics where compound assignments register both a write + * (`VarWrite`) and a read (`VarRead`) on the destination. + */ + predicate defines(Py::Variable v) { exists(Py::Name n | n = toAst(this) and n.defines(v)) } + + /** Holds if this flow node deletes the variable `v`. */ + predicate deletes(Py::Variable v) { exists(Py::Name n | n = toAst(this) and n.deletes(v)) } + + /** Holds if this flow node uses the variable `v`. */ + predicate uses(Py::Variable v) { + this.isLoad() and + exists(Py::Name u | u = toAst(this) and u.uses(v)) + or + exists(Py::PlaceHolder u | + u = toAst(this) and u.getVariable() = v and u.getCtx() instanceof Py::Load + ) + } + + /** Gets the identifier of this name node. */ + string getId() { + result = toAst(this).(Py::Name).getId() + or + result = toAst(this).(Py::PlaceHolder).getId() + } + + /** Holds if this is a use of a local variable. */ + predicate isLocal() { exists(Py::Variable v | this.uses(v) and v instanceof Py::LocalVariable) } + + /** Holds if this is a use of a non-local variable. */ + predicate isNonLocal() { + exists(Py::Variable v | this.uses(v) and v.getScope() != this.getScope()) + } + + /** Holds if this is a use of a global (including builtin) variable. */ + predicate isGlobal() { exists(Py::Variable v | this.uses(v) and v instanceof Py::GlobalVariable) } + + /** + * Holds if this is a use of `self` — the first parameter of an + * enclosing method. + * + * AST-level approximation: matches when the Name uses a `Variable` + * that is the first parameter of an enclosing `Function` defined + * inside a `Class`. + */ + predicate isSelf() { + exists(Py::Variable v, Py::Function f, Py::Class c | + this.uses(v) and + f = c.getAMethod() and + v.getScope() = f and + v = f.getArg(0).(Py::Name).getVariable() + ) + } +} + +/** A control flow node corresponding to a named constant (`None`, `True`, `False`). */ +class NameConstantNode extends NameNode { + NameConstantNode() { toAst(this) instanceof Py::NameConstant } +} + +/** A control flow node corresponding to a call. */ +class CallNode extends ControlFlowNode { + CallNode() { toAst(this) instanceof Py::Call } + + override Py::Call getNode() { result = super.getNode() } + + /** Gets the underlying Python `Call`. */ + Py::Call getCall() { result = toAst(this) } + + /** Gets the flow node for the function component of this call. */ + ControlFlowNode getFunction() { + exists(Py::Call c | + c = toAst(this) and + c.getFunc() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } + + /** Gets the flow node for the `n`th positional argument. */ + ControlFlowNode getArg(int n) { + exists(Py::Call c | + c = toAst(this) and + c.getArg(n) = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } + + /** Gets the flow node for the named argument with name `name`. */ + ControlFlowNode getArgByName(string name) { + exists(Py::Call c, Py::Keyword k | + c = toAst(this) and + k = c.getANamedArg() and + k.getValue() = toAst(result) and + k.getArg() = name and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } + + /** Gets a flow node corresponding to any argument. */ + ControlFlowNode getAnArg() { result = this.getArg(_) or result = this.getArgByName(_) } + + /** Gets the first tuple (`*args`) argument, if any. */ + ControlFlowNode getStarArg() { + exists(Py::Call c | + c = toAst(this) and + c.getStarArg() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } + + /** Gets a dictionary (`**kwargs`) argument, if any. */ + ControlFlowNode getKwargs() { + exists(Py::Call c | + c = toAst(this) and + c.getKwargs() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } + + /** Holds if this call is a decorator call applied to a class or a function. */ + predicate isDecoratorCall() { this.isClassDecoratorCall() or this.isFunctionDecoratorCall() } + + /** Holds if this call is a decorator call applied to a class. */ + predicate isClassDecoratorCall() { + exists(Py::ClassExpr cls | toAst(this) = cls.getADecoratorCall()) + } + + /** Holds if this call is a decorator call applied to a function. */ + predicate isFunctionDecoratorCall() { + exists(Py::FunctionExpr func | toAst(this) = func.getADecoratorCall()) + } +} + +/** A control flow node corresponding to an attribute expression. */ +class AttrNode extends ControlFlowNode { + AttrNode() { toAst(this) instanceof Py::Attribute } + + override Py::Attribute getNode() { result = super.getNode() } + + /** Gets the flow node for the object of the attribute expression. */ + ControlFlowNode getObject() { + exists(Py::Attribute a | + a = toAst(this) and + a.getObject() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } + + /** Gets the flow node for the object of this attribute expression, with the matching name. */ + ControlFlowNode getObject(string name) { + exists(Py::Attribute a | + a = toAst(this) and + a.getObject() = toAst(result) and + a.getName() = name and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } + + /** Gets the attribute name. */ + string getName() { exists(Py::Attribute a | a = toAst(this) and a.getName() = result) } +} + +/** A control flow node corresponding to an import statement (`import x`). */ +class ImportExprNode extends ControlFlowNode { + ImportExprNode() { toAst(this) instanceof Py::ImportExpr } + + override Py::ImportExpr getNode() { result = super.getNode() } +} + +/** A control flow node corresponding to a `from ... import name` expression. */ +class ImportMemberNode extends ControlFlowNode { + ImportMemberNode() { toAst(this) instanceof Py::ImportMember } + + override Py::ImportMember getNode() { result = super.getNode() } + + /** Gets the flow node for the module being imported from, with the matching name. */ + ControlFlowNode getModule(string name) { + exists(Py::ImportMember i | + i = toAst(this) and + i.getModule() = toAst(result) and + i.getName() = name and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } +} + +/** A control flow node corresponding to a `from ... import *` statement. */ +class ImportStarNode extends ControlFlowNode { + ImportStarNode() { toAst(this) instanceof Py::ImportStar } + + override Py::ImportStar getNode() { result = super.getNode() } + + /** Gets the flow node for the module being imported from. */ + ControlFlowNode getModule() { + exists(Py::ImportStar i | + i = toAst(this) and + i.getModuleExpr() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } +} + +/** A control flow node corresponding to a subscript expression. */ +class SubscriptNode extends ControlFlowNode { + SubscriptNode() { toAst(this) instanceof Py::Subscript } + + override Py::Subscript getNode() { result = super.getNode() } + + /** Gets the flow node for the value being subscripted. */ + ControlFlowNode getObject() { + exists(Py::Subscript s | + s = toAst(this) and + s.getObject() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } + + /** Gets the flow node for the index expression. */ + ControlFlowNode getIndex() { + exists(Py::Subscript s | + s = toAst(this) and + s.getIndex() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } +} + +/** A control flow node corresponding to a comparison operation. */ +class CompareNode extends ControlFlowNode { + CompareNode() { toAst(this) instanceof Py::Compare } + + override Py::Compare getNode() { result = super.getNode() } + + /** Holds if `left` and `right` are a pair of operands for this comparison. */ + predicate operands(ControlFlowNode left, Py::Cmpop op, ControlFlowNode right) { + exists(Py::Compare c, Py::Expr eleft, Py::Expr eright | + c = toAst(this) and eleft = toAst(left) and eright = toAst(right) + | + eleft = c.getLeft() and eright = c.getComparator(0) and op = c.getOp(0) + or + exists(int i | + eleft = c.getComparator(i - 1) and eright = c.getComparator(i) and op = c.getOp(i) + ) + ) and + left.getBasicBlock().dominates(this.getBasicBlock()) and + right.getBasicBlock().dominates(this.getBasicBlock()) + } +} + +/** A control flow node corresponding to a conditional expression (`x if c else y`). */ +class IfExprNode extends ControlFlowNode { + IfExprNode() { toAst(this) instanceof Py::IfExp } + + override Py::IfExp getNode() { result = super.getNode() } + + /** Gets the flow node for one of the value operands (true-branch or false-branch). */ + ControlFlowNode getAnOperand() { + exists(Py::IfExp ie | + ie = toAst(this) and + (toAst(result) = ie.getBody() or toAst(result) = ie.getOrelse()) + ) + } +} + +/** A control flow node corresponding to an assignment expression (walrus `:=`). */ +class AssignmentExprNode extends ControlFlowNode { + AssignmentExprNode() { toAst(this) instanceof Py::AssignExpr } + + override Py::AssignExpr getNode() { result = super.getNode() } + + /** Gets the flow node for the left-hand side. */ + ControlFlowNode getTarget() { + exists(Py::AssignExpr a | + a = toAst(this) and + a.getTarget() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } + + /** Gets the flow node for the right-hand side. */ + ControlFlowNode getValue() { + exists(Py::AssignExpr a | + a = toAst(this) and + a.getValue() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } +} + +/** A control flow node corresponding to a binary expression (`a + b` etc.). */ +class BinaryExprNode extends ControlFlowNode { + BinaryExprNode() { toAst(this) instanceof Py::BinaryExpr } + + override Py::BinaryExpr getNode() { result = super.getNode() } + + ControlFlowNode getLeft() { + exists(Py::BinaryExpr be | + be = toAst(this) and + be.getLeft() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } + + ControlFlowNode getRight() { + exists(Py::BinaryExpr be | + be = toAst(this) and + be.getRight() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } + + Py::Operator getOp() { result = toAst(this).(Py::BinaryExpr).getOp() } + + /** Holds if `left` and `right` are the operands and `op` is the operator. */ + predicate operands(ControlFlowNode left, Py::Operator op, ControlFlowNode right) { + left = this.getLeft() and right = this.getRight() and op = this.getOp() + } + + /** Gets either operand. */ + ControlFlowNode getAnOperand() { result = this.getLeft() or result = this.getRight() } +} + +/** A control flow node corresponding to a boolean expression (`a and b`, `a or b`). */ +class BoolExprNode extends ControlFlowNode { + BoolExprNode() { toAst(this) instanceof Py::BoolExpr } + + override Py::BoolExpr getNode() { result = super.getNode() } + + Py::Boolop getOp() { result = toAst(this).(Py::BoolExpr).getOp() } + + /** Gets any operand of this boolean expression. */ + ControlFlowNode getAnOperand() { + exists(Py::BoolExpr be | + be = toAst(this) and + be.getAValue() = toAst(result) + ) + } +} + +/** A control flow node corresponding to a unary expression (`-x`, `not x`, etc.). */ +class UnaryExprNode extends ControlFlowNode { + UnaryExprNode() { toAst(this) instanceof Py::UnaryExpr } + + override Py::UnaryExpr getNode() { result = super.getNode() } + + ControlFlowNode getOperand() { + exists(Py::UnaryExpr u | + u = toAst(this) and + u.getOperand() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } + + Py::Unaryop getOp() { result = toAst(this).(Py::UnaryExpr).getOp() } +} + +/** + * A control flow node that is a definition: it appears in a context that + * binds a variable (assignment target, parameter, etc.). + */ +class DefinitionNode extends ControlFlowNode { + DefinitionNode() { this.isStore() or this.isParameter() } + + /** Gets the value assigned, if any. */ + ControlFlowNode getValue() { + // For-target: the value is the for-loop's iter expression (which + // is also where `Cfg::ForNode` lives — its `getNode()` returns the + // enclosing `Py::For` statement). Treated specially because there + // is no AST node holding the result of `iter(next(seq))`; we use + // the iter expression's CFG node as the stand-in. + exists(Py::For f | + f.getTarget() = toAst(this) and + toAst(result) = f.getIter() + ) + or + exists(Py::AstNode value | value = assignedValue(toAst(this)) | + toAst(result) = value and + ( + result.getBasicBlock().dominates(this.getBasicBlock()) + or + result.isImport() + or + // The default value for a parameter is evaluated in the same basic block as + // the function definition, but the parameter belongs to the basic block of the + // function, so there is no dominance relationship between the two. + exists(Py::Parameter param | toAst(this) = param.asName()) + ) + ) + } +} + +/** + * Gets the AST node that holds the value assigned to `lhs` in a binding + * context. Mirrors `Flow.qll::assigned_value`. + */ +private Py::AstNode assignedValue(Py::Expr lhs) { + // lhs = result + exists(Py::Assign a | a.getATarget() = lhs and result = a.getValue()) + or + // lhs := result + exists(Py::AssignExpr a | a.getTarget() = lhs and result = a.getValue()) + or + // lhs: annotation = result + exists(Py::AnnAssign a | a.getTarget() = lhs and result = a.getValue()) + or + // import result as lhs (also covers plain `import lhs`, where alias.getAsname() = lhs) + exists(Py::Alias a | a.getAsname() = lhs and result = a.getValue()) + or + // lhs += x -> result is the (lhs + x) binary expression + exists(Py::AugAssign a, Py::BinaryExpr b | + b = a.getOperation() and result = b and lhs = b.getLeft() + ) + or + // Nested sequence assign: ..., lhs, ... = ..., result, ... + exists(Py::Assign a | nestedSequenceAssign(a.getATarget(), a.getValue(), lhs, result)) + or + // Parameter default + exists(Py::Parameter param | lhs = param.asName() and result = param.getDefault()) +} + +/** + * Helper for nested sequence assignments such as `(a, b), c = (1, 2), 3`. + */ +private predicate nestedSequenceAssign( + Py::Expr leftParent, Py::Expr rightParent, Py::Expr left, Py::Expr right +) { + exists(int i | + leftParent.(Py::Tuple).getElt(i) = left and rightParent.(Py::Tuple).getElt(i) = right + or + leftParent.(Py::List).getElt(i) = left and rightParent.(Py::List).getElt(i) = right + ) + or + exists(Py::Expr leftMid, Py::Expr rightMid | + nestedSequenceAssign(leftParent, rightParent, leftMid, rightMid) and + nestedSequenceAssign(leftMid, rightMid, left, right) + ) +} + +/** A control flow node corresponding to a deletion (`del x`). */ +class DeletionNode extends ControlFlowNode { + DeletionNode() { this.isDelete() } +} + +/** A control flow node corresponding to a `for` loop target. */ +class ForNode extends ControlFlowNode { + ForNode() { exists(Py::For f | toAst(this) = f.getIter()) } + + /** Gets the iterable expression. */ + ControlFlowNode getIter() { + result = this and result = result // canonical "after" of the iterable + } + + /** Gets the sequence expression (alias for `getIter()`, matches legacy Flow naming). */ + ControlFlowNode getSequence() { result = this.getIter() } + + /** Gets the target (loop variable) of the `for` loop. */ + ControlFlowNode getTarget() { + exists(Py::For f | + f.getIter() = toAst(this) and + f.getTarget() = toAst(result) + ) + } + + /** Holds if `target` is the loop variable and `sequence` is the iterable. */ + predicate iterates(ControlFlowNode target, ControlFlowNode sequence) { + target = this.getTarget() and sequence = this.getSequence() + } +} + +/** A control flow node corresponding to a `raise` statement. */ +class RaiseStmtNode extends ControlFlowNode { + RaiseStmtNode() { toAst(this) instanceof Py::Raise } + + override Py::Raise getNode() { result = super.getNode() } + + /** Gets the exception expression, if any. */ + ControlFlowNode getException() { + exists(Py::Raise r | + r = toAst(this) and + r.getException() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } +} + +/** A control flow node corresponding to a starred expression (`*x`). */ +class StarredNode extends ControlFlowNode { + StarredNode() { toAst(this) instanceof Py::Starred } + + /** Gets the value being starred. */ + ControlFlowNode getValue() { + exists(Py::Starred s | + s = toAst(this) and + s.getValue() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } +} + +/** A control flow node corresponding to an `except` clause's name binding. */ +class ExceptFlowNode extends ControlFlowNode { + ExceptFlowNode() { exists(Py::ExceptStmt e | toAst(this) = e.getName()) } + + /** Gets the CFG node for the bound `as`-name itself. */ + ControlFlowNode getName() { result = this } + + /** Gets the type expression of this exception handler. */ + ControlFlowNode getType() { + exists(Py::ExceptStmt e | + e.getName() = toAst(this) and + e.getType() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } +} + +/** A control flow node corresponding to an `except*` clause's name binding. */ +class ExceptGroupFlowNode extends ControlFlowNode { + ExceptGroupFlowNode() { exists(Py::ExceptGroupStmt e | toAst(this) = e.getName()) } + + /** Gets the CFG node for the bound `as`-name itself. */ + ControlFlowNode getName() { result = this } +} + +/** Abstract base class for sequence nodes (tuple, list). */ +abstract class SequenceNode extends ControlFlowNode { + /** Gets the `n`th element of this sequence. */ + abstract ControlFlowNode getElement(int n); + + /** Gets any element of this sequence. */ + ControlFlowNode getAnElement() { result = this.getElement(_) } +} + +/** A control flow node corresponding to a tuple literal. */ +class TupleNode extends SequenceNode { + TupleNode() { toAst(this) instanceof Py::Tuple } + + override ControlFlowNode getElement(int n) { + exists(Py::Tuple t | + t = toAst(this) and + t.getElt(n) = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } +} + +/** A control flow node corresponding to a list literal. */ +class ListNode extends SequenceNode { + ListNode() { toAst(this) instanceof Py::List } + + override ControlFlowNode getElement(int n) { + exists(Py::List l | + l = toAst(this) and + l.getElt(n) = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } +} + +/** A control flow node corresponding to a set literal. */ +class SetNode extends ControlFlowNode { + SetNode() { toAst(this) instanceof Py::Set } + + /** Gets the flow node for an element of the set. */ + ControlFlowNode getAnElement() { + exists(Py::Set s | + s = toAst(this) and + s.getAnElt() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } +} + +/** A control flow node corresponding to a dict literal. */ +class DictNode extends ControlFlowNode { + DictNode() { toAst(this) instanceof Py::Dict } + + /** Gets the flow node for a key of the dict. */ + ControlFlowNode getAKey() { + exists(Py::Dict d | + d = toAst(this) and + d.getAKey() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } + + /** Gets the flow node for a value of the dict. */ + ControlFlowNode getAValue() { + exists(Py::Dict d | + d = toAst(this) and + d.getAValue() = toAst(result) and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } +} + +/** A control flow node corresponding to an iterable in a `for` loop. */ +class IterableNode extends ControlFlowNode { + IterableNode() { + this instanceof SequenceNode + or + this instanceof SetNode + } + + /** Gets the control flow node for an element of this iterable. */ + ControlFlowNode getAnElement() { + result = this.(SequenceNode).getAnElement() + or + result = this.(SetNode).getAnElement() + } +} diff --git a/python/ql/test/library-tests/ControlFlow/store-load/StoreLoadTest.expected b/python/ql/test/library-tests/ControlFlow/store-load/StoreLoadTest.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/store-load/StoreLoadTest.ql b/python/ql/test/library-tests/ControlFlow/store-load/StoreLoadTest.ql new file mode 100644 index 00000000000..4ab2ef5be8f --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/store-load/StoreLoadTest.ql @@ -0,0 +1,41 @@ +/** + * Inline-expectations test for the store/load/delete/parameter + * classification predicates on the new-CFG facade. + * + * Each tag fires when the corresponding predicate (`isLoad`, + * `isStore`, `isDelete`, `isParameter`, `isAugLoad`, `isAugStore`) + * holds on the canonical CFG node wrapping a `Py::Name` with the + * given identifier. Subscript and attribute stores are not covered + * by these tags — only the `Name`-typed targets/loads they involve. + */ + +import python +import semmle.python.controlflow.internal.Cfg as Cfg +import utils.test.InlineExpectationsTest + +module StoreLoadTest implements TestSig { + string getARelevantTag() { result = ["load", "store", "delete", "param", "augload", "augstore"] } + + predicate hasActualResult(Location location, string element, string tag, string value) { + exists(Cfg::NameNode n | + location = n.getLocation() and + element = n.toString() and + value = n.getId() and + ( + n.isLoad() and not n.isAugLoad() and tag = "load" + or + n.isStore() and not n.isAugStore() and tag = "store" + or + n.isDelete() and tag = "delete" + or + n.isParameter() and tag = "param" + or + n.isAugLoad() and tag = "augload" + or + n.isAugStore() and tag = "augstore" + ) + ) + } +} + +import MakeTest diff --git a/python/ql/test/library-tests/ControlFlow/store-load/test.py b/python/ql/test/library-tests/ControlFlow/store-load/test.py new file mode 100644 index 00000000000..dfca45a0b47 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/store-load/test.py @@ -0,0 +1,56 @@ +# Store/load/delete/parameter classification on the new-CFG facade. +# +# Each annotated location carries the (sorted, deduplicated) set of +# kinds the CFG facade reports there. Comparing against the legacy +# 'semmle.python.Flow' classification is done by the comparison query +# 'StoreLoadParity.ql' — annotations here are only the positive +# assertions for the new facade. +# +# Tags: +# load= -- isLoad() fires on the Name +# store= -- isStore() fires +# delete= -- isDelete() fires +# param= -- isParameter() fires +# augload= -- isAugLoad() fires (the LHS of x += ... when read) +# augstore= -- isAugStore() fires (the LHS of x += ... when written) + + +# --- plain load / store / delete --- + +x = 1 # $ store=x +y = x + 1 # $ store=y load=x +print(y) # $ load=print load=y +del x # $ delete=x + + +# --- function definitions (parameters) --- + +def f(a, b=2, *args, c, **kwargs): # $ store=f param=a param=b param=args param=c param=kwargs + return a + b + c # $ load=a load=b load=c + + +# --- augmented assignment splits one Name into load + store halves --- + +def aug(): # $ store=aug + n = 0 # $ store=n + n += 1 # $ augload=n augstore=n + return n # $ load=n + + +# --- subscript / attribute stores --- + +class C: # $ store=C + pass + + +def stores(obj, container, idx): # $ store=stores param=obj param=container param=idx + obj.attr = 1 # $ load=obj + container[idx] = 2 # $ load=container load=idx + return obj # $ load=obj + + +# --- tuple unpacking --- + +def unpack(pair): # $ store=unpack param=pair + a, b = pair # $ store=a store=b load=pair + return a + b # $ load=a load=b From 41c9d8b80a9b930ab81a712db9cd04d2b7a4b000 Mon Sep 17 00:00:00 2001 From: yoff Date: Wed, 3 Jun 2026 09:46:03 +0000 Subject: [PATCH 04/90] Python: model exception edges for raise-prone expressions inside try/with MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new CFG previously only emitted exception edges for explicit `raise` and `assert` statements. As a result, code that became reachable only via the exception path of an arbitrary expression (e.g., the body of an `except` handler following a try-body whose `call()` could raise) was classified as dead, breaking analyses like StackTraceExposure, FileNotAlwaysClosed, ExceptionInfo, UseOfExit, and CatchingBaseException. This commit adds a `mayThrow` predicate over expressions that are known sources of implicit exceptions in Python (calls, attribute access, subscripts, arithmetic/comparison operators, imports, await/yield/yield from) plus `from m import *` at the statement level, and routes them through the shared CFG's `beginAbruptCompletion(_, _, ExceptionSuccessor, always=false)` hook. The set of exception sources is restricted to nodes that are syntactically inside a `try`/`with` statement in the same scope. This mirrors Java's `ControlFlowGraph::mayThrow`, which only emits exception edges where local handling can observe them — outside such contexts, the edges add CFG complexity (weakening BarrierGuard precision and breaking SSA continuity around augmented assignments and subscript stores) without analysis benefit, since exceptions just propagate to the function exit anyway. Net effect on the test suite: ~100 alerts restored across the exception- related query tests (StackTraceExposure +29, ExceptionInfo +17, FileNotAlwaysClosed +52, UseOfExit +1, CatchingBaseException restored) with no precision regressions. Affected `.expected` files and the regression-guard `dead_under_no_raise.py` are updated accordingly. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../controlflow/internal/AstNodeImpl.qll | 88 +++++++++++++++++++ .../bindings/dead_under_no_raise.py | 37 ++++---- 2 files changed, 107 insertions(+), 18 deletions(-) diff --git a/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll b/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll index 5d87b16f351..803d3ca6e27 100644 --- a/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll +++ b/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll @@ -1571,6 +1571,89 @@ private module Input implements InputSig1, InputSig2 { private string assertThrowTag() { result = "[assert-throw]" } + /** + * Holds if the AST node `n` may raise an exception at runtime as part of + * its normal evaluation (not via an explicit `raise`/`assert`, which are + * modelled separately). + * + * The set mirrors what the legacy CFG used to flag implicitly: function + * calls (anything can raise), attribute access (`AttributeError`), + * subscript access (`IndexError`/`KeyError`/`TypeError`), arithmetic and + * comparison operators (`TypeError`/`ZeroDivisionError`), imports + * (`ImportError`/`ModuleNotFoundError`), and generator/coroutine + * suspension points (`await`/`yield`/`yield from`). + * + * Bare `Name` reads are intentionally excluded — modelling every name + * read as `mayThrow` would explode CFG edge count for negligible + * analysis value. `BoolExpr`/`IfExp` containers are also excluded; the + * operands they evaluate contribute their own exception edges. + */ + private predicate exprMayThrow(Py::Expr e) { + e instanceof Py::Call + or + e instanceof Py::Attribute + or + e instanceof Py::Subscript + or + e instanceof Py::BinaryExpr + or + e instanceof Py::UnaryExpr + or + e instanceof Py::Compare + or + e instanceof Py::ImportExpr + or + e instanceof Py::ImportMember + or + e instanceof Py::Await + or + e instanceof Py::Yield + or + e instanceof Py::YieldFrom + } + + /** + * Holds if the statement `s` may raise an exception at runtime as part + * of its normal evaluation. Currently restricted to `from m import *` + * (which performs the import as a statement-level side effect). + */ + private predicate stmtMayThrow(Py::Stmt s) { s instanceof Py::ImportStar } + + /** + * Holds if `n` is syntactically inside the body, handlers, `else`, or + * `finally` of a `try` statement (or the body of a `with` statement, + * which compiles to an implicit try/finally for `__exit__`) in the + * same scope. + * + * This mirrors Java's `ControlFlowGraph::mayThrow`, which only emits + * exception edges when there is local exception handling that would + * observe them. Outside such contexts, exception edges would add CFG + * complexity (weakening BarrierGuard precision and breaking SSA + * continuity around augmented assignments and subscript stores) + * without any analysis benefit, since exceptions just propagate to + * the function exit anyway. + */ + private predicate inExceptionContext(Py::AstNode py) { + exists(Py::Try t | t.containsInScope(py)) + or + exists(Py::With w | w.containsInScope(py)) + } + + /** + * Holds if `n` may raise an exception during normal evaluation. See + * `exprMayThrow` and `stmtMayThrow` for the included AST classes. + * + * Restricted to nodes inside a `try`/`with` statement: matches Java's + * approach of only modelling exception flow where it can be observed + * by local handling. + */ + private predicate mayThrow(Ast::AstNode n) { + exists(Py::AstNode py | py = n.asExpr() or py = n.asStmt() | + (exprMayThrow(py) or stmtMayThrow(py)) and + inExceptionContext(py) + ) + } + predicate additionalNode(Ast::AstNode n, string tag, NormalSuccessor t) { n instanceof Ast::AssertStmt and tag = assertThrowTag() and t instanceof DirectSuccessor } @@ -1582,6 +1665,11 @@ private module Input implements InputSig1, InputSig2 { n.isAdditional(ast, assertThrowTag()) and c.asSimpleAbruptCompletion() instanceof ExceptionSuccessor and always = true + or + mayThrow(ast) and + n.isIn(ast) and + c.asSimpleAbruptCompletion() instanceof ExceptionSuccessor and + always = false } predicate endAbruptCompletion(Ast::AstNode ast, PreControlFlowNode n, AbruptCompletion c) { diff --git a/python/ql/test/library-tests/ControlFlow/bindings/dead_under_no_raise.py b/python/ql/test/library-tests/ControlFlow/bindings/dead_under_no_raise.py index dbfb857b536..9058f2b7116 100644 --- a/python/ql/test/library-tests/ControlFlow/bindings/dead_under_no_raise.py +++ b/python/ql/test/library-tests/ControlFlow/bindings/dead_under_no_raise.py @@ -1,15 +1,15 @@ -# Dead bindings under the "no expressions raise" CFG abstraction. +# Reachability of code following a try whose body always returns. # -# The new CFG does not currently model raise edges from arbitrary -# expressions. As a consequence, code that is only reachable through -# exception flow is (correctly) classified as dead and has no CFG node. -# Variable bindings in dead code do not need CFG nodes - SSA / dataflow -# over dead code is moot. +# The new CFG models exception edges for raise-prone expressions when +# they appear inside a `try` (or `with`) statement, mirroring Java's +# `mayThrow`. This means the body of a `try` has both a normal +# completion edge and an exception edge to its handlers, so code +# following the try-statement is reachable via the except-handler path +# even when the try-body would otherwise always return. # -# These tests act as a regression guard: the bindings below intentionally -# have no `cfgdefines=` annotations. If raise modelling is later added, -# the BindingsTest infrastructure will surface the new CFG nodes as -# unexpected results, and this file will need to be revisited. +# Code that is not reachable under either normal or exception flow +# (for example, the `else` clause of a try whose body unconditionally +# raises) remains correctly classified as dead. def f(obj): # $ cfgdefines=f cfgdefines=obj @@ -18,12 +18,12 @@ def f(obj): # $ cfgdefines=f cfgdefines=obj except TypeError: pass - # The first try's body always returns; its except handler does not - # raise or otherwise transfer control, so under "no expressions - # raise" the only paths out of the try-statement are dead. Everything - # below is unreachable. + # The try-body always returns, but `len(obj)` can raise (it is + # inside the try, so we model its exception edge). The + # `except TypeError: pass` handler falls through to here, making + # the code below reachable. try: - hint = type(obj).__length_hint__ + hint = type(obj).__length_hint__ # $ cfgdefines=hint except AttributeError: return None return hint @@ -35,7 +35,8 @@ def g(): # $ cfgdefines=g except: raise Exception("outer") else: - # Unreachable: the inner try body always raises, so the `else:` + # Unreachable: the inner try body always raises (via an explicit + # `raise`, which is modelled unconditionally), so the `else:` # clause never runs. hit_inner_else = True @@ -46,7 +47,7 @@ def h(cache, key): # $ cfgdefines=h cfgdefines=cache cfgdefines=key except KeyError: pass - # Same pattern as `f`: dead under "no expressions raise". - value = compute(key) + # Same pattern as `f`: reachable via the except-handler fall-through. + value = compute(key) # $ cfgdefines=value cache[key] = value return value From 47d2b05bc53191129311e00502debfc521b30508 Mon Sep 17 00:00:00 2001 From: yoff Date: Mon, 29 Jun 2026 13:17:31 +0000 Subject: [PATCH 05/90] Python: visit function parameter and return annotations in new CFG The new (shared-CFG-based) Python control flow graph in `semmle.python.controlflow.internal.Cfg` previously did not emit CFG nodes for parameter type annotations (`def f(x: T): ...`) or for the return type annotation (`-> T`). The legacy CFG emitted both, and a small number of framework models rely on this: `LocalSources.qll`'s `annotatedInstance` walks the parameter annotation expression by way of its CFG node to track that a parameter receives an instance of the annotated class. After the dataflow flip to the new CFG/SSA this regression manifested as lost flows in any test exercising annotation-based parameter tracking: FastAPI `Depends()` receivers, Pydantic request bodies, Starlette `WebSocket`, the call-graph type-annotation test, and so on. Extend `FunctionDefExpr` to visit each annotation as a child of the function-def expression, in CPython evaluation order: positional parameter annotations, `*args` annotation, keyword-only parameter annotations, `**kwargs` annotation, then the return annotation. (Lambda expressions have no annotations in Python syntax, so `LambdaExpr` is unchanged.) PEP 695 type parameters remain out of scope; they belong to the inner annotation scope, not the enclosing CFG. Restored test results across `framework/aiohttp`, `framework/fastapi`, `framework/lxml`, the `CallGraph-type-annotations` test, and `CWE-022-PathInjection`. Two FastAPI list-comprehension MISSING markers become positive (`taint_test.py:41,55`). CPython CFG consistency remains clean. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../2026-06-04-cfg-parameter-annotations.md | 4 ++ .../controlflow/internal/AstNodeImpl.qll | 63 +++++++++++++++++-- 2 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 python/ql/lib/change-notes/2026-06-04-cfg-parameter-annotations.md diff --git a/python/ql/lib/change-notes/2026-06-04-cfg-parameter-annotations.md b/python/ql/lib/change-notes/2026-06-04-cfg-parameter-annotations.md new file mode 100644 index 00000000000..96ba81e1610 --- /dev/null +++ b/python/ql/lib/change-notes/2026-06-04-cfg-parameter-annotations.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* The new (shared-CFG-based) Python control flow graph now visits parameter and return type annotations as CFG nodes for function definitions, matching the legacy CFG. This restores annotation-based type tracking through framework models such as FastAPI's `Depends()`, Pydantic request models, Starlette `WebSocket` handlers, and any other models that flow a class reference through `Parameter.getAnnotation()` to identify instances of the annotated class. diff --git a/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll b/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll index 803d3ca6e27..8199008e88c 100644 --- a/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll +++ b/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll @@ -1474,10 +1474,19 @@ module Ast implements AstSig { /** * A function definition expression (visits positional and keyword - * defaults, but NOT PEP 695 type parameters — those bind in an - * annotation scope that nests the function body, so they belong to - * the inner scope's CFG, not the enclosing scope's; the legacy CFG - * also omitted them). + * defaults followed by parameter and return type annotations, but NOT + * PEP 695 type parameters — those bind in an annotation scope that + * nests the function body, so they belong to the inner scope's CFG, + * not the enclosing scope's; the legacy CFG also omitted them). + * + * Evaluation order follows CPython: defaults are pushed first, then + * keyword-only defaults, then annotations (the `__annotations__` dict + * is built last, before `MAKE_FUNCTION`). Annotations are emitted as + * CFG nodes so that flows from a class reference into a parameter's + * type annotation are visible to dataflow (e.g. so that framework + * models like FastAPI's `Depends()` can use a parameter's type hint + * to track that the parameter receives an instance of the annotated + * class — see `LocalSources::annotatedInstance`). */ additional class FunctionDefExpr extends Expr { private Py::FunctionExpr funcExpr; @@ -1501,15 +1510,61 @@ module Ast implements AstSig { rank[n + 1](Py::Expr d, int i | d = funcExpr.getArgs().getKwDefault(i) | d order by i) } + /** + * Gets the `n`th annotation expression, in CPython evaluation + * order: positional parameter annotations (by argument position), + * `*args` annotation, keyword-only parameter annotations (by + * argument position), `**kwargs` annotation, then the return + * annotation. Each annotation appears at most once. + */ + Expr getAnnotation(int n) { + result.asExpr() = + rank[n + 1](Py::Expr a, int subOrder, int subIndex | + functionAnnotation(funcExpr, a, subOrder, subIndex) + | + a order by subOrder, subIndex + ) + } + int getNumberOfDefaults() { result = count(funcExpr.getArgs().getADefault()) } + int getNumberOfKwDefaults() { result = count(funcExpr.getArgs().getAKwDefault()) } + + int getNumberOfAnnotations() { + result = count(Py::Expr a | functionAnnotation(funcExpr, a, _, _)) + } + override AstNode getChild(int index) { result = this.getDefault(index) or result = this.getKwDefault(index - this.getNumberOfDefaults()) + or + result = this.getAnnotation(index - this.getNumberOfDefaults() - this.getNumberOfKwDefaults()) } } + /** + * Holds if `a` is an annotation of `funcExpr` in slot + * `(subOrder, subIndex)`. Slots are CPython evaluation order: + * positional param annotations (subOrder 0, subIndex = argument + * position), `*args` annotation (1, 0), keyword-only annotations + * (2, position), `**kwargs` annotation (3, 0), return annotation + * (4, 0). + */ + private predicate functionAnnotation( + Py::FunctionExpr funcExpr, Py::Expr a, int subOrder, int subIndex + ) { + a = funcExpr.getArgs().getAnnotation(subIndex) and subOrder = 0 + or + a = funcExpr.getArgs().getVarargannotation() and subOrder = 1 and subIndex = 0 + or + a = funcExpr.getArgs().getKwAnnotation(subIndex) and subOrder = 2 + or + a = funcExpr.getArgs().getKwargannotation() and subOrder = 3 and subIndex = 0 + or + a = funcExpr.getReturns() and subOrder = 4 and subIndex = 0 + } + /** A lambda expression (has default args evaluated at definition time). */ additional class LambdaExpr extends Expr { private Py::Lambda lambda; From 5c2e5bfe8eee8a8e573b5fef1d8d6cbf46c9c4d9 Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Sun, 5 Jul 2026 17:20:52 +0100 Subject: [PATCH 06/90] Convert qlref tests to inline expectations --- .../CWE-094-dataURL/CodeInjection.qlref | 3 +- .../Security/CWE-094-dataURL/test.js | 12 +++--- .../EnvValueAndKeyInjection.qlref | 3 +- .../CWE-099/EnvValueAndKeyInjection/test.js | 6 +-- .../EnvValueInjection/EnvValueInjection.qlref | 3 +- .../CWE-099/EnvValueInjection/test.js | 8 ++-- .../CWE-347/localsource/JsonWebToken.js | 10 ++--- ...odeJwtWithoutVerificationLocalSource.qlref | 3 +- .../Security/CWE-347/localsource/jose.js | 4 +- .../Security/CWE-347/localsource/jwtDecode.js | 4 +- .../Security/CWE-347/localsource/jwtSimple.js | 4 +- .../CWE-347/remotesource/JsonWebToken.js | 10 ++--- .../decodeJwtWithoutVerification.qlref | 3 +- .../Security/CWE-347/remotesource/jose.js | 4 +- .../CWE-347/remotesource/jwtDecode.js | 4 +- .../CWE-347/remotesource/jwtSimple.js | 4 +- .../experimental/Security/CWE-918/SSRF.qlref | 3 +- .../SsrfIpv6TransitionIncompleteGuard.qlref | 3 +- .../bad-private-ip-pkg.js | 2 +- .../bad-rfc1918-regex.js | 2 +- .../Security/CWE-918/check-domain.js | 10 ++--- .../Security/CWE-918/check-middleware.js | 2 +- .../Security/CWE-918/check-path.js | 10 ++--- .../Security/CWE-918/check-regex.js | 12 +++--- .../Security/CWE-918/check-validator.js | 14 +++---- .../MultipleArgumentsToSetConstructor.qlref | 3 +- .../MultipleArgumentsToSetConstructorBad.js | 2 +- .../MultipleArgumentsToSetConstructor/tst.js | 2 +- .../UnpromotedRouteHandlerCandidate.qlref | 3 +- .../UnpromotedRouteSetupCandidate.qlref | 3 +- .../frameworks/HTTP-heuristics/src/hapi.js | 2 +- .../frameworks/HTTP-heuristics/src/nodejs.js | 6 +-- .../HTTP-heuristics/src/route-objects.js | 14 +++---- .../frameworks/HTTP-heuristics/src/tst.js | 42 +++++++++---------- .../frameworks/Templating/CodeInjection.qlref | 3 +- .../frameworks/Templating/app.js | 26 ++++++------ .../Templating/views/angularjs_include.ejs | 4 +- .../Templating/views/angularjs_sinks.ejs | 4 +- .../frameworks/Templating/views/ejs_sinks.ejs | 6 +-- .../frameworks/Templating/views/hbs_sinks.hbs | 6 +-- .../frameworks/Templating/views/njk_sinks.njk | 10 ++--- 41 files changed, 145 insertions(+), 134 deletions(-) diff --git a/javascript/ql/test/experimental/Security/CWE-094-dataURL/CodeInjection.qlref b/javascript/ql/test/experimental/Security/CWE-094-dataURL/CodeInjection.qlref index 3caf7ab7b43..9ed18359d20 100644 --- a/javascript/ql/test/experimental/Security/CWE-094-dataURL/CodeInjection.qlref +++ b/javascript/ql/test/experimental/Security/CWE-094-dataURL/CodeInjection.qlref @@ -1 +1,2 @@ -experimental/Security/CWE-094-dataURL/CodeInjection.ql \ No newline at end of file +query: experimental/Security/CWE-094-dataURL/CodeInjection.ql +postprocess: utils/test/InlineExpectationsTestQuery.ql diff --git a/javascript/ql/test/experimental/Security/CWE-094-dataURL/test.js b/javascript/ql/test/experimental/Security/CWE-094-dataURL/test.js index a5a2e76fa3c..b8599b5687c 100644 --- a/javascript/ql/test/experimental/Security/CWE-094-dataURL/test.js +++ b/javascript/ql/test/experimental/Security/CWE-094-dataURL/test.js @@ -2,21 +2,21 @@ const { Worker } = require('node:worker_threads'); var app = require('express')(); app.post('/path', async function (req, res) { - const payload = req.query.queryParameter // like: payload = 'data:text/javascript,console.log("hello!");//' + const payload = req.query.queryParameter // $ Source // like: payload = 'data:text/javascript,console.log("hello!");//' let payloadURL = new URL(payload + sth) // NOT OK - new Worker(payloadURL); + new Worker(payloadURL); // $ Alert payloadURL = new URL(payload + sth) // NOT OK - new Worker(payloadURL); + new Worker(payloadURL); // $ Alert payloadURL = new URL(sth + payload) // OK new Worker(payloadURL); }); app.post('/path2', async function (req, res) { - const payload = req.query.queryParameter // like: payload = 'data:text/javascript,console.log("hello!");//' - await import(payload) // NOT OK - await import(payload + sth) // NOT OK + const payload = req.query.queryParameter // $ Source // like: payload = 'data:text/javascript,console.log("hello!");//' + await import(payload) // $ Alert // NOT OK + await import(payload + sth) // $ Alert // NOT OK await import(sth + payload) // OK }); diff --git a/javascript/ql/test/experimental/Security/CWE-099/EnvValueAndKeyInjection/EnvValueAndKeyInjection.qlref b/javascript/ql/test/experimental/Security/CWE-099/EnvValueAndKeyInjection/EnvValueAndKeyInjection.qlref index fde9a286e5a..dbd1332e35a 100644 --- a/javascript/ql/test/experimental/Security/CWE-099/EnvValueAndKeyInjection/EnvValueAndKeyInjection.qlref +++ b/javascript/ql/test/experimental/Security/CWE-099/EnvValueAndKeyInjection/EnvValueAndKeyInjection.qlref @@ -1 +1,2 @@ -experimental/Security/CWE-099/EnvValueAndKeyInjection.ql \ No newline at end of file +query: experimental/Security/CWE-099/EnvValueAndKeyInjection.ql +postprocess: utils/test/InlineExpectationsTestQuery.ql diff --git a/javascript/ql/test/experimental/Security/CWE-099/EnvValueAndKeyInjection/test.js b/javascript/ql/test/experimental/Security/CWE-099/EnvValueAndKeyInjection/test.js index a12377c9cec..87c151853e7 100644 --- a/javascript/ql/test/experimental/Security/CWE-099/EnvValueAndKeyInjection/test.js +++ b/javascript/ql/test/experimental/Security/CWE-099/EnvValueAndKeyInjection/test.js @@ -2,9 +2,9 @@ const http = require('node:http'); http.createServer((req, res) => { - const { EnvValue, EnvKey } = req.body; - process.env[EnvKey] = EnvValue; // NOT OK - process.env[EnvKey] = EnvValue; // NOT OK + const { EnvValue, EnvKey } = req.body; // $ Source + process.env[EnvKey] = EnvValue; // $ Alert // NOT OK + process.env[EnvKey] = EnvValue; // $ Alert // NOT OK res.end('env has been injected!'); }); diff --git a/javascript/ql/test/experimental/Security/CWE-099/EnvValueInjection/EnvValueInjection.qlref b/javascript/ql/test/experimental/Security/CWE-099/EnvValueInjection/EnvValueInjection.qlref index e03328beda4..9fc1b79b810 100644 --- a/javascript/ql/test/experimental/Security/CWE-099/EnvValueInjection/EnvValueInjection.qlref +++ b/javascript/ql/test/experimental/Security/CWE-099/EnvValueInjection/EnvValueInjection.qlref @@ -1 +1,2 @@ -experimental/Security/CWE-099/EnvValueInjection.ql \ No newline at end of file +query: experimental/Security/CWE-099/EnvValueInjection.ql +postprocess: utils/test/InlineExpectationsTestQuery.ql diff --git a/javascript/ql/test/experimental/Security/CWE-099/EnvValueInjection/test.js b/javascript/ql/test/experimental/Security/CWE-099/EnvValueInjection/test.js index cb28f01b88b..e95ef9cad77 100644 --- a/javascript/ql/test/experimental/Security/CWE-099/EnvValueInjection/test.js +++ b/javascript/ql/test/experimental/Security/CWE-099/EnvValueInjection/test.js @@ -1,10 +1,10 @@ const http = require('node:http'); http.createServer((req, res) => { - const { EnvValue } = req.body; - process.env["A_Critical_Env"] = EnvValue; // NOT OK - process.env[AKey] = EnvValue; // NOT OK - process.env.AKey = EnvValue; // NOT OK + const { EnvValue } = req.body; // $ Source + process.env["A_Critical_Env"] = EnvValue; // $ Alert // NOT OK + process.env[AKey] = EnvValue; // $ Alert // NOT OK + process.env.AKey = EnvValue; // $ Alert // NOT OK res.end('env has been injected!'); }); diff --git a/javascript/ql/test/experimental/Security/CWE-347/localsource/JsonWebToken.js b/javascript/ql/test/experimental/Security/CWE-347/localsource/JsonWebToken.js index 022b0bda11f..21c5b00e4fe 100644 --- a/javascript/ql/test/experimental/Security/CWE-347/localsource/JsonWebToken.js +++ b/javascript/ql/test/experimental/Security/CWE-347/localsource/JsonWebToken.js @@ -10,18 +10,18 @@ function aJWT() { } (function () { - const UserToken = aJwt() + const UserToken = aJwt() // $ Alert // BAD: no signature verification - jwtJsonwebtoken.decode(UserToken) // NOT OK + jwtJsonwebtoken.decode(UserToken) // $ Sink // NOT OK })(); (function () { - const UserToken = aJwt() + const UserToken = aJwt() // $ Alert // BAD: no signature verification - jwtJsonwebtoken.decode(UserToken) // NOT OK - jwtJsonwebtoken.verify(UserToken, getSecret(), { algorithms: ["HS256", "none"] }) // NOT OK + jwtJsonwebtoken.decode(UserToken) // $ Sink // NOT OK + jwtJsonwebtoken.verify(UserToken, getSecret(), { algorithms: ["HS256", "none"] }) // $ Sink // NOT OK })(); (function () { diff --git a/javascript/ql/test/experimental/Security/CWE-347/localsource/decodeJwtWithoutVerificationLocalSource.qlref b/javascript/ql/test/experimental/Security/CWE-347/localsource/decodeJwtWithoutVerificationLocalSource.qlref index ee8effa049c..36743e92759 100644 --- a/javascript/ql/test/experimental/Security/CWE-347/localsource/decodeJwtWithoutVerificationLocalSource.qlref +++ b/javascript/ql/test/experimental/Security/CWE-347/localsource/decodeJwtWithoutVerificationLocalSource.qlref @@ -1 +1,2 @@ -experimental/Security/CWE-347/decodeJwtWithoutVerificationLocalSource.ql \ No newline at end of file +query: experimental/Security/CWE-347/decodeJwtWithoutVerificationLocalSource.ql +postprocess: utils/test/InlineExpectationsTestQuery.ql diff --git a/javascript/ql/test/experimental/Security/CWE-347/localsource/jose.js b/javascript/ql/test/experimental/Security/CWE-347/localsource/jose.js index 625618e194d..c1e8597147d 100644 --- a/javascript/ql/test/experimental/Security/CWE-347/localsource/jose.js +++ b/javascript/ql/test/experimental/Security/CWE-347/localsource/jose.js @@ -9,10 +9,10 @@ function aJWT() { } (function () { - const UserToken = aJwt() + const UserToken = aJwt() // $ Alert // no signature verification - jose.decodeJwt(UserToken) // NOT OK + jose.decodeJwt(UserToken) // $ Sink // NOT OK })(); (async function () { diff --git a/javascript/ql/test/experimental/Security/CWE-347/localsource/jwtDecode.js b/javascript/ql/test/experimental/Security/CWE-347/localsource/jwtDecode.js index f3d4a40314c..21eb58cb40d 100644 --- a/javascript/ql/test/experimental/Security/CWE-347/localsource/jwtDecode.js +++ b/javascript/ql/test/experimental/Security/CWE-347/localsource/jwtDecode.js @@ -10,9 +10,9 @@ function aJWT() { } (function () { - const UserToken = aJwt() + const UserToken = aJwt() // $ Alert // jwt-decode // no signature verification - jwt_decode(UserToken) // NOT OK + jwt_decode(UserToken) // $ Sink // NOT OK })(); \ No newline at end of file diff --git a/javascript/ql/test/experimental/Security/CWE-347/localsource/jwtSimple.js b/javascript/ql/test/experimental/Security/CWE-347/localsource/jwtSimple.js index 73b79d86d75..320d382c76c 100644 --- a/javascript/ql/test/experimental/Security/CWE-347/localsource/jwtSimple.js +++ b/javascript/ql/test/experimental/Security/CWE-347/localsource/jwtSimple.js @@ -10,10 +10,10 @@ function aJWT() { } (function () { - const UserToken = aJwt() + const UserToken = aJwt() // $ Alert // BAD: no signature verification - jwt_simple.decode(UserToken, getSecret(), true); // NOT OK + jwt_simple.decode(UserToken, getSecret(), true); // $ Sink // NOT OK })(); (function () { diff --git a/javascript/ql/test/experimental/Security/CWE-347/remotesource/JsonWebToken.js b/javascript/ql/test/experimental/Security/CWE-347/remotesource/JsonWebToken.js index 0e39e95b632..d1199a226cf 100644 --- a/javascript/ql/test/experimental/Security/CWE-347/remotesource/JsonWebToken.js +++ b/javascript/ql/test/experimental/Security/CWE-347/remotesource/JsonWebToken.js @@ -7,18 +7,18 @@ function getSecret() { return "A Safe generated random key" } app.get('/jwtJsonwebtoken1', (req, res) => { - const UserToken = req.headers.authorization; + const UserToken = req.headers.authorization; // $ Alert // BAD: no signature verification - jwtJsonwebtoken.decode(UserToken) // NOT OK + jwtJsonwebtoken.decode(UserToken) // $ Sink // NOT OK }) app.get('/jwtJsonwebtoken2', (req, res) => { - const UserToken = req.headers.authorization; + const UserToken = req.headers.authorization; // $ Alert // BAD: no signature verification - jwtJsonwebtoken.decode(UserToken) // NOT OK - jwtJsonwebtoken.verify(UserToken, getSecret(), { algorithms: ["HS256", "none"] }) // NOT OK + jwtJsonwebtoken.decode(UserToken) // $ Sink // NOT OK + jwtJsonwebtoken.verify(UserToken, getSecret(), { algorithms: ["HS256", "none"] }) // $ Sink // NOT OK }) app.get('/jwtJsonwebtoken3', (req, res) => { diff --git a/javascript/ql/test/experimental/Security/CWE-347/remotesource/decodeJwtWithoutVerification.qlref b/javascript/ql/test/experimental/Security/CWE-347/remotesource/decodeJwtWithoutVerification.qlref index 9e7ea468ee7..d37a36dbcca 100644 --- a/javascript/ql/test/experimental/Security/CWE-347/remotesource/decodeJwtWithoutVerification.qlref +++ b/javascript/ql/test/experimental/Security/CWE-347/remotesource/decodeJwtWithoutVerification.qlref @@ -1 +1,2 @@ -experimental/Security/CWE-347/decodeJwtWithoutVerification.ql \ No newline at end of file +query: experimental/Security/CWE-347/decodeJwtWithoutVerification.ql +postprocess: utils/test/InlineExpectationsTestQuery.ql diff --git a/javascript/ql/test/experimental/Security/CWE-347/remotesource/jose.js b/javascript/ql/test/experimental/Security/CWE-347/remotesource/jose.js index 30c57650e33..28b7ca95f7e 100644 --- a/javascript/ql/test/experimental/Security/CWE-347/remotesource/jose.js +++ b/javascript/ql/test/experimental/Security/CWE-347/remotesource/jose.js @@ -8,9 +8,9 @@ function getSecret() { } app.get('/jose1', (req, res) => { - const UserToken = req.headers.authorization; + const UserToken = req.headers.authorization; // $ Alert // no signature verification - jose.decodeJwt(UserToken) // NOT OK + jose.decodeJwt(UserToken) // $ Sink // NOT OK }) diff --git a/javascript/ql/test/experimental/Security/CWE-347/remotesource/jwtDecode.js b/javascript/ql/test/experimental/Security/CWE-347/remotesource/jwtDecode.js index 76a26e0df36..6bdc92ba626 100644 --- a/javascript/ql/test/experimental/Security/CWE-347/remotesource/jwtDecode.js +++ b/javascript/ql/test/experimental/Security/CWE-347/remotesource/jwtDecode.js @@ -8,11 +8,11 @@ function getSecret() { } app.get('/jwtDecode', (req, res) => { - const UserToken = req.headers.authorization; + const UserToken = req.headers.authorization; // $ Alert // jwt-decode // no signature verification - jwt_decode(UserToken) // NOT OK + jwt_decode(UserToken) // $ Sink // NOT OK }) app.listen(port, () => { diff --git a/javascript/ql/test/experimental/Security/CWE-347/remotesource/jwtSimple.js b/javascript/ql/test/experimental/Security/CWE-347/remotesource/jwtSimple.js index 4803309e3e5..4652ffcc3c6 100644 --- a/javascript/ql/test/experimental/Security/CWE-347/remotesource/jwtSimple.js +++ b/javascript/ql/test/experimental/Security/CWE-347/remotesource/jwtSimple.js @@ -7,10 +7,10 @@ function getSecret() { return "A Safe generated random key" } app.get('/jwtSimple1', (req, res) => { - const UserToken = req.headers.authorization; + const UserToken = req.headers.authorization; // $ Alert // no signature verification - jwt_simple.decode(UserToken, getSecret(), true); // NOT OK + jwt_simple.decode(UserToken, getSecret(), true); // $ Sink // NOT OK }) app.get('/jwtSimple2', (req, res) => { diff --git a/javascript/ql/test/experimental/Security/CWE-918/SSRF.qlref b/javascript/ql/test/experimental/Security/CWE-918/SSRF.qlref index 05a9c8145e6..7819b4827f2 100644 --- a/javascript/ql/test/experimental/Security/CWE-918/SSRF.qlref +++ b/javascript/ql/test/experimental/Security/CWE-918/SSRF.qlref @@ -1 +1,2 @@ -./experimental/Security/CWE-918/SSRF.ql \ No newline at end of file +query: ./experimental/Security/CWE-918/SSRF.ql +postprocess: utils/test/InlineExpectationsTestQuery.ql diff --git a/javascript/ql/test/experimental/Security/CWE-918/SsrfIpv6TransitionIncompleteGuard/SsrfIpv6TransitionIncompleteGuard.qlref b/javascript/ql/test/experimental/Security/CWE-918/SsrfIpv6TransitionIncompleteGuard/SsrfIpv6TransitionIncompleteGuard.qlref index 50159ab72fe..c26b3d73324 100644 --- a/javascript/ql/test/experimental/Security/CWE-918/SsrfIpv6TransitionIncompleteGuard/SsrfIpv6TransitionIncompleteGuard.qlref +++ b/javascript/ql/test/experimental/Security/CWE-918/SsrfIpv6TransitionIncompleteGuard/SsrfIpv6TransitionIncompleteGuard.qlref @@ -1 +1,2 @@ -experimental/Security/CWE-918/SsrfIpv6TransitionIncompleteGuard.ql \ No newline at end of file +query: experimental/Security/CWE-918/SsrfIpv6TransitionIncompleteGuard.ql +postprocess: utils/test/InlineExpectationsTestQuery.ql diff --git a/javascript/ql/test/experimental/Security/CWE-918/SsrfIpv6TransitionIncompleteGuard/bad-private-ip-pkg.js b/javascript/ql/test/experimental/Security/CWE-918/SsrfIpv6TransitionIncompleteGuard/bad-private-ip-pkg.js index 972d7aad9b7..53e3d3660c2 100644 --- a/javascript/ql/test/experimental/Security/CWE-918/SsrfIpv6TransitionIncompleteGuard/bad-private-ip-pkg.js +++ b/javascript/ql/test/experimental/Security/CWE-918/SsrfIpv6TransitionIncompleteGuard/bad-private-ip-pkg.js @@ -8,6 +8,6 @@ async function validateUrlHost(host) { // NOT OK throw new Error('blocked private host'); } return fetch('http://' + host + '/'); -} +} // $ Alert[javascript/ssrf-ipv6-transition-incomplete-guard] module.exports = { validateUrlHost }; diff --git a/javascript/ql/test/experimental/Security/CWE-918/SsrfIpv6TransitionIncompleteGuard/bad-rfc1918-regex.js b/javascript/ql/test/experimental/Security/CWE-918/SsrfIpv6TransitionIncompleteGuard/bad-rfc1918-regex.js index be70a4a5e5d..4c067666242 100644 --- a/javascript/ql/test/experimental/Security/CWE-918/SsrfIpv6TransitionIncompleteGuard/bad-rfc1918-regex.js +++ b/javascript/ql/test/experimental/Security/CWE-918/SsrfIpv6TransitionIncompleteGuard/bad-rfc1918-regex.js @@ -13,6 +13,6 @@ function checkTargetHost(host) { // NOT OK throw new Error('blocked internal host'); } return http.get('http://' + host + '/'); -} +} // $ Alert[javascript/ssrf-ipv6-transition-incomplete-guard] module.exports = { checkTargetHost }; diff --git a/javascript/ql/test/experimental/Security/CWE-918/check-domain.js b/javascript/ql/test/experimental/Security/CWE-918/check-domain.js index 0821140ab5f..e46f4b0bb9e 100644 --- a/javascript/ql/test/experimental/Security/CWE-918/check-domain.js +++ b/javascript/ql/test/experimental/Security/CWE-918/check-domain.js @@ -13,8 +13,8 @@ const app = express(); app.get('/check-with-axios', req => { // without validation - const url = req.query.url; - axios.get(url); //SSRF + const url = req.query.url; // $ Source + axios.get(url); // $ Alert // SSRF // validating domain only const decodedURI = decodeURIComponent(req.query.url); @@ -22,8 +22,8 @@ app.get('/check-with-axios', req => { const { hostname } = url.parse(decodedURI); - if (isValidDomain(hostname, validDomains)) { - axios.get(req.query.url); //SSRF + if (isValidDomain(hostname, VALID_DOMAINS)) { + axios.get(req.query.url); // $ Alert // SSRF } }); @@ -31,4 +31,4 @@ const isValidDomain = (hostname, validDomains) => ( validDomains.some(domain => ( hostname === domain || hostname.endsWith(`.${domain}`)) ) -); \ No newline at end of file +); diff --git a/javascript/ql/test/experimental/Security/CWE-918/check-middleware.js b/javascript/ql/test/experimental/Security/CWE-918/check-middleware.js index 2a1e6d54166..f79250ae5f7 100644 --- a/javascript/ql/test/experimental/Security/CWE-918/check-middleware.js +++ b/javascript/ql/test/experimental/Security/CWE-918/check-middleware.js @@ -6,7 +6,7 @@ const express = require('express'); const app = express(); app.get('/check-with-axios', validationMiddleware, req => { - axios.get("test.com/" + req.query.tainted); // OK is sanitized by the middleware - False Positive + axios.get("test.com/" + req.query.tainted); // $ Alert // OK is sanitized by the middleware - False Positive }); diff --git a/javascript/ql/test/experimental/Security/CWE-918/check-path.js b/javascript/ql/test/experimental/Security/CWE-918/check-path.js index b26e4924460..25427ec7f8e 100644 --- a/javascript/ql/test/experimental/Security/CWE-918/check-path.js +++ b/javascript/ql/test/experimental/Security/CWE-918/check-path.js @@ -16,11 +16,11 @@ app.get('/check-with-axios', req => { const hardcoded = 'hardcodeado'; axios.get('test.com/' + hardcoded); // OK - axios.get('test.com/' + req.query.tainted); // SSRF + axios.get('test.com/' + req.query.tainted); // $ Alert // SSRF axios.get('test.com/' + Number(req.query.tainted)); // OK axios.get('test.com/' + req.user.id); // OK axios.get('test.com/' + encodeURIComponent(req.query.tainted)); // OK - axios.get(`/addresses/${req.query.tainted}`); // SSRF + axios.get(`/addresses/${req.query.tainted}`); // $ Alert // SSRF axios.get(`/addresses/${encodeURIComponent(req.query.tainted)}`); // OK if (Number.isInteger(req.query.tainted)) { @@ -30,11 +30,11 @@ app.get('/check-with-axios', req => { if (isValidInput(req.query.tainted)){ axios.get('test.com/' + req.query.tainted); // OK } else { - axios.get('test.com/' + req.query.tainted); // SSRF + axios.get('test.com/' + req.query.tainted); // $ Alert // SSRF } if (doesntCheckAnything(req.query.tainted)) { - axios.get('test.com/' + req.query.tainted); // SSRF + axios.get('test.com/' + req.query.tainted); // $ Alert // SSRF } if (isValidPath(req.query.tainted, VALID_PATHS)) { @@ -42,7 +42,7 @@ app.get('/check-with-axios', req => { } let baseURL = require('config').base - axios.get(`${baseURL}${req.query.tainted}`); // SSRF + axios.get(`${baseURL}${req.query.tainted}`); // $ Alert // SSRF if(!isValidInput(req.query.tainted)) { return; diff --git a/javascript/ql/test/experimental/Security/CWE-918/check-regex.js b/javascript/ql/test/experimental/Security/CWE-918/check-regex.js index 238aa906843..9d09d1832ef 100644 --- a/javascript/ql/test/experimental/Security/CWE-918/check-regex.js +++ b/javascript/ql/test/experimental/Security/CWE-918/check-regex.js @@ -13,7 +13,7 @@ app.get('/check-with-axios', req => { axios.get("test.com/" + req.query.tainted); // OK } if (req.query.tainted.match(/^.*$/)) { // anything - axios.get("test.com/" + req.query.tainted); // SSRF - False Negative + axios.get("test.com/" + req.query.tainted); // $ Alert // SSRF - False Negative } const baseURL = "test.com/" @@ -21,24 +21,24 @@ app.get('/check-with-axios', req => { axios.get(baseURL + req.params.tainted); // OK } if (!isValidPath(req.params.tainted) ) { - axios.get(baseURL + req.params.tainted); // SSRF + axios.get(baseURL + req.params.tainted); // $ Alert // SSRF } else { axios.get(baseURL + req.params.tainted); // OK } // Blacklists are not safe if (!req.query.tainted.match(/^[/\.%]+$/)) { - axios.get("test.com/" + req.query.tainted); // SSRF + axios.get("test.com/" + req.query.tainted); // $ Alert // SSRF } if (!isInBlacklist(req.params.tainted) ) { - axios.get(baseURL + req.params.tainted); // SSRF + axios.get(baseURL + req.params.tainted); // $ Alert // SSRF } if (!isValidPath(req.params.tainted)) { return; } - axios.get("test.com/" + req.query.tainted); // OK - False Positive + axios.get("test.com/" + req.query.tainted); // $ Alert // OK - False Positive if (req.query.tainted.matchAll(/^[0-9a-z]+$/g)) { // letters and numbers axios.get("test.com/" + req.query.tainted); // OK @@ -58,7 +58,7 @@ app.get('/check-with-axios', req => { axios.get(baseURL + req.params.tainted); // OK } if (!isValidPathMatchAll(req.params.tainted) ) { - axios.get(baseURL + req.params.tainted); // NOT OK - SSRF + axios.get(baseURL + req.params.tainted); // $ Alert // NOT OK - SSRF } else { axios.get(baseURL + req.params.tainted); // OK } diff --git a/javascript/ql/test/experimental/Security/CWE-918/check-validator.js b/javascript/ql/test/experimental/Security/CWE-918/check-validator.js index dfe3314b07b..631f032d956 100644 --- a/javascript/ql/test/experimental/Security/CWE-918/check-validator.js +++ b/javascript/ql/test/experimental/Security/CWE-918/check-validator.js @@ -12,7 +12,7 @@ app.get("/check-with-axios", req => { axios.get("test.com/" + req.query.tainted); // OK } if (isAlphanumeric(req.query.tainted)) { - axios.get("test.com/" + req.query.tainted); // SSRF + axios.get("test.com/" + req.query.tainted); // $ Alert // SSRF } if (validAlphanumeric(req.query.tainted)) { axios.get("test.com/" + req.query.tainted); // OK @@ -24,7 +24,7 @@ app.get("/check-with-axios", req => { axios.get("test.com/" + req.query.tainted); // OK } if (wrongValidation(req.query.tainted)) { - axios.get("test.com/" + req.query.tainted); // SSRF + axios.get("test.com/" + req.query.tainted); // $ Alert // SSRF } // numbers @@ -47,25 +47,25 @@ app.get("/check-with-axios", req => { axios.get("test.com/" + req.query.tainted); // OK } if (validHexa(req.query.tainted)) { - axios.get("test.com/" + req.query.tainted); // OK. False Positive + axios.get("test.com/" + req.query.tainted); // $ Alert // OK. False Positive } // with simple assignation - const numberURL = req.query.tainted; + const numberURL = req.query.tainted; // $ Source if (validNumber(numberURL)) { axios.get("test.com/" + numberURL); // OK } if (validNumber(numberURL)) { - axios.get("test.com/" + req.query.tainted); // OK. False Positive + axios.get("test.com/" + req.query.tainted); // $ Alert // OK. False Positive } if (validNumber(req.query.tainted)) { - axios.get("test.com/" + numberURL); // OK. False Positive + axios.get("test.com/" + numberURL); // $ Alert // OK. False Positive } if (validHexadecimal(req.query.tainted) || validHexaColor(req.query.tainted) || validDecimal(req.query.tainted) || validFloat(req.query.tainted) || validInt(req.query.tainted) || validNumber(req.query.tainted) || validOctal(req.query.tainted)) { - axios.get("test.com/" + req.query.tainted); // OK. False Positive + axios.get("test.com/" + req.query.tainted); // $ Alert // OK. False Positive } }); diff --git a/javascript/ql/test/experimental/StandardLibrary/MultipleArgumentsToSetConstructor/MultipleArgumentsToSetConstructor.qlref b/javascript/ql/test/experimental/StandardLibrary/MultipleArgumentsToSetConstructor/MultipleArgumentsToSetConstructor.qlref index 3cba54a3a0c..51cb2f3db8b 100644 --- a/javascript/ql/test/experimental/StandardLibrary/MultipleArgumentsToSetConstructor/MultipleArgumentsToSetConstructor.qlref +++ b/javascript/ql/test/experimental/StandardLibrary/MultipleArgumentsToSetConstructor/MultipleArgumentsToSetConstructor.qlref @@ -1 +1,2 @@ -experimental/StandardLibrary/MultipleArgumentsToSetConstructor.ql +query: experimental/StandardLibrary/MultipleArgumentsToSetConstructor.ql +postprocess: utils/test/InlineExpectationsTestQuery.ql diff --git a/javascript/ql/test/experimental/StandardLibrary/MultipleArgumentsToSetConstructor/MultipleArgumentsToSetConstructorBad.js b/javascript/ql/test/experimental/StandardLibrary/MultipleArgumentsToSetConstructor/MultipleArgumentsToSetConstructorBad.js index 4bce4b54c1b..ea51918032a 100644 --- a/javascript/ql/test/experimental/StandardLibrary/MultipleArgumentsToSetConstructor/MultipleArgumentsToSetConstructorBad.js +++ b/javascript/ql/test/experimental/StandardLibrary/MultipleArgumentsToSetConstructor/MultipleArgumentsToSetConstructorBad.js @@ -1,4 +1,4 @@ -const vowels = new Set('a', 'e', 'i', 'o', 'u'); +const vowels = new Set('a', 'e', 'i', 'o', 'u'); // $ Alert function isVowel(char) { return vowels.has(char.toLowerCase()); diff --git a/javascript/ql/test/experimental/StandardLibrary/MultipleArgumentsToSetConstructor/tst.js b/javascript/ql/test/experimental/StandardLibrary/MultipleArgumentsToSetConstructor/tst.js index 7f43ae5f966..1f12916af5b 100644 --- a/javascript/ql/test/experimental/StandardLibrary/MultipleArgumentsToSetConstructor/tst.js +++ b/javascript/ql/test/experimental/StandardLibrary/MultipleArgumentsToSetConstructor/tst.js @@ -1,6 +1,6 @@ let xs = [1, 2, 3]; let ys = [4, 5, 6]; -new Set(...xs, ...ys); // NOT OK +new Set(...xs, ...ys); // $ Alert // NOT OK new Set([...xs, ...ys]); // OK new Set(xs); // OK new Set(); // OK \ No newline at end of file diff --git a/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/UnpromotedRouteHandlerCandidate.qlref b/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/UnpromotedRouteHandlerCandidate.qlref index 51fb87eed72..c383d032bc0 100644 --- a/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/UnpromotedRouteHandlerCandidate.qlref +++ b/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/UnpromotedRouteHandlerCandidate.qlref @@ -1 +1,2 @@ -meta/analysis-quality/UnpromotedRouteHandlerCandidate.ql \ No newline at end of file +query: meta/analysis-quality/UnpromotedRouteHandlerCandidate.ql +postprocess: utils/test/InlineExpectationsTestQuery.ql diff --git a/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/UnpromotedRouteSetupCandidate.qlref b/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/UnpromotedRouteSetupCandidate.qlref index 5ce57dc19ba..2ef54efcfd0 100644 --- a/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/UnpromotedRouteSetupCandidate.qlref +++ b/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/UnpromotedRouteSetupCandidate.qlref @@ -1 +1,2 @@ -meta/analysis-quality/UnpromotedRouteSetupCandidate.ql \ No newline at end of file +query: meta/analysis-quality/UnpromotedRouteSetupCandidate.ql +postprocess: utils/test/InlineExpectationsTestQuery.ql diff --git a/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/src/hapi.js b/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/src/hapi.js index 581e2401e05..9ea46e7ce68 100644 --- a/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/src/hapi.js +++ b/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/src/hapi.js @@ -1 +1 @@ -function handler(request, h){} +function handler(request, h){} // $ Alert[js/unpromoted-route-handler-candidate] diff --git a/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/src/nodejs.js b/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/src/nodejs.js index 57bcde69d53..315c6dd1379 100644 --- a/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/src/nodejs.js +++ b/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/src/nodejs.js @@ -2,14 +2,14 @@ var http = require('http'); http.createServer(function(req, res){}); -unknown.createServer(function(req, res){}); +unknown.createServer(function(req, res){}); // $ Alert[js/unpromoted-route-setup-candidate] var createServer = http.createServer; createServer(function(req, res){}); http.createServer().on("request", function(req, res){}); -unknown.on("request", function(req, res){}); -unknown.once("request", function(req, res){}); +unknown.on("request", function(req, res){}); // $ Alert[js/unpromoted-route-setup-candidate] +unknown.once("request", function(req, res){}); // $ Alert[js/unpromoted-route-setup-candidate] function getHandler(){ return function(req, res){}; diff --git a/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/src/route-objects.js b/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/src/route-objects.js index 64dbe455560..2abb91d0f11 100644 --- a/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/src/route-objects.js +++ b/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/src/route-objects.js @@ -4,10 +4,10 @@ var app = express(); var route1 = { method: 'post', url: '/foo', - middleWares: [function(req, res){}], + middleWares: [function(req, res){}], // $ Alert[js/unpromoted-route-handler-candidate] handler(req, res) { - } + } // $ Alert[js/unpromoted-route-handler-candidate] }; app[route1.method](route1.url, route1.middleWares, route1.handler); @@ -19,14 +19,14 @@ var routes = [ url: '/foo', handler(req, res) { - } + } // $ Alert[js/unpromoted-route-handler-candidate] }, { method: 'post', url: '/foo', handler(req, res) { - } + } // $ Alert[js/unpromoted-route-handler-candidate] } ]; routes.forEach((route) => { @@ -39,7 +39,7 @@ var route2 = { url: '/foo', handler(req, res) { - } + } // $ Alert[js/unpromoted-route-handler-candidate] }; app[route2.method.toLowerCase()](route2.url, route2.handler); @@ -49,13 +49,13 @@ var route3 = { url: '/foo', handler(req, res) { - } + } // $ Alert[js/unpromoted-route-handler-candidate] }; function wrap(f){ return function(req, res){ f(req); - } + } // $ Alert[js/unpromoted-route-handler-candidate] } app[route3.method](route3.url, wrap(route3.handler)); confuse(wrap); // confuse the type inference diff --git a/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/src/tst.js b/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/src/tst.js index e2d6cfd1ebf..871ddf9603b 100644 --- a/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/src/tst.js +++ b/javascript/ql/test/library-tests/frameworks/HTTP-heuristics/src/tst.js @@ -3,9 +3,9 @@ var app = express(); app.get('/some/path', function(req, res) {}) -someOtherApp.get('/some/path', function(req, res) {}) +someOtherApp.get('/some/path', function(req, res) {}) // $ Alert[js/unpromoted-route-setup-candidate] -someOtherApp.get('/some/path', function(request, response) {}) +someOtherApp.get('/some/path', function(request, response) {}) // $ Alert[js/unpromoted-route-setup-candidate] someOtherApp.get('/some/path', function(r) { r.acceptsCharsets() @@ -27,23 +27,23 @@ someOtherApp.get('/some/path', function(r, s, n) { n('route') }) -someOtherApp.delete('/some/path', function(req, res) {}) +someOtherApp.delete('/some/path', function(req, res) {}) // $ Alert[js/unpromoted-route-setup-candidate] someOtherApp.get('/some/path', function(req, res) {}, - function(req, res) {}) + function(req, res) {}) // $ Alert[js/unpromoted-route-setup-candidate] someOtherApp.get('/some/path', [ function(req, res) {}, function(req, res) {} -]) +]) // $ Alert[js/unpromoted-route-setup-candidate] someOtherApp.get('/some/path', function() {}, - function(req, res) {}) + function(req, res) {}) // $ Alert[js/unpromoted-route-setup-candidate] -function f(req, res) {} +function f(req, res) {} // $ Alert[js/unpromoted-route-handler-candidate] function f(ctx, next) { ctx.acceptsCharsets() @@ -51,25 +51,25 @@ function f(ctx, next) { function f(req, res) { req() -} +} // $ Alert[js/unpromoted-route-handler-candidate] function called(req,res) { -} +} // $ Alert[js/unpromoted-route-handler-candidate] called() function f(req,res) { return; -} +} // $ Alert[js/unpromoted-route-handler-candidate] function f(req,res) { return x; -} +} // $ Alert[js/unpromoted-route-handler-candidate] function adHocTestsFor_HeuristicRouteHandler() { function rh_dead(req, res) { - } + } // $ Alert[js/unpromoted-route-handler-candidate] function rh_flowToSetup(req, res) { @@ -84,7 +84,7 @@ function adHocTestsFor_HeuristicRouteHandler() { function rh_flowToHeuristicSetup(req, res) { } - unknownApp.get('/some/path', rh_flowToHeuristicSetup) + unknownApp.get('/some/path', rh_flowToHeuristicSetup) // $ Alert[js/unpromoted-route-setup-candidate] } function adHocTestsFor_HeuristicRouteSetups() { @@ -93,22 +93,22 @@ function adHocTestsFor_HeuristicRouteSetups() { } app.get('/some/path', rh); - unknownApp.get('/some/path', rh); + unknownApp.get('/some/path', rh); // $ Alert[js/unpromoted-route-setup-candidate] - unknownApp.get('/some/path', [rh]); + unknownApp.get('/some/path', [rh]); // $ Alert[js/unpromoted-route-setup-candidate] unknownApp.get('/some/path', unknown); unknownApp.get('/some/path', [unknown]); - unknownApp.get('/some/path', unknown, rh); + unknownApp.get('/some/path', unknown, rh); // $ Alert[js/unpromoted-route-setup-candidate] } function adHocTestsFor_HeuristicRouteHandler_withTracking() { function get_rh_dead() { return function rh_dead(req, res) { - } + } // $ Alert[js/unpromoted-route-handler-candidate] } var rh_dead = get_rh_dead(); @@ -134,7 +134,7 @@ function adHocTestsFor_HeuristicRouteHandler_withTracking() { } } var rh_flowToHeuristicSetup = get_rh_flowToHeuristicSetup(); - unknownApp.get('/some/path', rh_flowToHeuristicSetup) + unknownApp.get('/some/path', rh_flowToHeuristicSetup) // $ Alert[js/unpromoted-route-setup-candidate] } function adHocTestsFor_HeuristicRouteSetups_withTracking() { @@ -146,13 +146,13 @@ function adHocTestsFor_HeuristicRouteSetups_withTracking() { var rh = get_rh(); app.get('/some/path', rh); - unknownApp.get('/some/path', rh); + unknownApp.get('/some/path', rh); // $ Alert[js/unpromoted-route-setup-candidate] - unknownApp.get('/some/path', [rh]); + unknownApp.get('/some/path', [rh]); // $ Alert[js/unpromoted-route-setup-candidate] unknownApp.get('/some/path', unknown); unknownApp.get('/some/path', [unknown]); - unknownApp.get('/some/path', unknown, rh); + unknownApp.get('/some/path', unknown, rh); // $ Alert[js/unpromoted-route-setup-candidate] } diff --git a/javascript/ql/test/library-tests/frameworks/Templating/CodeInjection.qlref b/javascript/ql/test/library-tests/frameworks/Templating/CodeInjection.qlref index fe9adbf3b64..bfeec8aec39 100644 --- a/javascript/ql/test/library-tests/frameworks/Templating/CodeInjection.qlref +++ b/javascript/ql/test/library-tests/frameworks/Templating/CodeInjection.qlref @@ -1 +1,2 @@ -Security/CWE-094/CodeInjection.ql +query: Security/CWE-094/CodeInjection.ql +postprocess: utils/test/InlineExpectationsTestQuery.ql diff --git a/javascript/ql/test/library-tests/frameworks/Templating/app.js b/javascript/ql/test/library-tests/frameworks/Templating/app.js index 8666d79b644..2822d40be5c 100644 --- a/javascript/ql/test/library-tests/frameworks/Templating/app.js +++ b/javascript/ql/test/library-tests/frameworks/Templating/app.js @@ -12,11 +12,11 @@ app.get('/ejs', (req, res) => { }, dataInStringLiteral: req.query.dataInStringLiteral, dataInStringLiteralRaw: req.query.dataInStringLiteralRaw, - dataInGeneratedCode: req.query.dataInGeneratedCode, + dataInGeneratedCode: req.query.dataInGeneratedCode, // $ Source dataInGeneratedCodeRaw: req.query.dataInGeneratedCodeRaw, - backslashSink1: req.query.backslashSink1, + backslashSink1: req.query.backslashSink1, // $ Source backslashSink2: req.query.backslashSink2, - dataInEventHandlerString: req.query.dataInEventHandlerString, + dataInEventHandlerString: req.query.dataInEventHandlerString, // $ Source dataInEventHandlerStringRaw: req.query.dataInEventHandlerStringRaw, }); }); @@ -31,11 +31,11 @@ app.get('/hbs', (req, res) => { }, dataInStringLiteral: req.query.dataInStringLiteral, dataInStringLiteralRaw: req.query.dataInStringLiteralRaw, - dataInGeneratedCode: req.query.dataInGeneratedCode, + dataInGeneratedCode: req.query.dataInGeneratedCode, // $ Source dataInGeneratedCodeRaw: req.query.dataInGeneratedCodeRaw, - backslashSink1: req.query.backslashSink1, + backslashSink1: req.query.backslashSink1, // $ Source backslashSink2: req.query.backslashSink2, - dataInEventHandlerString: req.query.dataInEventHandlerString, + dataInEventHandlerString: req.query.dataInEventHandlerString, // $ Source dataInEventHandlerStringRaw: req.query.dataInEventHandlerStringRaw, }); }); @@ -50,20 +50,20 @@ app.get('/njk', (req, res) => { }, dataInStringLiteral: req.query.dataInStringLiteral, dataInStringLiteralRaw: req.query.dataInStringLiteralRaw, - dataInGeneratedCode: req.query.dataInGeneratedCode, - dataInGeneratedCodeRaw: req.query.dataInGeneratedCodeRaw, + dataInGeneratedCode: req.query.dataInGeneratedCode, // $ Source + dataInGeneratedCodeRaw: req.query.dataInGeneratedCodeRaw, // $ Source dataInGeneratedCodeJsonRaw: req.query.dataInGeneratedCodeJsonRaw, - backslashSink1: req.query.backslashSink1, + backslashSink1: req.query.backslashSink1, // $ Source backslashSink2: req.query.backslashSink2, - dataInEventHandlerString: req.query.dataInEventHandlerString, - dataInEventHandlerStringRaw: req.query.dataInEventHandlerStringRaw, + dataInEventHandlerString: req.query.dataInEventHandlerString, // $ Source + dataInEventHandlerStringRaw: req.query.dataInEventHandlerStringRaw, // $ Source }); }); app.get('/angularjs', (req, res) => { res.render('angularjs_sinks', { - escapedHtml: req.query.escapedHtml, - rawHtml: req.query.rawHtml, + escapedHtml: req.query.escapedHtml, // $ Source + rawHtml: req.query.rawHtml, // $ Source }); }); diff --git a/javascript/ql/test/library-tests/frameworks/Templating/views/angularjs_include.ejs b/javascript/ql/test/library-tests/frameworks/Templating/views/angularjs_include.ejs index 2d02e173275..e9a6f436a8f 100644 --- a/javascript/ql/test/library-tests/frameworks/Templating/views/angularjs_include.ejs +++ b/javascript/ql/test/library-tests/frameworks/Templating/views/angularjs_include.ejs @@ -1,5 +1,5 @@
- <%= escapedHtml %> - <%- rawHtml %> + <%= escapedHtml %> + <%- rawHtml %>
diff --git a/javascript/ql/test/library-tests/frameworks/Templating/views/angularjs_sinks.ejs b/javascript/ql/test/library-tests/frameworks/Templating/views/angularjs_sinks.ejs index 47105bb360e..5dc0c0763cf 100644 --- a/javascript/ql/test/library-tests/frameworks/Templating/views/angularjs_sinks.ejs +++ b/javascript/ql/test/library-tests/frameworks/Templating/views/angularjs_sinks.ejs @@ -1,7 +1,7 @@ - <%= escapedHtml %> - <%- rawHtml %> + <%= escapedHtml %> + <%- rawHtml %> <% include angularjs_include %> diff --git a/javascript/ql/test/library-tests/frameworks/Templating/views/ejs_sinks.ejs b/javascript/ql/test/library-tests/frameworks/Templating/views/ejs_sinks.ejs index 42dfc124a70..b25de673433 100644 --- a/javascript/ql/test/library-tests/frameworks/Templating/views/ejs_sinks.ejs +++ b/javascript/ql/test/library-tests/frameworks/Templating/views/ejs_sinks.ejs @@ -10,15 +10,15 @@ var dataInStringLiteral = "<%= dataInStringLiteral %>"; var dataInStringLiteralRaw = "<%- dataInStringLiteralRaw %>"; - var dataInGeneratedCode = <%= dataInGeneratedCode %>; + var dataInGeneratedCode = <%= dataInGeneratedCode %>; // $ Alert var dataInGeneratedCodeRaw = <%- dataInGeneratedCodeRaw %>; - init("<%= backslashSink1 %>", "<%= backslashSink2 %>"); + init("<%= backslashSink1 %>", "<%= backslashSink2 %>"); // $ Alert var mustache = "{{ rawHtml }}"; - + <%- include('ejs_include1', { foo: rawHtml }) _%> diff --git a/javascript/ql/test/library-tests/frameworks/Templating/views/hbs_sinks.hbs b/javascript/ql/test/library-tests/frameworks/Templating/views/hbs_sinks.hbs index 198af1cd8d4..a723f95e774 100644 --- a/javascript/ql/test/library-tests/frameworks/Templating/views/hbs_sinks.hbs +++ b/javascript/ql/test/library-tests/frameworks/Templating/views/hbs_sinks.hbs @@ -22,15 +22,15 @@ var dataInStringLiteral = "{{ dataInStringLiteral }}"; var dataInStringLiteralRaw = "{{{ dataInStringLiteralRaw }}}"; - var dataInGeneratedCode = {{ dataInGeneratedCode }}; + var dataInGeneratedCode = {{ dataInGeneratedCode }}; // $ Alert var dataInGeneratedCodeRaw = {{{ dataInGeneratedCodeRaw }}}; - init("{{ backslashSink1 }}", "{{ backslashSink2 }}"); + init("{{ backslashSink1 }}", "{{ backslashSink2 }}"); // $ Alert var ejs = "<%= rawHtml %>"; - + diff --git a/javascript/ql/test/library-tests/frameworks/Templating/views/njk_sinks.njk b/javascript/ql/test/library-tests/frameworks/Templating/views/njk_sinks.njk index fcfba32a26c..dc34a5c0bbb 100644 --- a/javascript/ql/test/library-tests/frameworks/Templating/views/njk_sinks.njk +++ b/javascript/ql/test/library-tests/frameworks/Templating/views/njk_sinks.njk @@ -10,16 +10,16 @@ var dataInStringLiteral = "{{ dataInStringLiteral }}"; var dataInStringLiteralRaw = "{{ dataInStringLiteralRaw | safe }}"; - var dataInGeneratedCode = {{ dataInGeneratedCode }}; - var dataInGeneratedCodeRaw = {{ dataInGeneratedCodeRaw | safe }}; + var dataInGeneratedCode = {{ dataInGeneratedCode }}; // $ Alert + var dataInGeneratedCodeRaw = {{ dataInGeneratedCodeRaw | safe }}; // $ Alert var dataInGeneratedCodeJsonRaw = {{ dataInGeneratedCodeJsonRaw | json | safe }}; - init("{{ backslashSink1 }}", "{{ backslashSink2 }}"); + init("{{ backslashSink1 }}", "{{ backslashSink2 }}"); // $ Alert var ejs = "<%= rawHtml %>"; - - + + From de7723f214b3d63060add7a59f390d3633ac903b Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Sun, 5 Jul 2026 17:07:49 +0100 Subject: [PATCH 07/90] Add SPURIOUS and MISSING tags --- .../Security/CWE-918/check-middleware.js | 2 +- .../Security/CWE-918/check-regex.js | 20 +++++++++---------- .../Security/CWE-918/check-validator.js | 16 +++++++-------- .../missing-explicit-injection.js | 2 +- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/javascript/ql/test/experimental/Security/CWE-918/check-middleware.js b/javascript/ql/test/experimental/Security/CWE-918/check-middleware.js index f79250ae5f7..5895501e7bf 100644 --- a/javascript/ql/test/experimental/Security/CWE-918/check-middleware.js +++ b/javascript/ql/test/experimental/Security/CWE-918/check-middleware.js @@ -6,7 +6,7 @@ const express = require('express'); const app = express(); app.get('/check-with-axios', validationMiddleware, req => { - axios.get("test.com/" + req.query.tainted); // $ Alert // OK is sanitized by the middleware - False Positive + axios.get("test.com/" + req.query.tainted); // $ SPURIOUS: Alert // OK is sanitized by the middleware - False Positive }); diff --git a/javascript/ql/test/experimental/Security/CWE-918/check-regex.js b/javascript/ql/test/experimental/Security/CWE-918/check-regex.js index 9d09d1832ef..710a811857f 100644 --- a/javascript/ql/test/experimental/Security/CWE-918/check-regex.js +++ b/javascript/ql/test/experimental/Security/CWE-918/check-regex.js @@ -13,14 +13,14 @@ app.get('/check-with-axios', req => { axios.get("test.com/" + req.query.tainted); // OK } if (req.query.tainted.match(/^.*$/)) { // anything - axios.get("test.com/" + req.query.tainted); // $ Alert // SSRF - False Negative + axios.get("test.com/" + req.query.tainted); // $ Alert // SSRF } const baseURL = "test.com/" - if (isValidPath(req.params.tainted) ) { + if (isValidPath(req.params.tainted)) { axios.get(baseURL + req.params.tainted); // OK } - if (!isValidPath(req.params.tainted) ) { + if (!isValidPath(req.params.tainted)) { axios.get(baseURL + req.params.tainted); // $ Alert // SSRF } else { axios.get(baseURL + req.params.tainted); // OK @@ -30,7 +30,7 @@ app.get('/check-with-axios', req => { if (!req.query.tainted.match(/^[/\.%]+$/)) { axios.get("test.com/" + req.query.tainted); // $ Alert // SSRF } - if (!isInBlacklist(req.params.tainted) ) { + if (!isInBlacklist(req.params.tainted)) { axios.get(baseURL + req.params.tainted); // $ Alert // SSRF } @@ -38,7 +38,7 @@ app.get('/check-with-axios', req => { return; } - axios.get("test.com/" + req.query.tainted); // $ Alert // OK - False Positive + axios.get("test.com/" + req.query.tainted); // $ SPURIOUS: Alert // OK - False Positive if (req.query.tainted.matchAll(/^[0-9a-z]+$/g)) { // letters and numbers axios.get("test.com/" + req.query.tainted); // OK @@ -48,20 +48,20 @@ app.get('/check-with-axios', req => { } }); -const isValidPath = path => path.match(/^[0-9a-z]+$/); +const isValidPath = path => path.match(/^[0-9a-z]+$/); -const isInBlackList = path => path.match(/^[/\.%]+$/); +const isInBlackList = path => path.match(/^[/\.%]+$/); app.get('/check-with-axios', req => { const baseURL = "test.com/" - if (isValidPathMatchAll(req.params.tainted) ) { + if (isValidPathMatchAll(req.params.tainted)) { axios.get(baseURL + req.params.tainted); // OK } - if (!isValidPathMatchAll(req.params.tainted) ) { + if (!isValidPathMatchAll(req.params.tainted)) { axios.get(baseURL + req.params.tainted); // $ Alert // NOT OK - SSRF } else { axios.get(baseURL + req.params.tainted); // OK } }); -const isValidPathMatchAll = path => path.matchAll(/^[0-9a-z]+$/g); +const isValidPathMatchAll = path => path.matchAll(/^[0-9a-z]+$/g); diff --git a/javascript/ql/test/experimental/Security/CWE-918/check-validator.js b/javascript/ql/test/experimental/Security/CWE-918/check-validator.js index 631f032d956..37d9895b24e 100644 --- a/javascript/ql/test/experimental/Security/CWE-918/check-validator.js +++ b/javascript/ql/test/experimental/Security/CWE-918/check-validator.js @@ -47,7 +47,7 @@ app.get("/check-with-axios", req => { axios.get("test.com/" + req.query.tainted); // OK } if (validHexa(req.query.tainted)) { - axios.get("test.com/" + req.query.tainted); // $ Alert // OK. False Positive + axios.get("test.com/" + req.query.tainted); // $ SPURIOUS: Alert // OK. False Positive } // with simple assignation @@ -56,16 +56,16 @@ app.get("/check-with-axios", req => { axios.get("test.com/" + numberURL); // OK } if (validNumber(numberURL)) { - axios.get("test.com/" + req.query.tainted); // $ Alert // OK. False Positive + axios.get("test.com/" + req.query.tainted); // $ SPURIOUS: Alert // OK. False Positive } if (validNumber(req.query.tainted)) { - axios.get("test.com/" + numberURL); // $ Alert // OK. False Positive + axios.get("test.com/" + numberURL); // $ SPURIOUS: Alert // OK. False Positive } - if (validHexadecimal(req.query.tainted) || validHexaColor(req.query.tainted) || - validDecimal(req.query.tainted) || validFloat(req.query.tainted) || validInt(req.query.tainted) || - validNumber(req.query.tainted) || validOctal(req.query.tainted)) { - axios.get("test.com/" + req.query.tainted); // $ Alert // OK. False Positive + if (validHexadecimal(req.query.tainted) || validHexaColor(req.query.tainted) || + validDecimal(req.query.tainted) || validFloat(req.query.tainted) || validInt(req.query.tainted) || + validNumber(req.query.tainted) || validOctal(req.query.tainted)) { + axios.get("test.com/" + req.query.tainted); // $ SPURIOUS: Alert // OK. False Positive } }); @@ -93,6 +93,6 @@ const validHexaColor = url => validator.isHexColor(url); const validUUID = url => validator.isUUID(url); // unsafe validators -const wrongValidation = url => validator.isByteLength(url, {min:4,max:8}); +const wrongValidation = url => validator.isByteLength(url, { min: 4, max: 8 }); const isAlphanumeric = url => true; diff --git a/javascript/ql/test/query-tests/AngularJS/MissingExplicitInjection/missing-explicit-injection.js b/javascript/ql/test/query-tests/AngularJS/MissingExplicitInjection/missing-explicit-injection.js index 629b62d5b08..85186dfe8dc 100644 --- a/javascript/ql/test/query-tests/AngularJS/MissingExplicitInjection/missing-explicit-injection.js +++ b/javascript/ql/test/query-tests/AngularJS/MissingExplicitInjection/missing-explicit-injection.js @@ -21,7 +21,7 @@ controller: notInjected7 }; - function injected8(name){} // OK - false negative: we do not track through properties + function injected8(name){} // $ MISSING: Alert // false negative: we do not track through properties var obj8 = { controller: injected8 }; From 662c7b08e8abafda2d3b4d2f75a407ebb481701b Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 2 Jul 2026 14:31:30 +0200 Subject: [PATCH 08/90] Swift: Turn of caching and integrated driver in autobuild * Caching cases compiler invocations to be omitted. You can try this for yourself by turning on caching while building the project from the `xcode-hello` integration test, then cleaning of the build artifacts and building the project again while caching is enabled. This yields a database that is practically empty. * The integrated driver causes compiler invocations to depend on modules shipped with Xcode. Those are unfortunately incompatible with our extractor. --- swift/swift-autobuilder/BuildRunner.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/swift/swift-autobuilder/BuildRunner.cpp b/swift/swift-autobuilder/BuildRunner.cpp index 46108e8259d..2e0045d3c2b 100644 --- a/swift/swift-autobuilder/BuildRunner.cpp +++ b/swift/swift-autobuilder/BuildRunner.cpp @@ -79,6 +79,9 @@ bool buildXcodeTarget(const XcodeTarget& target, bool dryRun) { argv.push_back(target.name); argv.push_back("CODE_SIGNING_REQUIRED=NO"); argv.push_back("CODE_SIGNING_ALLOWED=NO"); + argv.push_back("COMPILATION_CACHE_ENABLE_CACHING=NO"); + argv.push_back("SWIFT_ENABLE_COMPILE_CACHE=NO"); + argv.push_back("SWIFT_USE_INTEGRATED_DRIVER=NO"); return run_build_command(argv, dryRun); } From 8ecdb2cde0c6dd563edf12adf563aa8935bb2ea0 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 2 Jul 2026 14:36:34 +0200 Subject: [PATCH 09/90] Swift: update integration tests to not use the integrated driver --- swift/ql/integration-tests/osx/hello-ios/test.py | 3 ++- swift/ql/integration-tests/osx/hello-xcode/test.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/swift/ql/integration-tests/osx/hello-ios/test.py b/swift/ql/integration-tests/osx/hello-ios/test.py index b5871f323cc..ecb4a940d8b 100644 --- a/swift/ql/integration-tests/osx/hello-ios/test.py +++ b/swift/ql/integration-tests/osx/hello-ios/test.py @@ -7,5 +7,6 @@ import pytest def test(codeql, swift, xcode_16): codeql.database.create( command="xcodebuild build " - "CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO", + "CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO " + "SWIFT_USE_INTEGRATED_DRIVER=NO", ) diff --git a/swift/ql/integration-tests/osx/hello-xcode/test.py b/swift/ql/integration-tests/osx/hello-xcode/test.py index 35b0d6dae58..cf6a17f3908 100644 --- a/swift/ql/integration-tests/osx/hello-xcode/test.py +++ b/swift/ql/integration-tests/osx/hello-xcode/test.py @@ -9,5 +9,6 @@ def test(codeql, swift, xcode_all): command="xcodebuild build " "-project codeql-swift-autobuild-test.xcodeproj " "-target codeql-swift-autobuild-test " - "CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO", + "CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO " + "SWIFT_USE_INTEGRATED_DRIVER=NO", ) From b4db71a38b9938cb9c5828bc096ff6aa48340143 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 2 Jul 2026 14:46:21 +0200 Subject: [PATCH 10/90] Swift: Update autobuild tests --- .../integration-tests/autobuilder/failure/diagnostics.expected | 2 +- .../swift-autobuilder/tests/hello-autobuilder/commands.expected | 2 +- .../tests/hello-targets-with-tests-suffix/commands.expected | 2 +- swift/swift-autobuilder/tests/hello-tests/commands.expected | 2 +- swift/swift-autobuilder/tests/hello-workspace/commands.expected | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/swift/ql/integration-tests/autobuilder/failure/diagnostics.expected b/swift/ql/integration-tests/autobuilder/failure/diagnostics.expected index 2f43e334bc0..248495e3bd2 100644 --- a/swift/ql/integration-tests/autobuilder/failure/diagnostics.expected +++ b/swift/ql/integration-tests/autobuilder/failure/diagnostics.expected @@ -1,5 +1,5 @@ { - "markdownMessage": "`autobuild` failed to run the build command:\n\n```\n/usr/bin/xcodebuild build -project /hello-failure.xcodeproj -target hello-failure CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO\n```\n\nSet up a [manual build command][1] or [check the logs of the autobuild step][2].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language\n[2]: https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/using-workflow-run-logs", + "markdownMessage": "`autobuild` failed to run the build command:\n\n```\n/usr/bin/xcodebuild build -project /hello-failure.xcodeproj -target hello-failure CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO COMPILATION_CACHE_ENABLE_CACHING=NO SWIFT_ENABLE_COMPILE_CACHE=NO SWIFT_USE_INTEGRATED_DRIVER=NO\n```\n\nSet up a [manual build command][1] or [check the logs of the autobuild step][2].\n\n[1]: https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#adding-build-steps-for-a-compiled-language\n[2]: https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/using-workflow-run-logs", "severity": "error", "source": { "extractorName": "swift", diff --git a/swift/swift-autobuilder/tests/hello-autobuilder/commands.expected b/swift/swift-autobuilder/tests/hello-autobuilder/commands.expected index 9d2be19b9c4..dd086268ddb 100644 --- a/swift/swift-autobuilder/tests/hello-autobuilder/commands.expected +++ b/swift/swift-autobuilder/tests/hello-autobuilder/commands.expected @@ -1 +1 @@ -/usr/bin/xcodebuild build -project ./hello-autobuilder.xcodeproj -target hello-autobuilder CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO +/usr/bin/xcodebuild build -project ./hello-autobuilder.xcodeproj -target hello-autobuilder CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO COMPILATION_CACHE_ENABLE_CACHING=NO SWIFT_ENABLE_COMPILE_CACHE=NO SWIFT_USE_INTEGRATED_DRIVER=NO diff --git a/swift/swift-autobuilder/tests/hello-targets-with-tests-suffix/commands.expected b/swift/swift-autobuilder/tests/hello-targets-with-tests-suffix/commands.expected index 4506ff8aa56..3d31114ca32 100644 --- a/swift/swift-autobuilder/tests/hello-targets-with-tests-suffix/commands.expected +++ b/swift/swift-autobuilder/tests/hello-targets-with-tests-suffix/commands.expected @@ -1 +1 @@ -/usr/bin/xcodebuild build -project ./Foo.xcodeproj -target FooDemo CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO +/usr/bin/xcodebuild build -project ./Foo.xcodeproj -target FooDemo CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO COMPILATION_CACHE_ENABLE_CACHING=NO SWIFT_ENABLE_COMPILE_CACHE=NO SWIFT_USE_INTEGRATED_DRIVER=NO diff --git a/swift/swift-autobuilder/tests/hello-tests/commands.expected b/swift/swift-autobuilder/tests/hello-tests/commands.expected index a34306fe74c..46e061896a8 100644 --- a/swift/swift-autobuilder/tests/hello-tests/commands.expected +++ b/swift/swift-autobuilder/tests/hello-tests/commands.expected @@ -1 +1 @@ -/usr/bin/xcodebuild build -project ./hello-tests.xcodeproj -target hello-tests CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO +/usr/bin/xcodebuild build -project ./hello-tests.xcodeproj -target hello-tests CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO COMPILATION_CACHE_ENABLE_CACHING=NO SWIFT_ENABLE_COMPILE_CACHE=NO SWIFT_USE_INTEGRATED_DRIVER=NO diff --git a/swift/swift-autobuilder/tests/hello-workspace/commands.expected b/swift/swift-autobuilder/tests/hello-workspace/commands.expected index ad85eb8c24b..d9d20e69d08 100644 --- a/swift/swift-autobuilder/tests/hello-workspace/commands.expected +++ b/swift/swift-autobuilder/tests/hello-workspace/commands.expected @@ -1 +1 @@ -/usr/bin/xcodebuild build -workspace ./Hello.xcworkspace -scheme hello-workspace CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO +/usr/bin/xcodebuild build -workspace ./Hello.xcworkspace -scheme hello-workspace CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO COMPILATION_CACHE_ENABLE_CACHING=NO SWIFT_ENABLE_COMPILE_CACHE=NO SWIFT_USE_INTEGRATED_DRIVER=NO From 36af59adbf9771e7911c8dc5310c8fde4f95f217 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 2 Jul 2026 15:34:22 +0200 Subject: [PATCH 11/90] Swift: Update integration test --- .../autobuilder/xcode-fails-spm-works/Files.macos_26.expected | 3 --- .../autobuilder/xcode-fails-spm-works/test.py | 3 +-- 2 files changed, 1 insertion(+), 5 deletions(-) delete mode 100644 swift/ql/integration-tests/autobuilder/xcode-fails-spm-works/Files.macos_26.expected diff --git a/swift/ql/integration-tests/autobuilder/xcode-fails-spm-works/Files.macos_26.expected b/swift/ql/integration-tests/autobuilder/xcode-fails-spm-works/Files.macos_26.expected deleted file mode 100644 index 8a3be429106..00000000000 --- a/swift/ql/integration-tests/autobuilder/xcode-fails-spm-works/Files.macos_26.expected +++ /dev/null @@ -1,3 +0,0 @@ -| Package.swift:0:0:0:0 | Package.swift | -| Sources/hello-world/hello_world.swift:0:0:0:0 | Sources/hello-world/hello_world.swift | -| file://:0:0:0:0 | | diff --git a/swift/ql/integration-tests/autobuilder/xcode-fails-spm-works/test.py b/swift/ql/integration-tests/autobuilder/xcode-fails-spm-works/test.py index 4beed91f233..298fd2726d0 100644 --- a/swift/ql/integration-tests/autobuilder/xcode-fails-spm-works/test.py +++ b/swift/ql/integration-tests/autobuilder/xcode-fails-spm-works/test.py @@ -3,7 +3,6 @@ import pytest @runs_on.macos -@pytest.mark.ql_test("DB-CHECK", xfail=not runs_on.macos_26) -@pytest.mark.ql_test("*", expected=f"{'.macos_26' if runs_on.macos_26 else ''}.expected") +@pytest.mark.ql_test("DB-CHECK", xfail=True) def test(codeql, swift): codeql.database.create() From 82728486202062b9f95bbfd4d679fde7ccf233b3 Mon Sep 17 00:00:00 2001 From: Taus Date: Sat, 4 Jul 2026 15:00:24 +0000 Subject: [PATCH 12/90] yeast: Implement IntoIterator for Id as a singleton There's an awkward divide in yeast between returning a list of nodes or optional node (both of which are iterable), and returning a single node (which is not). In practice, we would like all of these cases to be handled transparently: if a single node is returned, it behaves as if it were a singleton list containing that node. This gives us a uniform interface during translation -- no matter what is returned, it will be an iterable of nodes. To facilitate this, we make the slightly unorthodox choice of implementing IntoIterator for Id, with the behaviour detailed above. --- shared/yeast/src/lib.rs | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index fdfe4dd0fb0..c50304e623e 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -26,6 +26,12 @@ use query::QueryNode; /// without colliding with the impls for plain integers. /// /// Use `id.0` (or `id.into()`) to obtain the raw arena index. +/// +/// Implements [`IntoIterator`] as a singleton (`iter::once(self)`) +/// so that a bare `Id` can be used interchangeably with `Option` +/// / `Vec` in places that expect an iterable of ids (e.g. +/// [`crate::build::BuildCtx::translate`] and the field-splice +/// interpolation via [`IntoFieldIds`]). #[repr(transparent)] #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Debug, Hash, Serialize)] pub struct Id(pub usize); @@ -42,6 +48,14 @@ impl From for usize { } } +impl IntoIterator for Id { + type Item = Id; + type IntoIter = std::iter::Once; + fn into_iter(self) -> Self::IntoIter { + std::iter::once(self) + } +} + /// Field and Kind ids are provided by tree-sitter type FieldId = u16; type KindId = u16; @@ -49,9 +63,10 @@ type KindId = u16; /// Trait for values that can be appended to a field's id list inside a /// `tree!`/`trees!`/`rule!` template (in `{expr}` placeholders). /// -/// `Id` pushes a single id; the blanket impl for -/// `IntoIterator>` handles `Vec`, `Option`, -/// arbitrary iterators yielding `Id`, etc. +/// The blanket impl for `IntoIterator>` handles all +/// current shapes: `Vec`, `Option`, arbitrary iterators +/// yielding `Id`, and a bare `Id` itself (which is `IntoIterator` +/// via a singleton). /// /// This lets `{expr}` interpolate any of these shapes without a /// dedicated splice syntax — the macro emits the same trait-dispatched @@ -60,12 +75,6 @@ pub trait IntoFieldIds { fn extend_into(self, out: &mut Vec); } -impl IntoFieldIds for Id { - fn extend_into(self, out: &mut Vec) { - out.push(self); - } -} - impl IntoFieldIds for I where I: IntoIterator, From 13510b1dddf6d1cdb7709aacfde6f81d3dd7da1a Mon Sep 17 00:00:00 2001 From: Taus Date: Sat, 4 Jul 2026 15:00:48 +0000 Subject: [PATCH 13/90] yeast: Add BuildCtx::scoped for isolated context modification A previous commit added a translate_reset method on BuildCtx, which had the effect of performing a translation in a completely empty context. One issue with this is that this is an all-or-nothing proposition -- If you want to preserve _some_ parts of the context, you have to do something more complicated. Moreover, if you introduce a contextual value that _should_ be preserved, all of the existing uses of translate_reset now silently do the wrong thing. There are two patterns that we want to address. The first one is "modify the context in some way, then do a translation". If the translation is the last step of a Rust block, then we don't actually need translate_reset -- we could just reset the context and then call `translate`. The fact that the outer context is restored afterwards means it's okay to make destructive changes to `ctx.user_ctx` -- none of these changes will persist. The second pattern is the same, but where we want to do more translations using the original context, after having performed a translation with a modified context. In this case, we cannot just overwrite the context, since that would invalidate the subsequent translations. Instead, we introduce a new method `ctx.scoped` which takes a closure as an argument. With this we can now write ``` ctx.scoped(|ctx| ctx.reset(); ctx.translate(...)); ``` and the closure is run with a copy of `ctx` that has a clone of `user_ctx` on the inside, so no changes will persist. (You may wonder: why not just clone `ctx` and use the clone? The answer is that `ctx` owns mutable pointers to the AST etc., and this makes it awkward to just "clone" it. The closure circumvents this issue nicely, since it can borrow these pointers internally.) For now, this rewrite has the same behaviour as the version that used translate_reset -- we clear the entire `user_ctx`. However, we could imagine being more fine-grained in this approach, by implementing, say, SwiftContext::reset_modifiers (which would only affect what modifiers are currently in the context, leaving everything else as-is). --- shared/yeast/src/build.rs | 96 ++++++++++++------- shared/yeast/src/lib.rs | 20 ++++ .../extractor/src/languages/swift/swift.rs | 59 ++++++++---- 3 files changed, 121 insertions(+), 54 deletions(-) diff --git a/shared/yeast/src/build.rs b/shared/yeast/src/build.rs index 6c2a9d4181d..f550947ffd4 100644 --- a/shared/yeast/src/build.rs +++ b/shared/yeast/src/build.rs @@ -161,52 +161,74 @@ impl<'a, C> BuildCtx<'a, C> { } impl BuildCtx<'_, C> { - /// Recursively translate a node via the framework's rule machinery. - /// In a OneShot phase, applies OneShot rules to the given node and - /// returns the resulting node ids. In a Repeating phase, errors - /// (translation is not meaningful when input and output share a - /// schema). + /// Recursively translate every id in the given iterable via the + /// framework's rule machinery. In a OneShot phase, applies OneShot + /// rules to each id and returns the accumulated resulting node ids + /// in order. In a Repeating phase, errors (translation is not + /// meaningful when input and output share a schema). + /// + /// The single-`Id` case works too, because `Id: IntoIterator` as a singleton iterator — so `ctx.translate(some_id)?` + /// returns a `Vec` containing whatever `some_id` translated to. /// /// Errors if this `BuildCtx` was constructed by hand (without a /// translator handle) — for example, in unit tests that don't go /// through the rule driver. - pub fn translate>(&mut self, id: I) -> Result, String> { - let id = id.into(); - match &self.translator { - Some(t) => t.translate(self.ast, self.user_ctx, id), - None => Err("translate() called on a BuildCtx without a translator handle".into()), - } - } - - /// Translate every node in an iterator with a **fresh** user context - /// (reset to `C::default()`), restoring the previous context afterwards. - /// - /// Use when descending into a subtree — a body, expression, or statement - /// list — that must not inherit any of the surrounding translation - /// context (for example an enclosing binding modifier). Accepts optional - /// (`Option`) and repeated (`Vec`) captures (both `IntoIterator`); - /// for a single `Id`, wrap it in `std::iter::once(id)`. - pub fn translate_reset>( + pub fn translate>( &mut self, ids: impl IntoIterator, - ) -> Result, String> - where - C: Default, - { - let saved = std::mem::take(&mut *self.user_ctx); + ) -> Result, String> { + let translator = self + .translator + .as_ref() + .ok_or("translate() called on a BuildCtx without a translator handle")?; let mut out = Vec::new(); - let mut result = Ok(()); for id in ids { - match self.translate(id) { - Ok(v) => out.extend(v), - Err(e) => { - result = Err(e); - break; - } - } + let translated = translator.translate(self.ast, self.user_ctx, id.into())?; + out.extend(translated); } - *self.user_ctx = saved; - result.map(|()| out) + Ok(out) + } + + /// Run `f` with a temporary child [`BuildCtx`] whose `user_ctx` is + /// a fresh clone of the current one, sharing everything else + /// (`ast`, `captures`, `fresh`, `source_range`, `translator`) by + /// re-borrow. Any mutations `f` makes to the child's `user_ctx` + /// are discarded when it returns — no restore needed, because the + /// mutations only ever happened on a local clone. + /// + /// Use for the rare rule that needs to translate a subtree under a + /// modified context *and then continue using its own (unmodified) + /// context afterwards*. For rules where the modified translation + /// is the last use of `ctx`, mutate `ctx` in place — the + /// framework's rule-boundary save/restore cleans up on rule exit. + /// + /// Example: an outer rule that translates one child subtree with a + /// reset context, then continues with the outer context intact: + /// + /// ```ignore + /// let val = ctx.scoped(|ctx| { + /// ctx.reset(); + /// ctx.translate(val) + /// })?; + /// // `ctx` here is untouched by the reset inside the closure. + /// let other = ctx.translate(other_id)?; + /// ``` + pub fn scoped(&mut self, f: F) -> R + where + F: for<'b> FnOnce(&mut BuildCtx<'b, C>) -> R, + { + let mut child_user_ctx = self.user_ctx.clone(); + let mut child = BuildCtx { + ast: &mut *self.ast, + captures: self.captures, + fresh: self.fresh, + source_range: self.source_range, + user_ctx: &mut child_user_ctx, + translator: self.translator, + }; + f(&mut child) + // child_user_ctx dropped; the outer `self` is unaffected. } } diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index c50304e623e..516f99fc1ce 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -741,6 +741,16 @@ pub struct TranslatorHandle<'a, C> { inner: TranslatorImpl<'a, C>, } +// Manual `Copy` / `Clone` so `TranslatorHandle<'_, C>: Copy` holds +// regardless of whether `C: Copy`. `TranslatorImpl` contains only +// shared references, which are `Copy` unconditionally. +impl Copy for TranslatorHandle<'_, C> {} +impl Clone for TranslatorHandle<'_, C> { + fn clone(&self) -> Self { + *self + } +} + /// Internal phase-specific translation state. Kept private — callers /// interact with [`TranslatorHandle`] only. enum TranslatorImpl<'a, C> { @@ -761,6 +771,16 @@ enum TranslatorImpl<'a, C> { Repeating, } +// Manual `Copy` / `Clone` so `TranslatorImpl<'_, C>: Copy` holds +// regardless of whether `C: Copy`. All variants hold only shared +// references and small `Copy` scalars. +impl Copy for TranslatorImpl<'_, C> {} +impl Clone for TranslatorImpl<'_, C> { + fn clone(&self) -> Self { + *self + } +} + impl<'a, C: Clone> TranslatorHandle<'a, C> { /// Recursively apply OneShot rules to `id` and return the resulting /// node ids. Errors in a Repeating phase (where translation is not diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index 0f4f80a8f01..04537f55c09 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -45,12 +45,29 @@ impl SwiftContext { /// /// True exactly when an enclosing binding has published its modifier into /// `outer_modifiers`. This is reliable because non-binding subtrees - /// (bodies, initializer values, ...) are translated with a reset context - /// (see `BuildCtx::translate_reset`), so a bare identifier only sees a + /// (bodies, initializer values, ...) are translated after resetting the + /// context (see `reset`), so a bare identifier only sees a /// non-empty `outer_modifiers` when it really is a binding. fn in_binding_pattern(&self) -> bool { !self.outer_modifiers.is_empty() } + + /// Clear the context fields that must not propagate into an + /// expression / statement / body subtree. + /// + /// Mirrors `Default::default()` for `SwiftContext` today, but is a + /// named method so future context fields can opt in or out of + /// clearing here per-field. + /// + /// Called before recursively translating a body / initializer + /// slot. Most rules mutate `ctx` in place — the framework's + /// rule-boundary snapshot/restore cleans up on exit. Rules that + /// need the outer context intact *after* the reset-and-translate + /// (see e.g. the `property_binding` willSet/didSet rule) wrap the + /// mutation in `ctx.scoped(...)` instead. + fn reset(&mut self) { + *self = SwiftContext::default(); + } } /// Build a freshly-created `chained_declaration` modifier node if @@ -239,7 +256,7 @@ fn translation_rules() -> Vec> { name: (identifier #{name}) type: {ty} accessor_kind: (accessor_kind "get") - body: (block stmt: {ctx.translate_reset(body)?})) + body: (block stmt: {ctx.reset(); ctx.translate(body)?})) ), // Stored property with willSet/didSet observers (initializer // optional) → a `variable_declaration` followed by one @@ -260,12 +277,20 @@ fn translation_rules() -> Vec> { observers: (willset_didset_block willset: _? @@ws didset: _? @@ds)) => {{ - // The initializer value must not inherit the binding context - // (it may contain patterns, e.g. a switch expression), so - // translate it with a reset context. The observers keep the - // context: each willSet/didSet accessor emits the binding - // modifier and resets its own body. - let val = ctx.translate_reset(val)?; + // The initializer value must not inherit the binding + // context (it may contain patterns, e.g. a switch + // expression), so translate it inside a `ctx.scoped` + // block — the block receives a temporary `ctx` whose + // `user_ctx` is a clone; mutations to it are discarded + // when the block returns, so the outer `ctx` is intact + // for the observer loop below. The observers keep the + // outer context: each willSet/didSet accessor emits + // the binding modifier and, in turn, resets the + // context for its own body. + let val = ctx.scoped(|ctx| { + ctx.reset(); + ctx.translate(val) + })?; let var_decl = tree!( (variable_declaration @@ -295,7 +320,7 @@ fn translation_rules() -> Vec> { // The enclosing `property_declaration` leads `ctx.outer_modifiers` // with the `let`/`var` binding modifier, so the auto-translated name // pattern (the LHS) becomes a binding, while the initializer value is - // translated with a reset context (see `translate_reset`). + // translated with a reset context (see `SwiftContext::reset`). rule!( (property_binding name: @pattern @@ -307,7 +332,7 @@ fn translation_rules() -> Vec> { modifier: {chained_modifier(&mut ctx)} pattern: {pattern} type: {ty} - value: {ctx.translate_reset(val)?}) // reset context: the initializer must not see the binding + value: {ctx.reset(); ctx.translate(val)?}) ), // property_declaration: flatten declarators (each may translate // to multiple nodes — variable_declaration and/or @@ -1118,7 +1143,7 @@ fn translation_rules() -> Vec> { name: {ctx.property_name.ok_or("computed_getter outside property_binding context")?} type: {ctx.property_type} accessor_kind: (accessor_kind "get") - body: (block stmt: {ctx.translate_reset(body)?})) + body: (block stmt: {ctx.reset(); ctx.translate(body)?})) ), // Computed setter with explicit parameter name. rule!( @@ -1131,7 +1156,7 @@ fn translation_rules() -> Vec> { type: {ctx.property_type} accessor_kind: (accessor_kind "set") parameter: (parameter pattern: (name_pattern identifier: (identifier #{param}))) - body: (block stmt: {ctx.translate_reset(body)?})) + body: (block stmt: {ctx.reset(); ctx.translate(body)?})) ), // Computed setter without explicit parameter name; body optional. rule!( @@ -1143,7 +1168,7 @@ fn translation_rules() -> Vec> { name: {ctx.property_name.ok_or("computed_setter outside property_binding context")?} type: {ctx.property_type} accessor_kind: (accessor_kind "set") - body: (block stmt: {ctx.translate_reset(body)?})) + body: (block stmt: {ctx.reset(); ctx.translate(body)?})) ), // Computed modify → accessor_declaration rule!( @@ -1155,7 +1180,7 @@ fn translation_rules() -> Vec> { name: {ctx.property_name.ok_or("computed_modify outside property_binding context")?} type: {ctx.property_type} accessor_kind: (accessor_kind "modify") - body: (block stmt: {ctx.translate_reset(body)?})) + body: (block stmt: {ctx.reset(); ctx.translate(body)?})) ), // willset/didset block — spread to children (only reachable as a // fallback; the outer property_binding manual rule normally @@ -1173,7 +1198,7 @@ fn translation_rules() -> Vec> { modifier: {chained_modifier(&mut ctx)} name: {ctx.property_name.ok_or("willset_clause outside property_binding context")?} accessor_kind: (accessor_kind "willSet") - body: (block stmt: {ctx.translate_reset(body)?})) + body: (block stmt: {ctx.reset(); ctx.translate(body)?})) ), // didset clause → accessor_declaration (body optional). rule!( @@ -1184,7 +1209,7 @@ fn translation_rules() -> Vec> { modifier: {chained_modifier(&mut ctx)} name: {ctx.property_name.ok_or("didset_clause outside property_binding context")?} accessor_kind: (accessor_kind "didSet") - body: (block stmt: {ctx.translate_reset(body)?})) + body: (block stmt: {ctx.reset(); ctx.translate(body)?})) ), // Preprocessor conditionals — unsupported rule!((diagnostic) => (unsupported_node)), From 9b7a9c3851e49a68fa32b4ce9c20554851551af3 Mon Sep 17 00:00:00 2001 From: Taus Date: Mon, 6 Jul 2026 12:04:44 +0000 Subject: [PATCH 14/90] yeast: Use clone-and-drop for per-rule context isolation In order to avoid having context changes bubble up through the tree (or from one sibling to another) the current framework makes a copy of the context before calling `translate` recursively, and then restores it afterwards. However, this is a bit silly -- after we're done with all of the translations, there's really no need to restore the context (as it doesn't get accessed again). So instead we change it from "save and then restore" to "clone and then drop". Each rule invocation gets its own copy of the context, and simply drops it when it's done. --- shared/yeast/src/build.rs | 5 +-- shared/yeast/src/lib.rs | 36 ++++++++++--------- .../extractor/src/languages/swift/swift.rs | 9 ++--- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/shared/yeast/src/build.rs b/shared/yeast/src/build.rs index f550947ffd4..0877da22f86 100644 --- a/shared/yeast/src/build.rs +++ b/shared/yeast/src/build.rs @@ -200,8 +200,9 @@ impl BuildCtx<'_, C> { /// Use for the rare rule that needs to translate a subtree under a /// modified context *and then continue using its own (unmodified) /// context afterwards*. For rules where the modified translation - /// is the last use of `ctx`, mutate `ctx` in place — the - /// framework's rule-boundary save/restore cleans up on rule exit. + /// is the last use of `ctx`, mutate `ctx` in place — the framework + /// invokes each rule with a private clone of the user context, so + /// mutations are discarded on rule exit anyway. /// /// Example: an outer rule that translates one child subtree with a /// reset context, then continues with the outer context intact: diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index 516f99fc1ce..2f6da2039d1 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -999,10 +999,13 @@ fn apply_repeating_rules_inner( if Some(rule_ptr) == skip_rule { continue; } - // Snapshot the user context before invoking the rule so that any - // mutations the rule makes are visible during recursive translation - // of its result, but not leaked to the parent's siblings. - let snapshot = user_ctx.clone(); + // Give each rule attempt a private clone of the user context. + // Any mutations the rule makes are visible to its transform and + // to the recursive translation of its result, but never leak + // back to the parent — the clone is simply dropped when we + // return. This is also `?`-safe: an error return drops `local` + // without touching the caller's `user_ctx`. + let mut local = user_ctx.clone(); // Repeating rules don't need a real translator: their captures // aren't auto-translated (Repeating preserves the input schema), // and `ctx.translate(id)` errors if invoked from a Repeating @@ -1010,7 +1013,7 @@ fn apply_repeating_rules_inner( let translator = TranslatorHandle { inner: TranslatorImpl::Repeating, }; - let try_result = rule.try_rule(ast, id, fresh, user_ctx, translator)?; + let try_result = rule.try_rule(ast, id, fresh, &mut local, translator)?; if let Some(result_node) = try_result { // For non-repeated rules, suppress further application of *this* // rule on the result root, so a rule whose output matches its own @@ -1022,19 +1025,17 @@ fn apply_repeating_rules_inner( results.extend(apply_repeating_rules_inner( index, ast, - user_ctx, + &mut local, node, fresh, rewrite_depth + 1, next_skip, )?); } - *user_ctx = snapshot; return Ok(results); } - // Rule didn't match; restore any speculative changes (none expected - // since try_rule only mutates on match, but be defensive). - *user_ctx = snapshot; + // Rule didn't match; `local` is dropped as we loop to the next + // rule. } // Take the parent's fields by ownership: the recursion will rewrite @@ -1116,11 +1117,13 @@ fn apply_one_shot_rules_inner( for rule in index.rules_for_kind(node_kind) { if let Some(captures) = rule.try_match(ast, id)? { - // Snapshot the user context before invoking the rule so that any - // mutations the rule (or its transitively-translated captures) - // make are visible during this rule's transform, but not leaked - // to the parent's siblings. - let snapshot = user_ctx.clone(); + // Give the rule a private clone of the user context. Any + // mutations the rule (or its transitively-translated + // captures) make are visible during this rule's transform, + // but never leak back — the clone is dropped when we + // return. `?`-safe: an error return drops `local` without + // touching the caller's `user_ctx`. + let mut local = user_ctx.clone(); // Build the translator handle the transform will use to // recursively translate captures (or, for macro-generated // rules, the auto-translate prefix uses it to translate every @@ -1133,8 +1136,7 @@ fn apply_one_shot_rules_inner( matched_root: id, }, }; - let result = rule.run_transform(ast, captures, id, fresh, user_ctx, translator)?; - *user_ctx = snapshot; + let result = rule.run_transform(ast, captures, id, fresh, &mut local, translator)?; return Ok(result); } } diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index 04537f55c09..54cce1a7fc4 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -60,10 +60,11 @@ impl SwiftContext { /// clearing here per-field. /// /// Called before recursively translating a body / initializer - /// slot. Most rules mutate `ctx` in place — the framework's - /// rule-boundary snapshot/restore cleans up on exit. Rules that - /// need the outer context intact *after* the reset-and-translate - /// (see e.g. the `property_binding` willSet/didSet rule) wrap the + /// slot. Most rules mutate `ctx` in place — the framework invokes + /// each rule with a private clone of the user context, so + /// mutations are discarded on rule exit anyway. Rules that need + /// the outer context intact *after* the reset-and-translate (see + /// e.g. the `property_binding` willSet/didSet rule) wrap the /// mutation in `ctx.scoped(...)` instead. fn reset(&mut self) { *self = SwiftContext::default(); From 33da3ef74e01c8b559b47ea22dd7cfac40b500b4 Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 8 Jul 2026 16:35:01 +0200 Subject: [PATCH 15/90] yeast: Fix typo Co-authored-by: Jeroen Ketema <93738568+jketema@users.noreply.github.com> --- shared/yeast/src/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/yeast/src/build.rs b/shared/yeast/src/build.rs index 0877da22f86..68a4c883242 100644 --- a/shared/yeast/src/build.rs +++ b/shared/yeast/src/build.rs @@ -168,7 +168,7 @@ impl BuildCtx<'_, C> { /// meaningful when input and output share a schema). /// /// The single-`Id` case works too, because `Id: IntoIterator` as a singleton iterator — so `ctx.translate(some_id)?` + /// = Id>` is a singleton iterator — so `ctx.translate(some_id)?` /// returns a `Vec` containing whatever `some_id` translated to. /// /// Errors if this `BuildCtx` was constructed by hand (without a From 1fb4f9d20850339ddf071df498f3442e9c0c4414 Mon Sep 17 00:00:00 2001 From: Taus Date: Fri, 19 Jun 2026 14:53:35 +0000 Subject: [PATCH 16/90] yeast: Move schema and YAML loader into yeast-schema crate For type checking rules, we need to be able to load schemas (so we know what to check against). However, since we can't have yeast-macros depending on yeast (where the schema-handling code currently lives) as this would introduce a circular dependency, we instead split the schema-related code into its own yeast-schema crate. --- Cargo.lock | 10 + Cargo.toml | 1 + .../tree_sitter_extractors_deps/defs.bzl | 31 + shared/yeast-schema/BUILD.bazel | 12 + shared/yeast-schema/Cargo.toml | 9 + shared/yeast-schema/src/lib.rs | 33 + shared/yeast-schema/src/node_types_yaml.rs | 762 +++++++++++++++++ shared/yeast-schema/src/schema.rs | 340 ++++++++ shared/yeast/BUILD.bazel | 4 +- shared/yeast/Cargo.toml | 1 + shared/yeast/src/lib.rs | 17 +- shared/yeast/src/node_types_yaml.rs | 773 +----------------- shared/yeast/src/schema.rs | 315 +------ 13 files changed, 1268 insertions(+), 1040 deletions(-) create mode 100644 shared/yeast-schema/BUILD.bazel create mode 100644 shared/yeast-schema/Cargo.toml create mode 100644 shared/yeast-schema/src/lib.rs create mode 100644 shared/yeast-schema/src/node_types_yaml.rs create mode 100644 shared/yeast-schema/src/schema.rs diff --git a/Cargo.lock b/Cargo.lock index 4fab55a6444..76043ec0a43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3724,6 +3724,7 @@ dependencies = [ "tree-sitter-python", "tree-sitter-ruby", "yeast-macros", + "yeast-schema", ] [[package]] @@ -3735,6 +3736,15 @@ dependencies = [ "syn", ] +[[package]] +name = "yeast-schema" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "serde_yaml", +] + [[package]] name = "yoke" version = "0.8.0" diff --git a/Cargo.toml b/Cargo.toml index 62eb2e7e920..9c15b486062 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "shared/tree-sitter-extractor", "shared/yeast", "shared/yeast-macros", + "shared/yeast-schema", "ruby/extractor", "unified/extractor", "unified/extractor/tree-sitter-swift", diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl index 11842460638..7fbdfc4bbd4 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl @@ -403,6 +403,13 @@ _NORMAL_DEPENDENCIES = { "syn": Label("@vendor_ts__syn-2.0.106//:syn"), }, }, + "shared/yeast-schema": { + _COMMON_CONDITION: { + "serde": Label("@vendor_ts__serde-1.0.228//:serde"), + "serde_json": Label("@vendor_ts__serde_json-1.0.145//:serde_json"), + "serde_yaml": Label("@vendor_ts__serde_yaml-0.9.34-deprecated//:serde_yaml"), + }, + }, "unified/extractor": { _COMMON_CONDITION: { "clap": Label("@vendor_ts__clap-4.5.48//:clap"), @@ -456,6 +463,10 @@ _NORMAL_ALIASES = { _COMMON_CONDITION: { }, }, + "shared/yeast-schema": { + _COMMON_CONDITION: { + }, + }, "unified/extractor": { _COMMON_CONDITION: { }, @@ -488,6 +499,8 @@ _NORMAL_DEV_DEPENDENCIES = { }, "shared/yeast-macros": { }, + "shared/yeast-schema": { + }, "unified/extractor": { }, "unified/extractor/tree-sitter-swift": { @@ -513,6 +526,8 @@ _NORMAL_DEV_ALIASES = { }, "shared/yeast-macros": { }, + "shared/yeast-schema": { + }, "unified/extractor": { }, "unified/extractor/tree-sitter-swift": { @@ -536,6 +551,8 @@ _PROC_MACRO_DEPENDENCIES = { }, "shared/yeast-macros": { }, + "shared/yeast-schema": { + }, "unified/extractor": { }, "unified/extractor/tree-sitter-swift": { @@ -559,6 +576,8 @@ _PROC_MACRO_ALIASES = { }, "shared/yeast-macros": { }, + "shared/yeast-schema": { + }, "unified/extractor": { }, "unified/extractor/tree-sitter-swift": { @@ -582,6 +601,8 @@ _PROC_MACRO_DEV_DEPENDENCIES = { }, "shared/yeast-macros": { }, + "shared/yeast-schema": { + }, "unified/extractor": { }, "unified/extractor/tree-sitter-swift": { @@ -607,6 +628,8 @@ _PROC_MACRO_DEV_ALIASES = { }, "shared/yeast-macros": { }, + "shared/yeast-schema": { + }, "unified/extractor": { }, "unified/extractor/tree-sitter-swift": { @@ -630,6 +653,8 @@ _BUILD_DEPENDENCIES = { }, "shared/yeast-macros": { }, + "shared/yeast-schema": { + }, "unified/extractor": { }, "unified/extractor/tree-sitter-swift": { @@ -657,6 +682,8 @@ _BUILD_ALIASES = { }, "shared/yeast-macros": { }, + "shared/yeast-schema": { + }, "unified/extractor": { }, "unified/extractor/tree-sitter-swift": { @@ -682,6 +709,8 @@ _BUILD_PROC_MACRO_DEPENDENCIES = { }, "shared/yeast-macros": { }, + "shared/yeast-schema": { + }, "unified/extractor": { }, "unified/extractor/tree-sitter-swift": { @@ -705,6 +734,8 @@ _BUILD_PROC_MACRO_ALIASES = { }, "shared/yeast-macros": { }, + "shared/yeast-schema": { + }, "unified/extractor": { }, "unified/extractor/tree-sitter-swift": { diff --git a/shared/yeast-schema/BUILD.bazel b/shared/yeast-schema/BUILD.bazel new file mode 100644 index 00000000000..85f008a1aa6 --- /dev/null +++ b/shared/yeast-schema/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_rust//rust:defs.bzl", "rust_library") +load("//misc/bazel/3rdparty/tree_sitter_extractors_deps:defs.bzl", "aliases", "all_crate_deps") + +exports_files(["Cargo.toml"]) + +rust_library( + name = "yeast-schema", + srcs = glob(["src/**/*.rs"]), + aliases = aliases(), + visibility = ["//visibility:public"], + deps = all_crate_deps(), +) diff --git a/shared/yeast-schema/Cargo.toml b/shared/yeast-schema/Cargo.toml new file mode 100644 index 00000000000..4cf534d4f0c --- /dev/null +++ b/shared/yeast-schema/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "yeast-schema" +version = "0.1.0" +edition = "2021" + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +serde_yaml = "0.9" diff --git a/shared/yeast-schema/src/lib.rs b/shared/yeast-schema/src/lib.rs new file mode 100644 index 00000000000..8e15571c355 --- /dev/null +++ b/shared/yeast-schema/src/lib.rs @@ -0,0 +1,33 @@ +//! Schema definitions and YAML/JSON node-types loaders for YEAST. +//! +//! This crate carries the parts of the YEAST framework that don't need +//! `tree-sitter`: the [`schema::Schema`] type and its associated +//! [`schema::NodeType`] / [`schema::FieldCardinality`] helpers, plus the +//! YAML and JSON conversion helpers in [`node_types_yaml`]. +//! +//! It exists so that both the runtime crate (`yeast`) and the +//! compile-time `rules!` proc macro (`yeast-macros`) can build against a +//! single source of truth without dragging tree-sitter (a heavy C-backed +//! dep) into the proc-macro toolchain. +//! +//! Tree-sitter-aware adapters — building a `Schema` from a +//! `tree_sitter::Language`, or loading a YAML schema on top of one — +//! live in `yeast::schema` and `yeast::node_types_yaml` respectively. + +pub mod node_types_yaml; +pub mod schema; + +/// Field IDs are stable `u16`s, matching tree-sitter's representation so a +/// schema built from a tree-sitter language can preserve the language's +/// existing IDs. +pub type FieldId = u16; + +/// Kind IDs are stable `u16`s. Like `FieldId`, this matches tree-sitter's +/// representation. +pub type KindId = u16; + +/// Sentinel field id used to mean "the implicit unfielded slot" (what the +/// tree-sitter docs call `children` and what YEAST surfaces in queries as +/// the bare `child:` field). Reserved to avoid clashing with real field +/// IDs allocated by `Schema::register_field`. +pub const CHILD_FIELD: u16 = u16::MAX; diff --git a/shared/yeast-schema/src/node_types_yaml.rs b/shared/yeast-schema/src/node_types_yaml.rs new file mode 100644 index 00000000000..5f6a3906f7c --- /dev/null +++ b/shared/yeast-schema/src/node_types_yaml.rs @@ -0,0 +1,762 @@ +/// Converts a YAML node-types file to the tree-sitter `node-types.json` format. +/// +/// # YAML format +/// +/// ```yaml +/// supertypes: +/// _expression: +/// - assignment +/// - binary +/// +/// named: +/// assignment: +/// left: _lhs +/// right: _expression +/// identifier: +/// +/// unnamed: +/// - "+" +/// - "end" +/// ``` +/// +/// See the crate-level docs for the full format specification. +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Write; + +use crate::CHILD_FIELD; +use serde::Deserialize; +use serde_json::json; + +/// Top-level YAML structure. +#[derive(Deserialize, Default)] +struct YamlNodeTypes { + #[serde(default)] + supertypes: BTreeMap>, + #[serde(default)] + named: BTreeMap>>, + #[serde(default)] + unnamed: Vec, +} + +/// A reference to a node type. Can be: +/// - a plain string (resolved by looking up named vs unnamed) +/// - a map `{unnamed: "name"}` to force unnamed interpretation +#[derive(Deserialize, Debug, Clone)] +#[serde(untagged)] +enum TypeRef { + Name(String), + Explicit { unnamed: String }, +} + +/// A field value: either a single type ref or a list of them. +#[derive(Deserialize, Debug, Clone)] +#[serde(untagged)] +enum TypeRefOrList { + Single(TypeRef), + List(Vec), +} + +impl TypeRefOrList { + fn into_vec(self) -> Vec { + match self { + TypeRefOrList::Single(t) => vec![t], + TypeRefOrList::List(v) => v, + } + } +} + +/// Parsed field name: base name + multiplicity markers. +struct FieldSpec { + name: Option, // None for $children + multiple: bool, + required: bool, +} + +fn parse_field_name(raw: &str) -> FieldSpec { + let is_children = + raw == "$children" || raw == "$children?" || raw == "$children*" || raw == "$children+"; + + let suffix = raw.chars().last().filter(|c| matches!(c, '?' | '*' | '+')); + + let (multiple, required) = match suffix { + Some('?') => (false, false), + Some('*') => (true, false), + Some('+') => (true, true), + _ => (false, true), // bare field name = required, single + }; + + let name = if is_children { + None + } else { + let base = raw.trim_end_matches(['?', '*', '+']); + Some(base.to_string()) + }; + + FieldSpec { + name, + multiple, + required, + } +} + +/// Resolve a TypeRef to a (type, named) pair, given the sets of known named +/// and unnamed types. +fn resolve_type_ref_pair( + type_ref: &TypeRef, + named_types: &BTreeSet, + unnamed_types: &BTreeSet, +) -> (String, bool) { + match type_ref { + TypeRef::Explicit { unnamed } => (unnamed.clone(), false), + TypeRef::Name(name) => { + let is_named = named_types.contains(name); + let is_unnamed = unnamed_types.contains(name); + if is_named && is_unnamed { + (name.clone(), true) + } else if is_unnamed { + (name.clone(), false) + } else { + (name.clone(), true) + } + } + } +} + +/// Resolve a TypeRef to a {type, named} JSON record, given the sets of known named +/// and unnamed types. +fn resolve_type_ref( + type_ref: &TypeRef, + named_types: &BTreeSet, + unnamed_types: &BTreeSet, +) -> serde_json::Value { + let (kind, named) = resolve_type_ref_pair(type_ref, named_types, unnamed_types); + json!({"type": kind, "named": named}) +} + +/// Convert YAML string to node-types JSON string. +pub fn convert(yaml_input: &str) -> Result { + let yaml: YamlNodeTypes = + serde_yaml::from_str(yaml_input).map_err(|e| format!("Failed to parse YAML: {e}"))?; + + // Build the sets of known named and unnamed types for resolution. + let mut named_types = BTreeSet::new(); + for name in yaml.supertypes.keys() { + named_types.insert(name.clone()); + } + for name in yaml.named.keys() { + named_types.insert(name.clone()); + } + let unnamed_types: BTreeSet = yaml.unnamed.iter().cloned().collect(); + + let mut output = Vec::new(); + + // 1. Supertypes + for (name, members) in &yaml.supertypes { + let subtypes: Vec<_> = members + .iter() + .map(|m| resolve_type_ref(m, &named_types, &unnamed_types)) + .collect(); + output.push(json!({ + "type": name, + "named": true, + "subtypes": subtypes, + })); + } + + // 2. Named nodes + for (name, fields_opt) in &yaml.named { + let fields_map = match fields_opt { + None => { + // Leaf token: no fields, no children, no subtypes + output.push(json!({ + "type": name, + "named": true, + "fields": {}, + })); + continue; + } + Some(m) if m.is_empty() => { + output.push(json!({ + "type": name, + "named": true, + "fields": {}, + })); + continue; + } + Some(m) => m, + }; + + let mut json_fields = serde_json::Map::new(); + let mut json_children: Option = None; + + for (raw_field_name, type_refs) in fields_map { + let spec = parse_field_name(raw_field_name); + let types: Vec<_> = type_refs + .clone() + .into_vec() + .iter() + .map(|t| resolve_type_ref(t, &named_types, &unnamed_types)) + .collect(); + + // Cloning to make the borrow checker happy + let field_info = json!({ + "multiple": spec.multiple, + "required": spec.required, + "types": types, + }); + + if spec.name.is_none() { + // $children + json_children = Some(field_info); + } else { + json_fields.insert(spec.name.unwrap(), field_info); + } + } + + let mut entry = json!({ + "type": name, + "named": true, + "fields": json_fields, + }); + + if let Some(children) = json_children { + entry + .as_object_mut() + .unwrap() + .insert("children".to_string(), children); + } + + output.push(entry); + } + + // 3. Unnamed tokens + for name in &yaml.unnamed { + output.push(json!({ + "type": name, + "named": false, + })); + } + + serde_json::to_string_pretty(&output).map_err(|e| format!("Failed to serialize JSON: {e}")) +} + +/// Apply YAML node-type definitions to a mutable Schema. +/// Registers all types, fields, and allowed types from the YAML into the +/// schema. Public so callers can layer YAML node-types onto a Schema that +/// already has fields/kinds preregistered from another source (e.g. a +/// tree-sitter language). +pub fn extend_schema_from_yaml( + schema: &mut crate::schema::Schema, + yaml_input: &str, +) -> Result<(), String> { + let yaml: YamlNodeTypes = + serde_yaml::from_str(yaml_input).map_err(|e| format!("Failed to parse YAML: {e}"))?; + apply_yaml_to_schema(&yaml, schema); + Ok(()) +} + +fn apply_yaml_to_schema( + yaml: &YamlNodeTypes, + schema: &mut crate::schema::Schema, +) { + // Register all supertypes as node kinds + for name in yaml.supertypes.keys() { + schema.register_kind(name); + } + + // Register named node kinds and their fields + for (name, fields_opt) in &yaml.named { + schema.register_kind(name); + if let Some(fields) = fields_opt { + for raw_field_name in fields.keys() { + let spec = parse_field_name(raw_field_name); + if let Some(field_name) = &spec.name { + schema.register_field(field_name); + } + } + } + } + + // Register unnamed tokens as node kinds + for name in &yaml.unnamed { + schema.register_unnamed_kind(name); + } + + let mut named_types = BTreeSet::new(); + for name in yaml.supertypes.keys() { + named_types.insert(name.clone()); + } + for name in yaml.named.keys() { + named_types.insert(name.clone()); + } + let unnamed_types: BTreeSet = yaml.unnamed.iter().cloned().collect(); + + for (supertype, members) in &yaml.supertypes { + let node_types = members + .iter() + .map(|m| { + let (kind, named) = resolve_type_ref_pair(m, &named_types, &unnamed_types); + crate::schema::NodeType { kind, named } + }) + .collect(); + schema.set_supertype_members(supertype, node_types); + } + + // Register allowed field child types for type checking. + for (parent_kind, fields_opt) in &yaml.named { + let Some(fields) = fields_opt else { + continue; + }; + + for (raw_field_name, type_refs) in fields { + let spec = parse_field_name(raw_field_name); + let field_id = match &spec.name { + Some(name) => schema.register_field(name), + None => CHILD_FIELD, + }; + + let mut node_types = type_refs + .clone() + .into_vec() + .into_iter() + .map(|type_ref| { + let (kind, named) = resolve_type_ref_pair(&type_ref, &named_types, &unnamed_types); + crate::schema::NodeType { kind, named } + }) + .collect::>(); + node_types.sort_by(|a, b| a.kind.cmp(&b.kind).then(a.named.cmp(&b.named))); + node_types.dedup_by(|a, b| a.kind == b.kind && a.named == b.named); + schema.set_field_types(parent_kind, field_id, node_types); + schema.set_field_cardinality( + parent_kind, + field_id, + crate::schema::FieldCardinality { + multiple: spec.multiple, + required: spec.required, + }, + ); + } + } +} + +pub fn schema_from_yaml(yaml_input: &str) -> Result { + let mut schema = crate::schema::Schema::new(); + extend_schema_from_yaml(&mut schema, yaml_input)?; + Ok(schema) +} + +// --------------------------------------------------------------------------- +// JSON → YAML conversion +// --------------------------------------------------------------------------- + +/// JSON node-types structures (mirrors tree-sitter's format). +#[derive(Deserialize)] +struct JsonNodeInfo { + #[serde(rename = "type")] + kind: String, + named: bool, + #[serde(default)] + fields: BTreeMap, + children: Option, + #[serde(default)] + subtypes: Vec, +} + +#[derive(Deserialize)] +struct JsonNodeType { + #[serde(rename = "type")] + kind: String, + named: bool, +} + +#[derive(Deserialize)] +struct JsonFieldInfo { + multiple: bool, + required: bool, + types: Vec, +} + +/// Convert a tree-sitter node-types.json string to the YAML format. +pub fn convert_from_json(json_input: &str) -> Result { + let nodes: Vec = + serde_json::from_str(json_input).map_err(|e| format!("Failed to parse JSON: {e}"))?; + + // Collect all named and unnamed types for disambiguation decisions. + let mut all_named: BTreeSet = BTreeSet::new(); + let mut all_unnamed: BTreeSet = BTreeSet::new(); + for node in &nodes { + if node.named { + all_named.insert(node.kind.clone()); + } else { + all_unnamed.insert(node.kind.clone()); + } + } + + let mut supertypes: BTreeMap> = BTreeMap::new(); + let mut named: BTreeMap>> = BTreeMap::new(); + let mut unnamed: Vec = Vec::new(); + + for node in nodes { + if !node.named { + unnamed.push(node.kind); + continue; + } + + if !node.subtypes.is_empty() { + supertypes.insert(node.kind, node.subtypes); + continue; + } + + if node.fields.is_empty() && node.children.is_none() { + // Leaf token + named.insert(node.kind, None); + } else { + let mut fields = BTreeMap::new(); + for (name, info) in node.fields { + fields.insert(name, info); + } + if let Some(children) = node.children { + fields.insert("$children".to_string(), children); + } + named.insert(node.kind, Some(fields)); + } + } + + // Now emit YAML + let mut out = String::new(); + + // Supertypes + if !supertypes.is_empty() { + writeln!(out, "supertypes:").unwrap(); + for (name, members) in &supertypes { + writeln!(out, " {name}:").unwrap(); + for member in members { + let ref_str = format_type_ref(&member.kind, member.named, &all_named, &all_unnamed); + writeln!(out, " - {ref_str}").unwrap(); + } + } + writeln!(out).unwrap(); + } + + // Named + if !named.is_empty() { + writeln!(out, "named:").unwrap(); + for (name, fields_opt) in &named { + match fields_opt { + None => { + writeln!(out, " {name}:").unwrap(); + } + Some(fields) => { + writeln!(out, " {name}:").unwrap(); + for (field_name, info) in fields { + let suffix = field_suffix(info.multiple, info.required); + let yaml_name = if field_name == "$children" { + format!("$children{suffix}") + } else { + format!("{field_name}{suffix}") + }; + + let type_refs: Vec = info + .types + .iter() + .map(|t| format_type_ref(&t.kind, t.named, &all_named, &all_unnamed)) + .collect(); + + if type_refs.len() == 1 { + writeln!(out, " {yaml_name}: {}", type_refs[0]).unwrap(); + } else { + let list = type_refs + .iter() + .map(|s| s.as_str()) + .collect::>() + .join(", "); + writeln!(out, " {yaml_name}: [{list}]").unwrap(); + } + } + } + } + } + writeln!(out).unwrap(); + } + + // Unnamed + if !unnamed.is_empty() { + writeln!(out, "unnamed:").unwrap(); + for name in &unnamed { + writeln!(out, " - {}", force_quote(name)).unwrap(); + } + } + + Ok(out) +} + +fn field_suffix(multiple: bool, required: bool) -> &'static str { + match (multiple, required) { + (false, true) => "", + (false, false) => "?", + (true, true) => "+", + (true, false) => "*", + } +} + +/// Format a type reference for YAML output. Uses the disambiguation rule: +/// plain string if unambiguous, `{unnamed: name}` if the name exists as both +/// named and unnamed and we need the unnamed interpretation. +fn format_type_ref( + kind: &str, + named: bool, + all_named: &BTreeSet, + _all_unnamed: &BTreeSet, +) -> String { + if named { + quote_yaml(kind) + } else { + let is_also_named = all_named.contains(kind); + if is_also_named { + format!("{{unnamed: {}}}", force_quote(kind)) + } else { + force_quote(kind) + } + } +} + +/// Always wrap in double quotes. Used for unnamed node references so they're +/// visually distinct from named ones — YAML treats both forms as equivalent strings. +fn force_quote(s: &str) -> String { + format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\"")) +} + +/// Quote a YAML string value if it contains special characters or could be +/// misinterpreted. +fn quote_yaml(s: &str) -> String { + let needs_quoting = s.is_empty() + || s.contains(|c: char| { + matches!( + c, + ':' | '{' + | '}' + | '[' + | ']' + | ',' + | '&' + | '*' + | '#' + | '?' + | '|' + | '-' + | '<' + | '>' + | '=' + | '!' + | '%' + | '@' + | '`' + | '"' + | '\'' + ) + }) + || s.starts_with(' ') + || s.ends_with(' ') + || s == "true" + || s == "false" + || s == "null" + || s == "yes" + || s == "no" + || s.parse::().is_ok(); + + if needs_quoting { + format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\"")) + } else { + s.to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_basic_conversion() { + let yaml = r#" +supertypes: + _expression: + - assignment + - binary + +named: + assignment: + left: _lhs + right: _expression + binary: + left: [_expression, _simple_numeric] + operator: ["!=", "+"] + right: _expression + argument_list: + $children*: [_expression, block_argument] + identifier: + +unnamed: + - "!=" + - "+" + - "end" +"#; + + let json_str = convert(yaml).unwrap(); + let result: Vec = serde_json::from_str(&json_str).unwrap(); + + // Check supertype + let expr = &result[0]; + assert_eq!(expr["type"], "_expression"); + assert_eq!(expr["named"], true); + assert_eq!(expr["subtypes"].as_array().unwrap().len(), 2); + + // Check assignment + let assign = result.iter().find(|n| n["type"] == "assignment").unwrap(); + assert_eq!(assign["fields"]["left"]["required"], true); + assert_eq!(assign["fields"]["left"]["multiple"], false); + assert_eq!(assign["fields"]["left"]["types"][0]["type"], "_lhs"); + assert_eq!(assign["fields"]["left"]["types"][0]["named"], true); + + // Check binary.operator — "!=" and "+" should resolve to unnamed + let binary = result.iter().find(|n| n["type"] == "binary").unwrap(); + let op_types = binary["fields"]["operator"]["types"].as_array().unwrap(); + assert_eq!(op_types[0]["type"], "!="); + assert_eq!(op_types[0]["named"], false); + assert_eq!(op_types[1]["type"], "+"); + assert_eq!(op_types[1]["named"], false); + + // Check argument_list has children, not a field + let arg_list = result + .iter() + .find(|n| n["type"] == "argument_list") + .unwrap(); + assert!(arg_list.get("children").is_some()); + assert_eq!(arg_list["children"]["multiple"], true); + assert_eq!(arg_list["children"]["required"], false); + + // Check identifier is a leaf + let ident = result.iter().find(|n| n["type"] == "identifier").unwrap(); + assert_eq!(ident["fields"].as_object().unwrap().len(), 0); + + // Check unnamed tokens + let end = result.iter().find(|n| n["type"] == "end").unwrap(); + assert_eq!(end["named"], false); + } + + #[test] + fn test_explicit_unnamed_disambiguation() { + let yaml = r#" +named: + foo: + field: [{unnamed: bar}] + +unnamed: + - bar +"#; + + let json_str = convert(yaml).unwrap(); + let result: Vec = serde_json::from_str(&json_str).unwrap(); + let foo = result.iter().find(|n| n["type"] == "foo").unwrap(); + assert_eq!(foo["fields"]["field"]["types"][0]["named"], false); + } + + #[test] + fn test_field_suffixes() { + let yaml = r#" +named: + test_node: + required_single: foo + optional_single?: foo + required_multiple+: foo + optional_multiple*: foo +"#; + + let json_str = convert(yaml).unwrap(); + let result: Vec = serde_json::from_str(&json_str).unwrap(); + let node = result.iter().find(|n| n["type"] == "test_node").unwrap(); + let fields = node["fields"].as_object().unwrap(); + + assert_eq!(fields["required_single"]["required"], true); + assert_eq!(fields["required_single"]["multiple"], false); + + assert_eq!(fields["optional_single"]["required"], false); + assert_eq!(fields["optional_single"]["multiple"], false); + + assert_eq!(fields["required_multiple"]["required"], true); + assert_eq!(fields["required_multiple"]["multiple"], true); + + assert_eq!(fields["optional_multiple"]["required"], false); + assert_eq!(fields["optional_multiple"]["multiple"], true); + } + + #[test] + fn test_json_to_yaml() { + let json = r#"[ + {"type": "_expression", "named": true, "subtypes": [ + {"type": "assignment", "named": true}, + {"type": "identifier", "named": true} + ]}, + {"type": "assignment", "named": true, "fields": { + "left": {"multiple": false, "required": true, "types": [ + {"type": "_expression", "named": true} + ]}, + "right": {"multiple": false, "required": false, "types": [ + {"type": "_expression", "named": true} + ]} + }, "children": { + "multiple": true, "required": false, "types": [ + {"type": "identifier", "named": true} + ] + }}, + {"type": "identifier", "named": true, "fields": {}}, + {"type": "=", "named": false}, + {"type": "end", "named": false} + ]"#; + + let yaml = convert_from_json(json).unwrap(); + + // Verify key structures are present + assert!(yaml.contains("supertypes:")); + assert!(yaml.contains("_expression:")); + assert!(yaml.contains("named:")); + assert!(yaml.contains("assignment:")); + assert!(yaml.contains("left:")); + assert!(yaml.contains("right?:")); + assert!(yaml.contains("$children*:")); + assert!(yaml.contains("identifier:")); + assert!(yaml.contains("unnamed:")); + assert!(yaml.contains("\"=\"")); + assert!(yaml.contains("end")); + } + + #[test] + fn test_round_trip() { + let yaml_input = r#" +supertypes: + _expression: + - assignment + - identifier + +named: + assignment: + left: _expression + right?: _expression + $children*: identifier + identifier: + +unnamed: + - "=" + - end +"#; + + // YAML → JSON → YAML + let json = convert(yaml_input).unwrap(); + let yaml_output = convert_from_json(&json).unwrap(); + // YAML → JSON again (should be identical) + let json2 = convert(&yaml_output).unwrap(); + + let v1: serde_json::Value = serde_json::from_str(&json).unwrap(); + let v2: serde_json::Value = serde_json::from_str(&json2).unwrap(); + assert_eq!(v1, v2); + } +} diff --git a/shared/yeast-schema/src/schema.rs b/shared/yeast-schema/src/schema.rs new file mode 100644 index 00000000000..4acd14377a4 --- /dev/null +++ b/shared/yeast-schema/src/schema.rs @@ -0,0 +1,340 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::{FieldId, KindId, CHILD_FIELD}; + +#[derive(Clone, Debug)] +pub struct NodeType { + pub kind: String, + pub named: bool, +} + +/// Multiplicity/optionality of a field declaration. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FieldCardinality { + /// Whether the field may hold more than one child. + pub multiple: bool, + /// Whether at least one child must be present. + pub required: bool, +} + +/// A schema defining node kinds and field names for the output AST. +/// Built from a node-types.yml file, independent of any tree-sitter grammar. +/// +/// # Memory management +/// +/// `register_field`/`register_kind`/`register_unnamed_kind` (and their +/// `_with_id` siblings) use `Box::leak` to obtain `&'static str` names. This +/// is intentional: the `&'static str` names appear pervasively in `Node`, +/// `AstCursor`, query patterns, and the extractor's TRAP output, where +/// adding a lifetime would propagate widely. +/// +/// The leak is bounded by the number of distinct kind/field names registered. +/// Schemas are expected to be constructed once per process (e.g. at extractor +/// startup) and reused. Repeated construction in long-running processes will +/// leak memory unboundedly and should be avoided. +#[derive(Clone)] +pub struct Schema { + field_ids: BTreeMap, + field_names: BTreeMap, + next_field_id: FieldId, + kind_ids: BTreeMap, + unnamed_kind_ids: BTreeMap, + kind_names: BTreeMap, + next_kind_id: KindId, + field_types: BTreeMap<(String, FieldId), Vec>, + field_cardinalities: BTreeMap<(String, FieldId), FieldCardinality>, + supertypes: BTreeMap>, +} + +impl Default for Schema { + fn default() -> Self { + Self::new() + } +} + +impl Schema { + pub fn new() -> Self { + Self { + field_ids: BTreeMap::new(), + field_names: BTreeMap::new(), + next_field_id: 1, // 0 is reserved + kind_ids: BTreeMap::new(), + unnamed_kind_ids: BTreeMap::new(), + kind_names: BTreeMap::new(), + next_kind_id: 1, // 0 is reserved + field_types: BTreeMap::new(), + field_cardinalities: BTreeMap::new(), + supertypes: BTreeMap::new(), + } + } + + /// Register a field name, returning its ID. + /// If already registered, returns the existing ID. + pub fn register_field(&mut self, name: &str) -> FieldId { + if name == "child" { + return CHILD_FIELD; + } + if let Some(&id) = self.field_ids.get(name) { + return id; + } + let id = self.next_field_id; + assert!(id < CHILD_FIELD, "too many fields"); + self.next_field_id += 1; + let leaked: &'static str = Box::leak(name.to_string().into_boxed_str()); + self.field_ids.insert(name.to_string(), id); + self.field_names.insert(id, leaked); + id + } + + /// Register a field name with a specific ID, e.g. when importing IDs + /// from an external source like a tree-sitter language. If the name is + /// already registered (with any ID), nothing is changed and the + /// existing ID is returned. + pub fn register_field_with_id(&mut self, name: &str, id: FieldId) -> FieldId { + if name == "child" { + return CHILD_FIELD; + } + if let Some(&existing) = self.field_ids.get(name) { + return existing; + } + assert!(id < CHILD_FIELD, "too many fields"); + let leaked: &'static str = Box::leak(name.to_string().into_boxed_str()); + self.field_ids.insert(name.to_string(), id); + self.field_names.insert(id, leaked); + if id >= self.next_field_id { + self.next_field_id = id + 1; + } + id + } + + /// Register a named node kind name, returning its ID. + /// If already registered, returns the existing ID. + pub fn register_kind(&mut self, name: &str) -> KindId { + if let Some(&id) = self.kind_ids.get(name) { + return id; + } + let id = self.next_kind_id; + self.next_kind_id += 1; + let leaked: &'static str = Box::leak(name.to_string().into_boxed_str()); + self.kind_ids.insert(name.to_string(), id); + self.kind_names.insert(id, leaked); + id + } + + /// Register a named node kind with a specific ID, e.g. when importing + /// IDs from a tree-sitter language. If the name is already registered, + /// nothing is changed and the existing ID is returned. + pub fn register_kind_with_id(&mut self, name: &str, id: KindId) -> KindId { + if let Some(&existing) = self.kind_ids.get(name) { + return existing; + } + let leaked: &'static str = Box::leak(name.to_string().into_boxed_str()); + self.kind_ids.insert(name.to_string(), id); + self.kind_names.insert(id, leaked); + if id >= self.next_kind_id { + self.next_kind_id = id + 1; + } + id + } + + /// Register an unnamed token kind (e.g. `"="`, `"end"`), returning its ID. + /// If already registered, returns the existing ID. + pub fn register_unnamed_kind(&mut self, name: &str) -> KindId { + if let Some(&id) = self.unnamed_kind_ids.get(name) { + return id; + } + let id = self.next_kind_id; + self.next_kind_id += 1; + let leaked: &'static str = Box::leak(name.to_string().into_boxed_str()); + self.unnamed_kind_ids.insert(name.to_string(), id); + self.kind_names.insert(id, leaked); + id + } + + /// Register an unnamed token kind with a specific ID. If the name is + /// already registered as unnamed, nothing is changed and the existing + /// ID is returned. + pub fn register_unnamed_kind_with_id(&mut self, name: &str, id: KindId) -> KindId { + if let Some(&existing) = self.unnamed_kind_ids.get(name) { + return existing; + } + let leaked: &'static str = Box::leak(name.to_string().into_boxed_str()); + self.unnamed_kind_ids.insert(name.to_string(), id); + self.kind_names.insert(id, leaked); + if id >= self.next_kind_id { + self.next_kind_id = id + 1; + } + id + } + + /// Track a name for a kind ID without registering it as named or + /// unnamed. Useful when importing tree-sitter ID tables that may + /// contain duplicate IDs across the named/unnamed split. + pub fn record_kind_name(&mut self, id: KindId, name: &'static str) { + self.kind_names.entry(id).or_insert(name); + if id >= self.next_kind_id { + self.next_kind_id = id + 1; + } + } + + pub fn field_id_for_name(&self, name: &str) -> Option { + if name == "child" { + return Some(CHILD_FIELD); + } + self.field_ids.get(name).copied() + } + + pub fn field_name_for_id(&self, id: FieldId) -> Option<&'static str> { + if id == CHILD_FIELD { + return Some("child"); + } + self.field_names.get(&id).copied() + } + + pub fn id_for_node_kind(&self, kind: &str) -> Option { + self.kind_ids.get(kind).copied() + } + + pub fn id_for_unnamed_node_kind(&self, kind: &str) -> Option { + self.unnamed_kind_ids.get(kind).copied() + } + + /// Has `kind` been registered as a named kind (concrete node or + /// supertype)? + pub fn has_named_kind(&self, kind: &str) -> bool { + self.id_for_node_kind(kind).is_some() + } + + /// Has `kind` been registered as an unnamed token kind? + pub fn has_unnamed_kind(&self, kind: &str) -> bool { + self.id_for_unnamed_node_kind(kind).is_some() + } + + /// Is `field_name` declared as a field on `parent_kind`? + /// `field_name == None` checks for the implicit unfielded slot + /// (`$children`/`CHILD_FIELD`). + pub fn has_field(&self, parent_kind: &str, field_name: Option<&str>) -> bool { + let field_id = match field_name { + Some(name) => match self.field_id_for_name(name) { + Some(id) => id, + None => return false, + }, + None => CHILD_FIELD, + }; + self.field_types(parent_kind, field_id).is_some() + } + + pub fn node_kind_for_id(&self, id: KindId) -> Option<&'static str> { + self.kind_names.get(&id).copied() + } + + pub fn set_field_types( + &mut self, + parent_kind: &str, + field_id: FieldId, + node_types: Vec, + ) { + self.field_types + .insert((parent_kind.to_string(), field_id), node_types); + } + + pub fn field_types( + &self, + parent_kind: &str, + field_id: FieldId, + ) -> Option<&Vec> { + self.field_types + .get(&(parent_kind.to_string(), field_id)) + } + + pub fn set_field_cardinality( + &mut self, + parent_kind: &str, + field_id: FieldId, + cardinality: FieldCardinality, + ) { + self.field_cardinalities + .insert((parent_kind.to_string(), field_id), cardinality); + } + + /// Returns the declared cardinality for a field, if known. + pub fn field_cardinality( + &self, + parent_kind: &str, + field_id: FieldId, + ) -> Option { + self.field_cardinalities + .get(&(parent_kind.to_string(), field_id)) + .copied() + } + + /// Returns an iterator over all `(field_id, field_name)` pairs that are + /// declared as required (`required: true`) for the given `parent_kind`. + pub fn required_fields_for_kind<'a>( + &'a self, + parent_kind: &'a str, + ) -> impl Iterator)> + 'a { + self.field_cardinalities + .iter() + .filter(move |((kind, _), card)| kind == parent_kind && card.required) + .map(move |((_, field_id), _)| { + let name = self.field_name_for_id(*field_id); + (*field_id, name) + }) + } + + pub fn set_supertype_members(&mut self, supertype: &str, node_types: Vec) { + self.supertypes.insert(supertype.to_string(), node_types); + } + + /// Returns the declared members of a supertype, if known. + pub fn supertype_members(&self, supertype: &str) -> Option<&Vec> { + self.supertypes.get(supertype) + } + + /// Is `kind` a known supertype (an abstract grouping)? + pub fn is_supertype(&self, kind: &str) -> bool { + self.supertypes.contains_key(kind) + } + + fn allows_node( + &self, + node_type: &NodeType, + node_kind: &str, + node_named: bool, + active: &mut BTreeSet, + ) -> bool { + if node_type.kind == node_kind && node_type.named == node_named { + return true; + } + + if !node_type.named { + return false; + } + + let Some(members) = self.supertypes.get(&node_type.kind) else { + return false; + }; + + if !active.insert(node_type.kind.clone()) { + return false; + } + + let matched = members + .iter() + .any(|member| self.allows_node(member, node_kind, node_named, active)); + active.remove(&node_type.kind); + matched + } + + pub fn node_matches_types( + &self, + node_kind: &str, + node_named: bool, + node_types: &[NodeType], + ) -> bool { + node_types.iter().any(|node_type| { + self.allows_node(node_type, node_kind, node_named, &mut BTreeSet::new()) + }) + } +} diff --git a/shared/yeast/BUILD.bazel b/shared/yeast/BUILD.bazel index fe0b01bb87b..5217f20ec67 100644 --- a/shared/yeast/BUILD.bazel +++ b/shared/yeast/BUILD.bazel @@ -14,5 +14,7 @@ rust_library( "//shared/yeast-macros", ], visibility = ["//visibility:public"], - deps = all_crate_deps(), + deps = all_crate_deps() + [ + "//shared/yeast-schema", + ], ) diff --git a/shared/yeast/Cargo.toml b/shared/yeast/Cargo.toml index 166887c324c..518a0d1cefc 100644 --- a/shared/yeast/Cargo.toml +++ b/shared/yeast/Cargo.toml @@ -10,6 +10,7 @@ serde_json = "1.0.108" serde_yaml = "0.9" tree-sitter = ">= 0.23.0" yeast-macros = { path = "../yeast-macros" } +yeast-schema = { path = "../yeast-schema" } tree-sitter-ruby = "0.23" tree-sitter-python = "0.23" diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index 2f6da2039d1..c4b44807726 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -57,8 +57,13 @@ impl IntoIterator for Id { } /// Field and Kind ids are provided by tree-sitter -type FieldId = u16; -type KindId = u16; +type FieldId = yeast_schema::FieldId; +type KindId = yeast_schema::KindId; + +/// Sentinel field id used to mean "the implicit unfielded slot". +/// Re-exported from `yeast-schema` so the runtime and the schema share a +/// single value. +pub use yeast_schema::CHILD_FIELD; /// Trait for values that can be appended to a field's id list inside a /// `tree!`/`trees!`/`rule!` template (in `{expr}` placeholders). @@ -157,8 +162,6 @@ impl YeastSourceRange for &T { } } -pub const CHILD_FIELD: u16 = u16::MAX; - #[derive(Debug)] pub struct AstCursor<'a> { ast: &'a Ast, @@ -304,7 +307,7 @@ impl std::fmt::Debug for Ast { impl Ast { /// Construct an AST from a TS tree pub fn from_tree(language: tree_sitter::Language, tree: &tree_sitter::Tree) -> Self { - let schema = schema::Schema::from_language(&language); + let schema = schema::from_language(&language); Self::from_tree_with_schema(schema, tree, &language) } @@ -1251,7 +1254,7 @@ impl DesugaringConfig { pub fn build_schema(&self, language: &tree_sitter::Language) -> Result { match self.output_node_types_yaml { Some(yaml) => node_types_yaml::schema_from_yaml_with_language(yaml, language), - None => Ok(schema::Schema::from_language(language)), + None => Ok(schema::from_language(language)), } } } @@ -1265,7 +1268,7 @@ pub struct Runner<'a, C = ()> { impl<'a, C> Runner<'a, C> { /// Create a runner using the input grammar's schema for output. pub fn new(language: tree_sitter::Language, phases: &'a [Phase]) -> Self { - let schema = schema::Schema::from_language(&language); + let schema = schema::from_language(&language); Self { language, schema, diff --git a/shared/yeast/src/node_types_yaml.rs b/shared/yeast/src/node_types_yaml.rs index f4d9f2a1c42..7beb4bb25be 100644 --- a/shared/yeast/src/node_types_yaml.rs +++ b/shared/yeast/src/node_types_yaml.rs @@ -1,767 +1,22 @@ -/// Converts a YAML node-types file to the tree-sitter `node-types.json` format. -/// -/// # YAML format -/// -/// ```yaml -/// supertypes: -/// _expression: -/// - assignment -/// - binary -/// -/// named: -/// assignment: -/// left: _lhs -/// right: _expression -/// identifier: -/// -/// unnamed: -/// - "+" -/// - "end" -/// ``` -/// -/// See the crate-level docs for the full format specification. -use std::collections::{BTreeMap, BTreeSet}; -use std::fmt::Write; +//! YAML/JSON node-types loaders for YEAST. +//! +//! The pure YAML/JSON conversion routines live in [`yeast_schema::node_types_yaml`]. +//! This module re-exports them and adds the tree-sitter-aware adapter +//! [`schema_from_yaml_with_language`]. -use crate::CHILD_FIELD; -use serde::Deserialize; -use serde_json::json; +pub use yeast_schema::node_types_yaml::{ + convert, convert_from_json, extend_schema_from_yaml, schema_from_yaml, +}; -/// Top-level YAML structure. -#[derive(Deserialize, Default)] -struct YamlNodeTypes { - #[serde(default)] - supertypes: BTreeMap>, - #[serde(default)] - named: BTreeMap>>, - #[serde(default)] - unnamed: Vec, -} - -/// A reference to a node type. Can be: -/// - a plain string (resolved by looking up named vs unnamed) -/// - a map `{unnamed: "name"}` to force unnamed interpretation -#[derive(Deserialize, Debug, Clone)] -#[serde(untagged)] -enum TypeRef { - Name(String), - Explicit { unnamed: String }, -} - -/// A field value: either a single type ref or a list of them. -#[derive(Deserialize, Debug, Clone)] -#[serde(untagged)] -enum TypeRefOrList { - Single(TypeRef), - List(Vec), -} - -impl TypeRefOrList { - fn into_vec(self) -> Vec { - match self { - TypeRefOrList::Single(t) => vec![t], - TypeRefOrList::List(v) => v, - } - } -} - -/// Parsed field name: base name + multiplicity markers. -struct FieldSpec { - name: Option, // None for $children - multiple: bool, - required: bool, -} - -fn parse_field_name(raw: &str) -> FieldSpec { - let is_children = - raw == "$children" || raw == "$children?" || raw == "$children*" || raw == "$children+"; - - let suffix = raw.chars().last().filter(|c| matches!(c, '?' | '*' | '+')); - - let (multiple, required) = match suffix { - Some('?') => (false, false), - Some('*') => (true, false), - Some('+') => (true, true), - _ => (false, true), // bare field name = required, single - }; - - let name = if is_children { - None - } else { - let base = raw.trim_end_matches(['?', '*', '+']); - Some(base.to_string()) - }; - - FieldSpec { - name, - multiple, - required, - } -} - -/// Resolve a TypeRef to a (type, named) pair, given the sets of known named -/// and unnamed types. -fn resolve_type_ref_pair( - type_ref: &TypeRef, - named_types: &BTreeSet, - unnamed_types: &BTreeSet, -) -> (String, bool) { - match type_ref { - TypeRef::Explicit { unnamed } => (unnamed.clone(), false), - TypeRef::Name(name) => { - let is_named = named_types.contains(name); - let is_unnamed = unnamed_types.contains(name); - if is_named && is_unnamed { - (name.clone(), true) - } else if is_unnamed { - (name.clone(), false) - } else { - (name.clone(), true) - } - } - } -} - -/// Resolve a TypeRef to a {type, named} JSON record, given the sets of known named -/// and unnamed types. -fn resolve_type_ref( - type_ref: &TypeRef, - named_types: &BTreeSet, - unnamed_types: &BTreeSet, -) -> serde_json::Value { - let (kind, named) = resolve_type_ref_pair(type_ref, named_types, unnamed_types); - json!({"type": kind, "named": named}) -} - -/// Convert YAML string to node-types JSON string. -pub fn convert(yaml_input: &str) -> Result { - let yaml: YamlNodeTypes = - serde_yaml::from_str(yaml_input).map_err(|e| format!("Failed to parse YAML: {e}"))?; - - // Build the sets of known named and unnamed types for resolution. - let mut named_types = BTreeSet::new(); - for name in yaml.supertypes.keys() { - named_types.insert(name.clone()); - } - for name in yaml.named.keys() { - named_types.insert(name.clone()); - } - let unnamed_types: BTreeSet = yaml.unnamed.iter().cloned().collect(); - - let mut output = Vec::new(); - - // 1. Supertypes - for (name, members) in &yaml.supertypes { - let subtypes: Vec<_> = members - .iter() - .map(|m| resolve_type_ref(m, &named_types, &unnamed_types)) - .collect(); - output.push(json!({ - "type": name, - "named": true, - "subtypes": subtypes, - })); - } - - // 2. Named nodes - for (name, fields_opt) in &yaml.named { - let fields_map = match fields_opt { - None => { - // Leaf token: no fields, no children, no subtypes - output.push(json!({ - "type": name, - "named": true, - "fields": {}, - })); - continue; - } - Some(m) if m.is_empty() => { - output.push(json!({ - "type": name, - "named": true, - "fields": {}, - })); - continue; - } - Some(m) => m, - }; - - let mut json_fields = serde_json::Map::new(); - let mut json_children: Option = None; - - for (raw_field_name, type_refs) in fields_map { - let spec = parse_field_name(raw_field_name); - let types: Vec<_> = type_refs - .clone() - .into_vec() - .iter() - .map(|t| resolve_type_ref(t, &named_types, &unnamed_types)) - .collect(); - - // Cloning to make the borrow checker happy - let field_info = json!({ - "multiple": spec.multiple, - "required": spec.required, - "types": types, - }); - - if spec.name.is_none() { - // $children - json_children = Some(field_info); - } else { - json_fields.insert(spec.name.unwrap(), field_info); - } - } - - let mut entry = json!({ - "type": name, - "named": true, - "fields": json_fields, - }); - - if let Some(children) = json_children { - entry - .as_object_mut() - .unwrap() - .insert("children".to_string(), children); - } - - output.push(entry); - } - - // 3. Unnamed tokens - for name in &yaml.unnamed { - output.push(json!({ - "type": name, - "named": false, - })); - } - - serde_json::to_string_pretty(&output).map_err(|e| format!("Failed to serialize JSON: {e}")) -} - -/// Apply YAML node-type definitions to a mutable Schema. -/// Registers all types, fields, and allowed types from the YAML into the schema. -fn apply_yaml_to_schema(yaml: &YamlNodeTypes, schema: &mut crate::schema::Schema) { - // Register all supertypes as node kinds - for name in yaml.supertypes.keys() { - schema.register_kind(name); - } - - // Register named node kinds and their fields - for (name, fields_opt) in &yaml.named { - schema.register_kind(name); - if let Some(fields) = fields_opt { - for raw_field_name in fields.keys() { - let spec = parse_field_name(raw_field_name); - if let Some(field_name) = &spec.name { - schema.register_field(field_name); - } - } - } - } - - // Register unnamed tokens as node kinds - for name in &yaml.unnamed { - schema.register_unnamed_kind(name); - } - - let mut named_types = BTreeSet::new(); - for name in yaml.supertypes.keys() { - named_types.insert(name.clone()); - } - for name in yaml.named.keys() { - named_types.insert(name.clone()); - } - let unnamed_types: BTreeSet = yaml.unnamed.iter().cloned().collect(); - - for (supertype, members) in &yaml.supertypes { - let node_types = members - .iter() - .map(|m| { - let (kind, named) = resolve_type_ref_pair(m, &named_types, &unnamed_types); - crate::schema::NodeType { kind, named } - }) - .collect(); - schema.set_supertype_members(supertype, node_types); - } - - // Register allowed field child types for type checking. - for (parent_kind, fields_opt) in &yaml.named { - let Some(fields) = fields_opt else { - continue; - }; - - for (raw_field_name, type_refs) in fields { - let spec = parse_field_name(raw_field_name); - let field_id = match &spec.name { - Some(name) => schema.register_field(name), - None => CHILD_FIELD, - }; - - let mut node_types = type_refs - .clone() - .into_vec() - .into_iter() - .map(|type_ref| { - let (kind, named) = - resolve_type_ref_pair(&type_ref, &named_types, &unnamed_types); - crate::schema::NodeType { kind, named } - }) - .collect::>(); - node_types.sort_by(|a, b| a.kind.cmp(&b.kind).then(a.named.cmp(&b.named))); - node_types.dedup_by(|a, b| a.kind == b.kind && a.named == b.named); - schema.set_field_types(parent_kind, field_id, node_types); - schema.set_field_cardinality( - parent_kind, - field_id, - crate::schema::FieldCardinality { - multiple: spec.multiple, - required: spec.required, - }, - ); - } - } -} - -pub fn schema_from_yaml(yaml_input: &str) -> Result { - let yaml: YamlNodeTypes = - serde_yaml::from_str(yaml_input).map_err(|e| format!("Failed to parse YAML: {e}"))?; - - let mut schema = crate::schema::Schema::new(); - apply_yaml_to_schema(&yaml, &mut schema); - - Ok(schema) -} - -/// Build a Schema from a YAML string, extending a tree-sitter Language. -/// The Schema inherits all field/kind names from the Language, plus any -/// additional ones defined in the YAML. +/// Build a Schema from a YAML string, layered on top of a tree-sitter +/// `Language`. The Schema inherits all field/kind names from the language +/// (preserving the language's IDs), plus any additional ones defined in +/// the YAML. pub fn schema_from_yaml_with_language( yaml_input: &str, language: &tree_sitter::Language, ) -> Result { - let yaml: YamlNodeTypes = - serde_yaml::from_str(yaml_input).map_err(|e| format!("Failed to parse YAML: {e}"))?; - - let mut schema = crate::schema::Schema::from_language(language); - apply_yaml_to_schema(&yaml, &mut schema); - + let mut schema = crate::schema::from_language(language); + extend_schema_from_yaml(&mut schema, yaml_input)?; Ok(schema) } - -// --------------------------------------------------------------------------- -// JSON → YAML conversion -// --------------------------------------------------------------------------- - -/// JSON node-types structures (mirrors tree-sitter's format). -#[derive(Deserialize)] -struct JsonNodeInfo { - #[serde(rename = "type")] - kind: String, - named: bool, - #[serde(default)] - fields: BTreeMap, - children: Option, - #[serde(default)] - subtypes: Vec, -} - -#[derive(Deserialize)] -struct JsonNodeType { - #[serde(rename = "type")] - kind: String, - named: bool, -} - -#[derive(Deserialize)] -struct JsonFieldInfo { - multiple: bool, - required: bool, - types: Vec, -} - -/// Convert a tree-sitter node-types.json string to the YAML format. -pub fn convert_from_json(json_input: &str) -> Result { - let nodes: Vec = - serde_json::from_str(json_input).map_err(|e| format!("Failed to parse JSON: {e}"))?; - - // Collect all named and unnamed types for disambiguation decisions. - let mut all_named: BTreeSet = BTreeSet::new(); - let mut all_unnamed: BTreeSet = BTreeSet::new(); - for node in &nodes { - if node.named { - all_named.insert(node.kind.clone()); - } else { - all_unnamed.insert(node.kind.clone()); - } - } - - let mut supertypes: BTreeMap> = BTreeMap::new(); - let mut named: BTreeMap>> = BTreeMap::new(); - let mut unnamed: Vec = Vec::new(); - - for node in nodes { - if !node.named { - unnamed.push(node.kind); - continue; - } - - if !node.subtypes.is_empty() { - supertypes.insert(node.kind, node.subtypes); - continue; - } - - if node.fields.is_empty() && node.children.is_none() { - // Leaf token - named.insert(node.kind, None); - } else { - let mut fields = BTreeMap::new(); - for (name, info) in node.fields { - fields.insert(name, info); - } - if let Some(children) = node.children { - fields.insert("$children".to_string(), children); - } - named.insert(node.kind, Some(fields)); - } - } - - // Now emit YAML - let mut out = String::new(); - - // Supertypes - if !supertypes.is_empty() { - writeln!(out, "supertypes:").unwrap(); - for (name, members) in &supertypes { - writeln!(out, " {name}:").unwrap(); - for member in members { - let ref_str = format_type_ref(&member.kind, member.named, &all_named, &all_unnamed); - writeln!(out, " - {ref_str}").unwrap(); - } - } - writeln!(out).unwrap(); - } - - // Named - if !named.is_empty() { - writeln!(out, "named:").unwrap(); - for (name, fields_opt) in &named { - match fields_opt { - None => { - writeln!(out, " {name}:").unwrap(); - } - Some(fields) => { - writeln!(out, " {name}:").unwrap(); - for (field_name, info) in fields { - let suffix = field_suffix(info.multiple, info.required); - let yaml_name = if field_name == "$children" { - format!("$children{suffix}") - } else { - format!("{field_name}{suffix}") - }; - - let type_refs: Vec = info - .types - .iter() - .map(|t| format_type_ref(&t.kind, t.named, &all_named, &all_unnamed)) - .collect(); - - if type_refs.len() == 1 { - writeln!(out, " {yaml_name}: {}", type_refs[0]).unwrap(); - } else { - let list = type_refs - .iter() - .map(|s| s.as_str()) - .collect::>() - .join(", "); - writeln!(out, " {yaml_name}: [{list}]").unwrap(); - } - } - } - } - } - writeln!(out).unwrap(); - } - - // Unnamed - if !unnamed.is_empty() { - writeln!(out, "unnamed:").unwrap(); - for name in &unnamed { - writeln!(out, " - {}", force_quote(name)).unwrap(); - } - } - - Ok(out) -} - -fn field_suffix(multiple: bool, required: bool) -> &'static str { - match (multiple, required) { - (false, true) => "", - (false, false) => "?", - (true, true) => "+", - (true, false) => "*", - } -} - -/// Format a type reference for YAML output. Uses the disambiguation rule: -/// plain string if unambiguous, `{unnamed: name}` if the name exists as both -/// named and unnamed and we need the unnamed interpretation. -fn format_type_ref( - kind: &str, - named: bool, - all_named: &BTreeSet, - _all_unnamed: &BTreeSet, -) -> String { - if named { - quote_yaml(kind) - } else { - let is_also_named = all_named.contains(kind); - if is_also_named { - format!("{{unnamed: {}}}", force_quote(kind)) - } else { - force_quote(kind) - } - } -} - -/// Always wrap in double quotes. Used for unnamed node references so they're -/// visually distinct from named ones — YAML treats both forms as equivalent strings. -fn force_quote(s: &str) -> String { - format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\"")) -} - -/// Quote a YAML string value if it contains special characters or could be -/// misinterpreted. -fn quote_yaml(s: &str) -> String { - let needs_quoting = s.is_empty() - || s.contains(|c: char| { - matches!( - c, - ':' | '{' - | '}' - | '[' - | ']' - | ',' - | '&' - | '*' - | '#' - | '?' - | '|' - | '-' - | '<' - | '>' - | '=' - | '!' - | '%' - | '@' - | '`' - | '"' - | '\'' - ) - }) - || s.starts_with(' ') - || s.ends_with(' ') - || s == "true" - || s == "false" - || s == "null" - || s == "yes" - || s == "no" - || s.parse::().is_ok(); - - if needs_quoting { - format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\"")) - } else { - s.to_string() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_basic_conversion() { - let yaml = r#" -supertypes: - _expression: - - assignment - - binary - -named: - assignment: - left: _lhs - right: _expression - binary: - left: [_expression, _simple_numeric] - operator: ["!=", "+"] - right: _expression - argument_list: - $children*: [_expression, block_argument] - identifier: - -unnamed: - - "!=" - - "+" - - "end" -"#; - - let json_str = convert(yaml).unwrap(); - let result: Vec = serde_json::from_str(&json_str).unwrap(); - - // Check supertype - let expr = &result[0]; - assert_eq!(expr["type"], "_expression"); - assert_eq!(expr["named"], true); - assert_eq!(expr["subtypes"].as_array().unwrap().len(), 2); - - // Check assignment - let assign = result.iter().find(|n| n["type"] == "assignment").unwrap(); - assert_eq!(assign["fields"]["left"]["required"], true); - assert_eq!(assign["fields"]["left"]["multiple"], false); - assert_eq!(assign["fields"]["left"]["types"][0]["type"], "_lhs"); - assert_eq!(assign["fields"]["left"]["types"][0]["named"], true); - - // Check binary.operator — "!=" and "+" should resolve to unnamed - let binary = result.iter().find(|n| n["type"] == "binary").unwrap(); - let op_types = binary["fields"]["operator"]["types"].as_array().unwrap(); - assert_eq!(op_types[0]["type"], "!="); - assert_eq!(op_types[0]["named"], false); - assert_eq!(op_types[1]["type"], "+"); - assert_eq!(op_types[1]["named"], false); - - // Check argument_list has children, not a field - let arg_list = result - .iter() - .find(|n| n["type"] == "argument_list") - .unwrap(); - assert!(arg_list.get("children").is_some()); - assert_eq!(arg_list["children"]["multiple"], true); - assert_eq!(arg_list["children"]["required"], false); - - // Check identifier is a leaf - let ident = result.iter().find(|n| n["type"] == "identifier").unwrap(); - assert_eq!(ident["fields"].as_object().unwrap().len(), 0); - - // Check unnamed tokens - let end = result.iter().find(|n| n["type"] == "end").unwrap(); - assert_eq!(end["named"], false); - } - - #[test] - fn test_explicit_unnamed_disambiguation() { - let yaml = r#" -named: - foo: - field: [{unnamed: bar}] - -unnamed: - - bar -"#; - - let json_str = convert(yaml).unwrap(); - let result: Vec = serde_json::from_str(&json_str).unwrap(); - let foo = result.iter().find(|n| n["type"] == "foo").unwrap(); - assert_eq!(foo["fields"]["field"]["types"][0]["named"], false); - } - - #[test] - fn test_field_suffixes() { - let yaml = r#" -named: - test_node: - required_single: foo - optional_single?: foo - required_multiple+: foo - optional_multiple*: foo -"#; - - let json_str = convert(yaml).unwrap(); - let result: Vec = serde_json::from_str(&json_str).unwrap(); - let node = result.iter().find(|n| n["type"] == "test_node").unwrap(); - let fields = node["fields"].as_object().unwrap(); - - assert_eq!(fields["required_single"]["required"], true); - assert_eq!(fields["required_single"]["multiple"], false); - - assert_eq!(fields["optional_single"]["required"], false); - assert_eq!(fields["optional_single"]["multiple"], false); - - assert_eq!(fields["required_multiple"]["required"], true); - assert_eq!(fields["required_multiple"]["multiple"], true); - - assert_eq!(fields["optional_multiple"]["required"], false); - assert_eq!(fields["optional_multiple"]["multiple"], true); - } - - #[test] - fn test_json_to_yaml() { - let json = r#"[ - {"type": "_expression", "named": true, "subtypes": [ - {"type": "assignment", "named": true}, - {"type": "identifier", "named": true} - ]}, - {"type": "assignment", "named": true, "fields": { - "left": {"multiple": false, "required": true, "types": [ - {"type": "_expression", "named": true} - ]}, - "right": {"multiple": false, "required": false, "types": [ - {"type": "_expression", "named": true} - ]} - }, "children": { - "multiple": true, "required": false, "types": [ - {"type": "identifier", "named": true} - ] - }}, - {"type": "identifier", "named": true, "fields": {}}, - {"type": "=", "named": false}, - {"type": "end", "named": false} - ]"#; - - let yaml = convert_from_json(json).unwrap(); - - // Verify key structures are present - assert!(yaml.contains("supertypes:")); - assert!(yaml.contains("_expression:")); - assert!(yaml.contains("named:")); - assert!(yaml.contains("assignment:")); - assert!(yaml.contains("left:")); - assert!(yaml.contains("right?:")); - assert!(yaml.contains("$children*:")); - assert!(yaml.contains("identifier:")); - assert!(yaml.contains("unnamed:")); - assert!(yaml.contains("\"=\"")); - assert!(yaml.contains("end")); - } - - #[test] - fn test_round_trip() { - let yaml_input = r#" -supertypes: - _expression: - - assignment - - identifier - -named: - assignment: - left: _expression - right?: _expression - $children*: identifier - identifier: - -unnamed: - - "=" - - end -"#; - - // YAML → JSON → YAML - let json = convert(yaml_input).unwrap(); - let yaml_output = convert_from_json(&json).unwrap(); - // YAML → JSON again (should be identical) - let json2 = convert(&yaml_output).unwrap(); - - let v1: serde_json::Value = serde_json::from_str(&json).unwrap(); - let v2: serde_json::Value = serde_json::from_str(&json2).unwrap(); - assert_eq!(v1, v2); - } -} diff --git a/shared/yeast/src/schema.rs b/shared/yeast/src/schema.rs index da13bb8b6b7..daa8ad98eb5 100644 --- a/shared/yeast/src/schema.rs +++ b/shared/yeast/src/schema.rs @@ -1,285 +1,54 @@ -use std::collections::{BTreeMap, BTreeSet}; +//! YEAST schema types. +//! +//! The schema struct itself lives in the [`yeast_schema`] crate (so it can +//! be shared with the `yeast-macros` proc-macro crate without dragging +//! tree-sitter into proc-macro compiles). This module re-exports its +//! public API and supplies the one tree-sitter-aware adapter the runtime +//! needs: [`from_language`]. -use crate::{FieldId, KindId, CHILD_FIELD}; +pub use yeast_schema::schema::{FieldCardinality, NodeType, Schema}; -#[derive(Clone, Debug)] -pub struct NodeType { - pub kind: String, - pub named: bool, -} +/// Build a [`Schema`] from a tree-sitter language, importing all its +/// known field and kind names so the resulting schema's IDs line up with +/// the language's own IDs (i.e. `field_name_for_id` agrees). +pub fn from_language(language: &tree_sitter::Language) -> Schema { + let mut schema = Schema::new(); -/// Multiplicity/optionality of a field declaration. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct FieldCardinality { - /// Whether the field may hold more than one child. - pub multiple: bool, - /// Whether at least one child must be present. - pub required: bool, -} - -/// A schema defining node kinds and field names for the output AST. -/// Built from a node-types.yml file, independent of any tree-sitter grammar. -/// -/// # Memory management -/// -/// `register_field`/`register_kind`/`register_unnamed_kind` use `Box::leak` -/// to obtain `&'static str` names. This is intentional: the `&'static str` -/// names appear pervasively in `Node`, `AstCursor`, query patterns, and the -/// extractor's TRAP output, where adding a lifetime would propagate widely. -/// -/// The leak is bounded by the number of distinct kind/field names registered. -/// Schemas are expected to be constructed once per process (e.g. at extractor -/// startup) and reused. Repeated construction in long-running processes will -/// leak memory unboundedly and should be avoided. -#[derive(Clone)] -pub struct Schema { - field_ids: BTreeMap, - field_names: BTreeMap, - next_field_id: FieldId, - kind_ids: BTreeMap, - unnamed_kind_ids: BTreeMap, - kind_names: BTreeMap, - next_kind_id: KindId, - field_types: BTreeMap<(String, FieldId), Vec>, - field_cardinalities: BTreeMap<(String, FieldId), FieldCardinality>, - supertypes: BTreeMap>, -} - -impl Default for Schema { - fn default() -> Self { - Self::new() - } -} - -impl Schema { - pub fn new() -> Self { - Self { - field_ids: BTreeMap::new(), - field_names: BTreeMap::new(), - next_field_id: 1, // 0 is reserved - kind_ids: BTreeMap::new(), - unnamed_kind_ids: BTreeMap::new(), - kind_names: BTreeMap::new(), - next_kind_id: 1, // 0 is reserved - field_types: BTreeMap::new(), - field_cardinalities: BTreeMap::new(), - supertypes: BTreeMap::new(), + // Import all field names, preserving tree-sitter's IDs. + for id in 1..=language.field_count() as u16 { + if let Some(name) = language.field_name_for_id(id) { + schema.register_field_with_id(name, id); } } - /// Create a schema from a tree-sitter language, importing all its - /// known field and kind names. - pub fn from_language(language: &tree_sitter::Language) -> Self { - let mut schema = Self::new(); - // Import all field names, preserving tree-sitter's IDs - for id in 1..=language.field_count() as u16 { - if let Some(name) = language.field_name_for_id(id) { - schema.field_ids.insert(name.to_string(), id); - schema.field_names.insert(id, name); - if id >= schema.next_field_id { - schema.next_field_id = id + 1; + // Import all node kind names, preserving tree-sitter's IDs. + // Track named and unnamed variants separately. For both, prefer the + // canonical ID returned by `id_for_node_kind`, since some languages + // have multiple IDs for the same name (e.g. the reserved error token + // at ID 0 may share a name with a real token). + for id in 0..language.node_kind_count() as u16 { + if let Some(name) = language.node_kind_for_id(id) { + if name.is_empty() { + continue; + } + let is_named = language.node_kind_is_named(id); + if is_named { + let canonical_id = language.id_for_node_kind(name, true); + if canonical_id != 0 && schema.id_for_node_kind(name).is_none() { + schema.register_kind_with_id(name, canonical_id); + } + } else { + let canonical_id = language.id_for_node_kind(name, false); + if canonical_id != 0 && schema.id_for_unnamed_node_kind(name).is_none() { + schema.register_unnamed_kind_with_id(name, canonical_id); } } + // Always track the name for any ID we encounter (so + // `node_kind_for_id` works for the literal `id` we saw, even + // when it isn't the canonical one). + schema.record_kind_name(id, name); } - // Import all node kind names, preserving tree-sitter's IDs. - // Track named and unnamed variants separately. For both named and - // unnamed kinds, use the canonical ID from id_for_node_kind, since - // some languages have multiple IDs for the same name (e.g., the - // reserved error token at ID 0 may share a name with a real token). - for id in 0..language.node_kind_count() as u16 { - if let Some(name) = language.node_kind_for_id(id) { - if !name.is_empty() { - let is_named = language.node_kind_is_named(id); - if is_named { - let canonical_id = language.id_for_node_kind(name, true); - if canonical_id != 0 && !schema.kind_ids.contains_key(name) { - schema.kind_ids.insert(name.to_string(), canonical_id); - schema.kind_names.insert(canonical_id, name); - } - } else { - let canonical_id = language.id_for_node_kind(name, false); - if canonical_id != 0 && !schema.unnamed_kind_ids.contains_key(name) { - schema - .unnamed_kind_ids - .insert(name.to_string(), canonical_id); - schema.kind_names.insert(canonical_id, name); - } - } - // Always track the name for any ID we encounter - schema.kind_names.entry(id).or_insert(name); - if id >= schema.next_kind_id { - schema.next_kind_id = id + 1; - } - } - } - } - schema } - /// Register a field name, returning its ID. - /// If already registered, returns the existing ID. - pub fn register_field(&mut self, name: &str) -> FieldId { - if name == "child" { - return CHILD_FIELD; - } - if let Some(&id) = self.field_ids.get(name) { - return id; - } - let id = self.next_field_id; - assert!(id < CHILD_FIELD, "too many fields"); - self.next_field_id += 1; - let leaked: &'static str = Box::leak(name.to_string().into_boxed_str()); - self.field_ids.insert(name.to_string(), id); - self.field_names.insert(id, leaked); - id - } - - /// Register a named node kind name, returning its ID. - /// If already registered, returns the existing ID. - pub fn register_kind(&mut self, name: &str) -> KindId { - if let Some(&id) = self.kind_ids.get(name) { - return id; - } - let id = self.next_kind_id; - self.next_kind_id += 1; - let leaked: &'static str = Box::leak(name.to_string().into_boxed_str()); - self.kind_ids.insert(name.to_string(), id); - self.kind_names.insert(id, leaked); - id - } - - /// Register an unnamed token kind (e.g. `"="`, `"end"`), returning its ID. - /// If already registered, returns the existing ID. - pub fn register_unnamed_kind(&mut self, name: &str) -> KindId { - if let Some(&id) = self.unnamed_kind_ids.get(name) { - return id; - } - let id = self.next_kind_id; - self.next_kind_id += 1; - let leaked: &'static str = Box::leak(name.to_string().into_boxed_str()); - self.unnamed_kind_ids.insert(name.to_string(), id); - self.kind_names.insert(id, leaked); - id - } - - pub fn field_id_for_name(&self, name: &str) -> Option { - if name == "child" { - return Some(CHILD_FIELD); - } - self.field_ids.get(name).copied() - } - - pub fn field_name_for_id(&self, id: FieldId) -> Option<&'static str> { - if id == CHILD_FIELD { - return Some("child"); - } - self.field_names.get(&id).copied() - } - - pub fn id_for_node_kind(&self, kind: &str) -> Option { - self.kind_ids.get(kind).copied() - } - - pub fn id_for_unnamed_node_kind(&self, kind: &str) -> Option { - self.unnamed_kind_ids.get(kind).copied() - } - - pub fn node_kind_for_id(&self, id: KindId) -> Option<&'static str> { - self.kind_names.get(&id).copied() - } - - pub fn set_field_types( - &mut self, - parent_kind: &str, - field_id: FieldId, - node_types: Vec, - ) { - self.field_types - .insert((parent_kind.to_string(), field_id), node_types); - } - - pub fn field_types(&self, parent_kind: &str, field_id: FieldId) -> Option<&Vec> { - self.field_types.get(&(parent_kind.to_string(), field_id)) - } - - pub fn set_field_cardinality( - &mut self, - parent_kind: &str, - field_id: FieldId, - cardinality: FieldCardinality, - ) { - self.field_cardinalities - .insert((parent_kind.to_string(), field_id), cardinality); - } - - /// Returns the declared cardinality for a field, if known. - pub fn field_cardinality( - &self, - parent_kind: &str, - field_id: FieldId, - ) -> Option { - self.field_cardinalities - .get(&(parent_kind.to_string(), field_id)) - .copied() - } - - /// Returns an iterator over all `(field_id, field_name)` pairs that are - /// declared as required (`required: true`) for the given `parent_kind`. - pub fn required_fields_for_kind<'a>( - &'a self, - parent_kind: &'a str, - ) -> impl Iterator)> + 'a { - self.field_cardinalities - .iter() - .filter(move |((kind, _), card)| kind == parent_kind && card.required) - .map(move |((_, field_id), _)| { - let name = self.field_name_for_id(*field_id); - (*field_id, name) - }) - } - - pub fn set_supertype_members(&mut self, supertype: &str, node_types: Vec) { - self.supertypes.insert(supertype.to_string(), node_types); - } - - fn allows_node( - &self, - node_type: &NodeType, - node_kind: &str, - node_named: bool, - active: &mut BTreeSet, - ) -> bool { - if node_type.kind == node_kind && node_type.named == node_named { - return true; - } - - if !node_type.named { - return false; - } - - let Some(members) = self.supertypes.get(&node_type.kind) else { - return false; - }; - - if !active.insert(node_type.kind.clone()) { - return false; - } - - let matched = members - .iter() - .any(|member| self.allows_node(member, node_kind, node_named, active)); - active.remove(&node_type.kind); - matched - } - - pub fn node_matches_types( - &self, - node_kind: &str, - node_named: bool, - node_types: &[NodeType], - ) -> bool { - node_types.iter().any(|node_type| { - self.allows_node(node_type, node_kind, node_named, &mut BTreeSet::new()) - }) - } + schema } From ea36f2d7f8fbfccae9d05fa5addd4a1ab4d97d06 Mon Sep 17 00:00:00 2001 From: Taus Date: Fri, 19 Jun 2026 15:31:02 +0000 Subject: [PATCH 17/90] yeast: add `rules!` macro This macro allows the easy addition of multiple rules at the same time. In addition, it also accepts an input and output schema, which eventually will be used to check the validity of the rewrite rules. --- shared/yeast-macros/src/lib.rs | 40 +++ shared/yeast-macros/src/parse.rs | 243 +++++++++++++++++++ shared/yeast/doc/yeast.md | 41 ++++ shared/yeast/src/lib.rs | 2 +- shared/yeast/tests/input-types.yml | 40 +++ shared/yeast/tests/test.rs | 120 +++++++++ unified/extractor/tests/rules_macro_smoke.rs | 25 ++ 7 files changed, 510 insertions(+), 1 deletion(-) create mode 100644 shared/yeast/tests/input-types.yml create mode 100644 unified/extractor/tests/rules_macro_smoke.rs diff --git a/shared/yeast-macros/src/lib.rs b/shared/yeast-macros/src/lib.rs index 7db97f9fb70..0df96c91d26 100644 --- a/shared/yeast-macros/src/lib.rs +++ b/shared/yeast-macros/src/lib.rs @@ -113,3 +113,43 @@ pub fn rule(input: TokenStream) -> TokenStream { Err(err) => err.to_compile_error().into(), } } + +/// Bundle a list of YEAST rewrite rules with input/output node-types +/// schema paths. Returns a `Vec`; substitutable for +/// `vec![rule!(...), ...]`. +/// +/// Each comma-separated item in the bracketed list may be: +/// +/// 1. A **bare rule body** `(query) => (template)` — the `rule!(...)` +/// wrapper is implicit. +/// 2. An explicit `rule!(...)` invocation, possibly chained as +/// `rule!(...).repeated()` or path-prefixed as `yeast::rule!(...)`. +/// 3. Any other expression returning a `Rule` (helper-function calls, +/// conditionals). +/// +/// ```ignore +/// let translation_rules: Vec = yeast::rules! { +/// input: "tree-sitter-swift/node-types.yml", +/// output: "ast_types.yml", +/// [ +/// (source_file (_)* @cs) => (top_level body: {..cs}), +/// (simple_identifier) @id => (name_expr identifier: (identifier #{id})), +/// rule!((integer_literal) @lit => (int_literal #{lit})).repeated(), +/// helper_fn(), +/// ] +/// }; +/// ``` +/// +/// Paths are resolved relative to the consuming crate's `CARGO_MANIFEST_DIR` +/// (the same convention `include_str!` uses for relative paths). The +/// resolved paths are also emitted as `include_str!` references so the +/// consuming crate gets invalidated when a schema YAML changes, prepping +/// the ground for compile-time type-checking against those schemas. +#[proc_macro] +pub fn rules(input: TokenStream) -> TokenStream { + let input2: TokenStream2 = input.into(); + match parse::parse_rules_top(input2) { + Ok(output) => output.into(), + Err(err) => err.to_compile_error().into(), + } +} diff --git a/shared/yeast-macros/src/parse.rs b/shared/yeast-macros/src/parse.rs index 2ab6236fdac..01070c74bbc 100644 --- a/shared/yeast-macros/src/parse.rs +++ b/shared/yeast-macros/src/parse.rs @@ -897,6 +897,219 @@ fn expect_repetition(tokens: &mut Tokens) -> Result { } } +// --------------------------------------------------------------------------- +// rules! parsing — bundle a list of rules with input/output schema paths. +// +// The macro accepts both bare rule bodies (`(query) => (template)`) and +// explicit `rule!(...)` invocations. The schema paths are recorded but +// not yet consumed; a later change layers compile-time type-checking on +// top, using these paths to load the input/output schemas. +// --------------------------------------------------------------------------- + +/// Parse `rules! { input: "path", output: "path", [ items, ... ] }`. +/// +/// Each item in the bracketed list can be: +/// * a **bare rule body** `(query) => (template)` — wrapped implicitly +/// in `yeast::rule! { ... }` for codegen; +/// * an explicit `rule!(...)` (or `rule!(...).repeated()`, +/// `yeast::rule!(...)`, etc.) — passed through verbatim; +/// * any other expression returning a `Rule` (helper-function calls, +/// conditionals) — passed through verbatim. +/// +/// Returns a `Vec` containing the items in order. The expansion +/// also emits `include_str!` references to the resolved schema paths so +/// Cargo treats them as inputs to the consuming crate; this validates +/// path existence at compile time and prepares the ground for later +/// schema-aware checks. +pub fn parse_rules_top(input: TokenStream) -> Result { + let mut tokens = input.into_iter().peekable(); + + let input_path = parse_named_string_arg(&mut tokens, "input")?; + expect_punct(&mut tokens, ',', "expected `,` after input path")?; + let output_path = parse_named_string_arg(&mut tokens, "output")?; + expect_punct(&mut tokens, ',', "expected `,` after output path")?; + + // Resolve paths relative to the consuming crate's CARGO_MANIFEST_DIR + // so callers can write paths like "tree-sitter-swift/node-types.yml" + // alongside their other workspace-relative includes (e.g. include_str!). + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").map_err(|_| { + syn::Error::new( + Span::call_site(), + "rules!: CARGO_MANIFEST_DIR is not set; cannot resolve schema paths", + ) + })?; + let resolve_path = |raw: &str| -> std::path::PathBuf { + let p = std::path::PathBuf::from(raw); + if p.is_absolute() { + p + } else { + std::path::PathBuf::from(&manifest_dir).join(p) + } + }; + let input_abs = resolve_path(&input_path.value); + let output_abs = resolve_path(&output_path.value); + + let list = expect_group(&mut tokens, Delimiter::Bracket)?; + if let Some(tok) = tokens.next() { + return Err(syn::Error::new_spanned( + tok, + "unexpected token after `rules!` list", + )); + } + + let items = split_top_level_commas(list.stream()); + let emitted_items: Vec = items + .into_iter() + .map(|item| { + // Bare rule body — wrap in `yeast::rule! { ... }` so the + // existing rule-construction macro handles codegen. Other + // items pass through unchanged. + if has_top_level_arrow(&item) { + quote! { yeast::rule! { #item } } + } else { + item + } + }) + .collect(); + + // Emit `include_str!` references to both schema files so Cargo + // treats them as inputs to the consuming crate's compilation. The + // `const _` bindings are unused; rustc/LLVM drop them after the + // file-input dependency edge is recorded. Absolute paths are used + // because `include_str!` resolves relative paths against the source + // file, while `rules!`'s own paths are relative to + // `CARGO_MANIFEST_DIR`. + let input_abs_str = input_abs.to_string_lossy().into_owned(); + let output_abs_str = output_abs.to_string_lossy().into_owned(); + let input_lit = proc_macro2::Literal::string(&input_abs_str); + let output_lit = proc_macro2::Literal::string(&output_abs_str); + + Ok(quote! { + { + const _: &::core::primitive::str = ::core::include_str!(#input_lit); + const _: &::core::primitive::str = ::core::include_str!(#output_lit); + vec![ #(#emitted_items),* ] + } + }) +} + +/// True iff `item` contains a `=>` operator at the top level (not nested +/// inside any group). Used to detect bare rule bodies inside `rules!`. +fn has_top_level_arrow(item: &TokenStream) -> bool { + let toks: Vec = item.clone().into_iter().collect(); + find_top_level_arrow(&toks).is_some() +} + +/// Find the index of the first token of a top-level `=>` operator (the +/// `=`), ignoring `=>` inside any group. Returns `None` if not present. +fn find_top_level_arrow(toks: &[TokenTree]) -> Option { + let mut i = 0; + while i + 1 < toks.len() { + if let (TokenTree::Punct(p1), TokenTree::Punct(p2)) = (&toks[i], &toks[i + 1]) { + if p1.as_char() == '=' + && p1.spacing() == proc_macro2::Spacing::Joint + && p2.as_char() == '>' + { + return Some(i); + } + } + i += 1; + } + None +} + +/// A string literal argument named `expected_name` parsed from `name: "value"`. +struct NamedString { + value: String, + #[allow(dead_code)] + span: Span, +} + +fn parse_named_string_arg(tokens: &mut Tokens, expected_name: &str) -> Result { + let name = expect_ident( + tokens, + &format!("expected `{expected_name}:` argument"), + )?; + if name != expected_name { + return Err(syn::Error::new_spanned( + name, + format!("expected `{expected_name}:` argument"), + )); + } + expect_punct( + tokens, + ':', + &format!("expected `:` after `{expected_name}`"), + )?; + let lit = expect_literal(tokens)?; + let span = lit.span(); + let value = string_literal_value(&lit).ok_or_else(|| { + syn::Error::new( + span, + format!("`{expected_name}` must be a string literal path"), + ) + })?; + Ok(NamedString { value, span }) +} + +/// Read a literal as a plain Rust string, stripping the surrounding quotes +/// and unescaping. Falls back to `None` if the literal isn't a string. +fn string_literal_value(lit: &Literal) -> Option { + let raw = lit.to_string(); + let bytes = raw.as_bytes(); + // Match plain `"..."` literals; reject byte strings, raw strings (for + // simplicity), char literals, numbers, etc. + if bytes.first() != Some(&b'"') || bytes.last() != Some(&b'"') { + return None; + } + let mut out = String::with_capacity(raw.len()); + let mut chars = raw[1..raw.len() - 1].chars(); + while let Some(c) = chars.next() { + if c != '\\' { + out.push(c); + continue; + } + match chars.next()? { + 'n' => out.push('\n'), + 't' => out.push('\t'), + 'r' => out.push('\r'), + '\\' => out.push('\\'), + '\'' => out.push('\''), + '"' => out.push('"'), + '0' => out.push('\0'), + other => { + // Unknown escape — give up rather than silently mis-parse. + out.push('\\'); + out.push(other); + } + } + } + Some(out) +} + +/// Split a token stream into top-level comma-separated items. Commas inside +/// any group token (parens, brackets, braces) are ignored so that things +/// like `rule!(a, b)` aren't accidentally split. +fn split_top_level_commas(stream: TokenStream) -> Vec { + let mut items = Vec::new(); + let mut current: Vec = Vec::new(); + for tt in stream { + if let TokenTree::Punct(p) = &tt { + if p.as_char() == ',' && p.spacing() == proc_macro2::Spacing::Alone { + if !current.is_empty() { + items.push(current.drain(..).collect()); + } + continue; + } + } + current.push(tt); + } + if !current.is_empty() { + items.push(current.into_iter().collect()); + } + items +} + fn maybe_wrap_capture(tokens: &mut Tokens, base: TokenStream) -> Result { if peek_is_at(tokens) { let name = consume_capture_marker(tokens)?; @@ -970,3 +1183,33 @@ fn maybe_wrap_list_capture(tokens: &mut Tokens, elem: TokenStream) -> Result` is present. + let toks = quote! { (a) => (b) }; + assert!(has_top_level_arrow(&toks)); + // `rule!((a) => (b))`: the `=>` is INSIDE the macro group, so + // it's not at top level. Must NOT be detected as a bare body. + let toks = quote! { rule!((a) => (b)) }; + assert!(!has_top_level_arrow(&toks)); + // Helper call: no `=>` anywhere. + let toks = quote! { make_rule() }; + assert!(!has_top_level_arrow(&toks)); + // Match expressions inside a block: `=>` is inside braces. + let toks = quote! { { match x { 1 => 2, _ => 3 } } }; + assert!(!has_top_level_arrow(&toks)); + // Bare shorthand form: top-level `=>` followed by a bare ident. + let toks = quote! { (a) => kind }; + assert!(has_top_level_arrow(&toks)); + } +} diff --git a/shared/yeast/doc/yeast.md b/shared/yeast/doc/yeast.md index 8aa050592f6..3427597be2e 100644 --- a/shared/yeast/doc/yeast.md +++ b/shared/yeast/doc/yeast.md @@ -437,3 +437,44 @@ For the dbscheme/QL code generator, set `Language::desugar` to a `DesugaringConfig` carrying the same YAML; the generator converts it to JSON for downstream code generation. The `phases` field of the config is unused at code-generation time. + +## The `rules!` macro + +The [`rules!`] macro bundles a list of rewrite rules with the input and +output node-types schema paths. It's a drop-in replacement for the +hand-written `vec![rule!(...), rule!(...), ...]` form and accepts a +slightly looser syntax: bare rule bodies don't need an explicit +`rule!(...)` wrapper. + +```rust +let translation_rules: Vec = yeast::rules! { + input: "tree-sitter-swift/node-types.yml", + output: "ast_types.yml", + [ + (simple_identifier) @name + => + (name_expr identifier: (identifier #{name})), + + (integer_literal) @lit + => + (int_literal #{lit}), + ] +}; +``` + +Each comma-separated item in the bracketed list may be: + +- A **bare rule body** `(query) => (template)` — no `rule!(...)` wrapper. +- An explicit `rule!(...)` invocation, with optional postfix calls such + as `rule!(...).repeated()`. +- Any other expression returning a `Rule` (helper functions, etc.). + +Schema paths are resolved relative to the consuming crate's +`CARGO_MANIFEST_DIR` (the same convention `include_str!` uses for +relative paths). The resolved paths are emitted as `include_str!` +references in the expansion so the consuming crate's incremental cache +invalidates when a schema YAML changes — laying the groundwork for +schema-aware compile-time checks on the rule bodies. + +The `Vec` produced by `rules!` flows into `add_phase` exactly as +before. \ No newline at end of file diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index c4b44807726..e6759f1ddb9 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -15,7 +15,7 @@ pub mod schema; pub mod tree_builder; mod visitor; -pub use yeast_macros::{query, rule, tree, trees}; +pub use yeast_macros::{query, rule, rules, tree, trees}; use captures::Captures; use query::QueryNode; diff --git a/shared/yeast/tests/input-types.yml b/shared/yeast/tests/input-types.yml new file mode 100644 index 00000000000..6bc184ec647 --- /dev/null +++ b/shared/yeast/tests/input-types.yml @@ -0,0 +1,40 @@ +# Test input schema for yeast rules! macro tests. Covers a small subset of +# tree-sitter-ruby kinds used by the test rules. Kept deliberately small so +# the macro's compile-time loader can be exercised over a known surface. + +named: + program: + $children*: [assignment, call, identifier, for] + + assignment: + left: [identifier, left_assignment_list] + right: [identifier, integer, call] + + left_assignment_list: + $children*: identifier + + for: + pattern: [identifier, left_assignment_list] + value: in + body: do + + in: + $children: [identifier, call] + + do: + $children*: [identifier, assignment, call] + + call: + receiver: [identifier, call] + method: identifier + + identifier: + integer: + +unnamed: + - "=" + - "," + - "for" + - "in" + - "do" + - "end" diff --git a/shared/yeast/tests/test.rs b/shared/yeast/tests/test.rs index 57a9e17dbd4..4a2fb26f4fe 100644 --- a/shared/yeast/tests/test.rs +++ b/shared/yeast/tests/test.rs @@ -1322,3 +1322,123 @@ fn test_hash_brace_uses_capture_location_for_leaf() { assert_eq!(bar.start_byte(), 4); assert_eq!(bar.end_byte(), 7); } + +// ---- `rules!` macro tests (compile-time type-checking) ---- + +/// `rules!` should accept well-typed rules using the bare-rule-body +/// syntax (no inner `rule!` invocations) and produce a `Vec` that +/// behaves identically to a plain `vec![rule!(...)]` list. +#[test] +fn test_rules_macro_accepts_bare_rule_body() { + let rules: Vec = yeast::rules! { + input: "tests/input-types.yml", + output: "tests/node-types.yml", + [ + (assignment + left: (_) @left + right: (_) @right + ) + => + (assignment + left: {right} + right: {left} + ), + ] + }; + + let dump = run_and_dump("x = 1", rules); + assert_dump_eq( + &dump, + r#" + program + assignment + left: integer "1" + right: identifier "x" + "#, + ); +} + +/// The bare-rule-body shorthand `=> output_kind` should also be accepted. +#[test] +fn test_rules_macro_accepts_bare_shorthand_form() { + let rules: Vec = yeast::rules! { + input: "tests/input-types.yml", + output: "tests/node-types.yml", + [ + (assignment + left: (_) @method + right: (_) @receiver + ) + => call, + ] + }; + + let dump = run_and_dump("x = 1", rules); + assert_dump_eq( + &dump, + r#" + program + call + method: identifier "x" + receiver: integer "1" + "#, + ); +} + +/// Backwards-compat: explicit `rule!(...)` invocations inside `rules!` +/// should still type-check and behave the same as the bare form. +#[test] +fn test_rules_macro_accepts_explicit_rule_macro() { + let rules: Vec = yeast::rules! { + input: "tests/input-types.yml", + output: "tests/node-types.yml", + [ + rule!( + (assignment + left: (_) @left + right: (_) @right + ) + => + (assignment + left: {right} + right: {left} + ) + ), + ] + }; + assert_eq!(rules.len(), 1); +} + +/// `rules!` should pass through items that aren't bare rule bodies or +/// `rule!(...)` calls (e.g. helper-function calls returning a `Rule`), +/// without type-checking them. Bare and explicit rules in the same list +/// still get checked. +#[test] +fn test_rules_macro_allows_non_rule_items() { + fn extra() -> yeast::Rule { + rule!((identifier) => (identifier "extra")) + } + let rules: Vec = yeast::rules! { + input: "tests/input-types.yml", + output: "tests/node-types.yml", + [ + (integer) => (integer "checked"), + extra(), + ] + }; + assert_eq!(rules.len(), 2); +} + +/// `rules!` should accept lists that mix bare-rule and explicit-rule items. +#[test] +fn test_rules_macro_mixes_bare_and_explicit_forms() { + let rules: Vec = yeast::rules! { + input: "tests/input-types.yml", + output: "tests/node-types.yml", + [ + (integer) => (integer "I"), + rule!((identifier) => (identifier "S")), + ] + }; + assert_eq!(rules.len(), 2); +} diff --git a/unified/extractor/tests/rules_macro_smoke.rs b/unified/extractor/tests/rules_macro_smoke.rs new file mode 100644 index 00000000000..cde8ae3ca4a --- /dev/null +++ b/unified/extractor/tests/rules_macro_smoke.rs @@ -0,0 +1,25 @@ +/// Smoke test: load a few real Swift translation rules through the new +/// `yeast::rules!` macro using the bare-rule-body syntax, and confirm the +/// input + output schemas accept them. Compiles only — any type-checking +/// error surfaces as a compile-time error. +#[test] +fn rules_macro_compiles_against_real_swift_schemas() { + let _rules: Vec = yeast::rules! { + input: "tree-sitter-swift/node-types.yml", + output: "ast_types.yml", + [ + (simple_identifier) @name + => + (name_expr + identifier: (identifier #{name})), + + (integer_literal) @lit + => + (int_literal #{lit}), + + (line_string_literal) @lit + => + (string_literal #{lit}), + ] + }; +} From ee04938dedea3e1a83b4dd38ecc68c90b48abe10 Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 2 Jul 2026 12:05:23 +0000 Subject: [PATCH 18/90] yeast: Require type annotations on root-level Rust interpolations In order to facilitate static type checking of rules (and to make it easier for human readers as well), rust blocks at the root level (i.e. rules of the form `... => { ... }`) must now have a type annotation in front. All other forms are unaffected: if the right hand side of a rule is a tree, we can read the type of the root node directly. For interpolations that happen inside of such a tree, we can recover the type by looking at what field we're interpolating into, and consulting the output schema. All existing uses have been updated to have the appropriate type annotations, though these are of course not checked yet (and so could be wrong). Finally, this commit also removes the final catch-all rule `_ @node => {node}`. Because of the preceding rule that matches `(_) @node`, this rule would only ever match unnamed nodes, and I think in practice it did not match at all (at least not in our current set of tests). To give it a proper type we would have to add some notion of an "any" type, which I would like to avoid. If it _does_ turn out to be needed, we can easily add it back (ideally with a test-case that shows why it's still needed). --- shared/yeast-macros/src/parse.rs | 143 +++++++++++++++++- shared/yeast/doc/yeast.md | 65 +++++++- shared/yeast/tests/test.rs | 116 +++++++++++++- .../extractor/src/languages/swift/swift.rs | 59 ++++---- 4 files changed, 337 insertions(+), 46 deletions(-) diff --git a/shared/yeast-macros/src/parse.rs b/shared/yeast-macros/src/parse.rs index 01070c74bbc..986a9bac641 100644 --- a/shared/yeast-macros/src/parse.rs +++ b/shared/yeast-macros/src/parse.rs @@ -617,6 +617,76 @@ fn extract_captures_inner( } } +/// A rule's return-type annotation, when the body is a Rust block. Written +/// between `=>` and the block body using the schema's own vocabulary: +/// +/// ```text +/// => kind { … } // single node of that kind +/// => kind? { … } // Option (0 or 1) +/// => kind* { … } // Vec (0+) +/// ``` +/// +/// Template bodies (`=> (kind …)`) never carry an annotation — the +/// output kind is the template root. The shorthand `=> kind` (no +/// body) also carries no annotation. See `parse_rule_top` for dispatch. +#[derive(Clone, Debug)] +struct ReturnAnnotation { + kind: Ident, + multiplicity: AnnotationMultiplicity, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +enum AnnotationMultiplicity { + Single, + Optional, + Repeated, +} + +/// Peek at the token stream to decide whether the transform following +/// `=>` is a **new** annotation form (`kind [? | *] { … }`). If so, +/// consume the annotation and return it, leaving the `{ … }` body in +/// the stream for the caller to parse. Otherwise leave the stream +/// untouched and return `None`. +/// +/// The lookahead distinguishes: +/// `kind {` → annotation (single) +/// `kind? {` → annotation (optional) +/// `kind* {` → annotation (repeated) +/// `kind` → shorthand form (no `{` follows) — NOT an annotation +/// anything else → template or bare block — NOT an annotation +fn try_consume_return_annotation(tokens: &mut Tokens) -> Result> { + // Must start with an identifier (the kind name). + let mut lookahead = tokens.clone(); + let Some(TokenTree::Ident(_)) = lookahead.next() else { + return Ok(None); + }; + // Then optionally `?` or `*`, then a `{` group. + let after_suffix = match lookahead.peek() { + Some(TokenTree::Punct(p)) if p.as_char() == '?' || p.as_char() == '*' => { + lookahead.next(); + lookahead.peek() + } + other => other, + }; + if !matches!(after_suffix, Some(TokenTree::Group(g)) if g.delimiter() == Delimiter::Brace) { + return Ok(None); + } + // Commit: consume the ident + suffix from the real stream. + let kind = expect_ident(tokens, "expected output-kind name in annotation")?; + let multiplicity = match tokens.peek() { + Some(TokenTree::Punct(p)) if p.as_char() == '?' => { + tokens.next(); + AnnotationMultiplicity::Optional + } + Some(TokenTree::Punct(p)) if p.as_char() == '*' => { + tokens.next(); + AnnotationMultiplicity::Repeated + } + _ => AnnotationMultiplicity::Single, + }; + Ok(Some(ReturnAnnotation { kind, multiplicity })) +} + /// Parse `rule!( query => transform )`. pub fn parse_rule_top(input: TokenStream) -> Result { let mut tokens = input.into_iter().peekable(); @@ -688,8 +758,52 @@ pub fn parse_rule_top(input: TokenStream) -> Result { }) .collect(); - // Parse transform: either shorthand `=> kind_name` or full `=> (template ...)` - let transform_body = if peek_is_field(&mut tokens) && { + // Parse transform: the token(s) after `=>` fall into one of three + // shapes, dispatched in order: + // + // 1. `kind [? | *] { rust_body }` — annotated Rust body (NEW). + // Static-analysis-ready: the annotation declares the output + // kind and multiplicity in the schema's own vocabulary. + // 2. `kind` alone — shorthand: emit `(kind field: {@cap})…` from + // the query's captures. + // 3. anything else — full template form (`(kind …)` or bare + // `{ … }` splice via `parse_direct_list`). + let annotation = try_consume_return_annotation(&mut tokens)?; + + let transform_body = if let Some(annotation) = annotation { + // Annotation form: `=> kind [? | *] { rust_body }`. + let body_group = expect_group(&mut tokens, Delimiter::Brace)?; + if let Some(tok) = tokens.next() { + return Err(syn::Error::new_spanned( + tok, + "unexpected token after annotated rule body", + )); + } + let body = body_group.stream(); + // The annotation is not yet consumed by codegen — it will drive + // typed handles once the schema-driven codegen lands. For now, + // emit a self-documenting reference to the annotated kind and + // preserve today's `Vec` closure return so behavior + // is unchanged. + let kind_str = annotation.kind.to_string(); + let mult_str = match annotation.multiplicity { + AnnotationMultiplicity::Single => "single", + AnnotationMultiplicity::Optional => "optional", + AnnotationMultiplicity::Repeated => "repeated", + }; + let _ = (kind_str, mult_str); // silence unused warnings until wired + + // For now, adapt the user's typed return value to the framework's + // `Vec` closure result. This uses `IntoFieldIds`, which + // already accepts a bare `Id`, an iterable of ids, or `Option` + // — matching the three annotation multiplicities. + quote! { + let __value = { #body }; + let mut __ids: Vec = Vec::new(); + yeast::IntoFieldIds::extend_into(__value, &mut __ids); + __ids + } + } else if peek_is_field(&mut tokens) && { // Shorthand form: bare identifier = output node kind. // Auto-generate template from captures. let mut lookahead = tokens.clone(); @@ -749,6 +863,26 @@ pub fn parse_rule_top(input: TokenStream) -> Result { vec![__id] } } else { + // Reject bare `{ ... }` transforms — they used to be accepted + // as either a Rust body producing a `Vec` or a template + // consisting of a single `{cap}` splice. Both patterns lost + // static-analysis information (no visible output kind), so we + // now require rules with block bodies to use the annotation + // form `=> kind [? | *] { ... }`. Templates must start with a + // parenthesized node (e.g. `(if_expr ...)`). + if let Some(TokenTree::Group(g)) = tokens.peek() { + if g.delimiter() == Delimiter::Brace { + let span = g.span(); + return Err(syn::Error::new( + span, + "bare `{...}` rule bodies are no longer accepted; \ + use the annotation form `=> kind [? | *] { ... }` \ + (where the kind names the output node's schema kind, \ + optionally suffixed with `?` or `*` for multiplicity)", + )); + } + } + // Full template form let transform_items = parse_direct_list(&mut tokens, &ctx_ident)?; @@ -1026,10 +1160,7 @@ struct NamedString { } fn parse_named_string_arg(tokens: &mut Tokens, expected_name: &str) -> Result { - let name = expect_ident( - tokens, - &format!("expected `{expected_name}:` argument"), - )?; + let name = expect_ident(tokens, &format!("expected `{expected_name}:` argument"))?; if name != expected_name { return Err(syn::Error::new_spanned( name, diff --git a/shared/yeast/doc/yeast.md b/shared/yeast/doc/yeast.md index 3427597be2e..90edb510c1a 100644 --- a/shared/yeast/doc/yeast.md +++ b/shared/yeast/doc/yeast.md @@ -312,13 +312,15 @@ already conforms to the output schema. For rules that need the raw (input-schema) capture — typically to read its source text or to translate it explicitly with mutable context state between calls — use `@@name` instead. The body sees the original -input-schema `Id`: +input-schema `Id`. Because these rules always have a Rust block body, +they use the annotation form (see [the `rule!` macro +section](#the-rule-macro) for the full grammar): ```rust yeast::rule!( (assignment left: (_) @@raw_lhs right: (_) @rhs) => - { + call { // raw_lhs is untranslated: read its original source text. let text = ctx.ast.source_text(raw_lhs); // rhs is already translated by the auto-translate prefix. @@ -372,26 +374,79 @@ automatically: single captures bind as `Id`, repeated captures (after ## The `rule!` macro -`rule!` combines a query and a transform into a single declaration: +`rule!` combines a query and a transform into a single declaration. +There are three transform forms, each suited to a different level of +rule complexity: ```rust -// Full template form +// 1. Template form — a tree literal describing the output. yeast::rule!( (query_pattern field: (_) @capture) => (output_template field: {capture}) ) -// Shorthand form — captures become fields on the output node +// 2. Shorthand form — captures become fields on a bare output kind. yeast::rule!( (query_pattern field: (_) @capture) => output_kind ) + +// 3. Annotation form — a Rust block body preceded by the output kind. +yeast::rule!( + (query_pattern child: (_)+ @@children) + => + output_kind* { + // arbitrary Rust; must evaluate to a value compatible with the + // declared multiplicity (see below). + let mut result = Vec::new(); + for child in children { + result.extend(ctx.translate(child)?); + } + result + } +) ``` The shorthand `=> kind` form auto-generates the template, mapping each capture name to a field of the same name on the output node. +### Annotation form + +Rules that need imperative logic — mutating [`BuildCtx`] state per +iteration, computing intermediate values, or looping over captures — +use the annotation form. It has three shapes distinguished by a suffix +on the output-kind identifier: + +| Syntax | Body must evaluate to | Meaning | +|---------------------|-------------------------------------|--------------------------------| +| `=> kind { ... }` | a single node id of `kind` | Emit exactly one node. | +| `=> kind? { ... }` | an `Option` of a node id of `kind` | Emit 0 or 1 nodes (`None`/`Some`). | +| `=> kind* { ... }` | an iterable of node ids of `kind` | Emit 0+ nodes; flattens into the enclosing splice slot. | + +The suffix mirrors the `?` / `*` markers used elsewhere in the schema +DSL (see [`ast_types.yml`](../../../unified/extractor/ast_types.yml)): +bare identifier = required single, `?` = optional single, `*` = +repeated. + +The annotation names the schema kind of the output, giving the macro +enough information for future static analysis (e.g. computing the +static output type of translated captures at their consumer sites). + +**Bare `=> { ... }` block bodies are rejected** — every Rust-block body +must carry an annotation, so the output kind is always visible without +having to inspect the block's expression. + +### Choosing between the forms + +Prefer the simplest form that fits: + +- If the whole transform is a tree literal, use the **template form**. +- If the transform is a template whose root matches a query capture + 1:1, use the **shorthand form**. +- If the transform needs Rust logic (loops, `let` bindings, calls to + `ctx.translate`, etc.), use the **annotation form**. + ## Integration with the extractor A YEAST desugaring pass is configured with a [`DesugaringConfig`], which diff --git a/shared/yeast/tests/test.rs b/shared/yeast/tests/test.rs index 4a2fb26f4fe..3a24709dd9f 100644 --- a/shared/yeast/tests/test.rs +++ b/shared/yeast/tests/test.rs @@ -989,7 +989,7 @@ fn test_one_shot_recurses_into_returned_capture() { yeast::rule!( (assignment left: (_) @left right: (_) @right) => - {left} + identifier { left } ), yeast::rule!((identifier) => (identifier "ID")), yeast::rule!((integer) => (integer "INT")), @@ -1084,7 +1084,7 @@ fn test_raw_capture_marker() { yeast::rule!( (assignment left: (_) @@raw_lhs right: (_) @rhs) => - { + call { let text = ctx.ast.source_text(raw_lhs); tree!((call method: (identifier #{text.as_str()}) @@ -1139,7 +1139,7 @@ fn test_raw_capture_marker_explicit_translate() { yeast::rule!( (assignment left: (_) @@raw_lhs right: (_) @rhs) => - { + call { let translated_lhs = ctx.translate(raw_lhs)?; tree!((call method: {translated_lhs} @@ -1442,3 +1442,113 @@ fn test_rules_macro_mixes_bare_and_explicit_forms() { }; assert_eq!(rules.len(), 2); } + +// ---- Rule-body return-type annotation tests ---- +// +// The annotation form `=> kind [? | *] { rust_body }` is the future +// interface for Rust-bodied rules: the schema-vocabulary annotation +// declares the rule's output kind for static analysis. Today's codegen +// does NOT yet consume the annotation (it just adapts the returned +// value to `Vec` via `IntoFieldIds`); these tests only exercise +// the parser + the runtime-equivalence property. + +/// Annotation form with `*` (repeated): the rule body returns a +/// `Vec` and the annotation says the outputs are `assignment`s. +#[test] +fn test_rule_annotation_repeated() { + // Behaviourally equivalent to a two-node splice template. + let r: Rule = rule!( + (assignment left: (_) @l right: (_) @r) + => + assignment* { + let a1 = tree!((assignment left: {l} right: {r})); + let a2 = tree!((assignment left: {r} right: {l})); + vec![a1, a2] + } + ); + let ast = run_and_ast("x = 1", vec![r]); + // Just verify the run completes without a schema error; two + // top-level `assignment` nodes should appear as siblings. + let mut count = 0usize; + for id in ast.reachable_node_ids() { + if let Some(n) = ast.get_node(id) { + if n.kind_name() == "assignment" { + count += 1; + } + } + } + assert!( + count >= 2, + "expected at least two assignment nodes, got {count}" + ); +} + +/// Annotation form with `?` (optional): the rule body returns +/// `Option`. This uses `None` so the rule effectively deletes the +/// node. +#[test] +fn test_rule_annotation_optional_none() { + // Delete every `integer` (returning None yields no output nodes). + let r: Rule = rule!( + (integer) @lit + => + integer? { + let _ = lit; + None:: + } + ); + let ast = run_and_ast("42", vec![r]); + // No integer node should survive. + for id in ast.reachable_node_ids() { + if let Some(n) = ast.get_node(id) { + assert_ne!(n.kind_name(), "integer", "integer should have been deleted"); + } + } +} + +/// Annotation form (single): the rule body returns a bare `Id`. +#[test] +fn test_rule_annotation_single() { + // Identity on assignment nodes, expressed with the annotation form. + let r: Rule = rule!( + (assignment left: (_) @l right: (_) @r) + => + assignment { + tree!((assignment left: {l} right: {r})) + } + ); + let ast = run_and_ast("x = 1", vec![r]); + let mut has_assignment = false; + for id in ast.reachable_node_ids() { + if let Some(n) = ast.get_node(id) { + if n.kind_name() == "assignment" { + has_assignment = true; + } + } + } + assert!(has_assignment, "expected an assignment node"); +} + +/// The shorthand `=> kind` form (no body, no annotation) must still be +/// distinguished from the annotation form and continue to work. +#[test] +fn test_shorthand_still_works_alongside_annotation_syntax() { + let r: Rule = rule!( + (assignment left: (_) @method right: (_) @receiver) + => + call + ); + let ast = run_and_ast("x = 1", vec![r]); + let mut has_call = false; + for id in ast.reachable_node_ids() { + if let Some(n) = ast.get_node(id) { + if n.kind_name() == "call" { + has_call = true; + } + } + } + assert!( + has_call, + "shorthand form should still produce a `call` node" + ); +} diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index 54cce1a7fc4..cb3fdff8875 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -133,8 +133,8 @@ fn translation_rules() -> Vec> { ) ), // Declarations may be wrapped in local/global wrapper nodes. - rule!((global_declaration _ @inner) => {inner}), - rule!((local_declaration _ @inner) => {inner}), + rule!((global_declaration _ @inner) => stmt { inner }), + rule!((local_declaration _ @inner) => stmt { inner }), // ---- Literals ---- rule!((integer_literal) => (int_literal)), rule!((hex_literal) => (int_literal)), @@ -227,7 +227,7 @@ fn translation_rules() -> Vec> { type: _? @ty computed_value: (computed_property accessor: _+ @@accessors)) => - {{ + accessor_declaration* { ctx.property_name = Some(tree!((identifier #{pattern}))); ctx.property_type = ty; @@ -239,7 +239,7 @@ fn translation_rules() -> Vec> { result.extend(ctx.translate(acc)?); } result - }} + } ), // Computed property: shorthand getter (no explicit get/set, just // statements) → a single accessor_declaration with kind "get". @@ -277,7 +277,7 @@ fn translation_rules() -> Vec> { value: _? @@val observers: (willset_didset_block willset: _? @@ws didset: _? @@ds)) => - {{ + member* { // The initializer value must not inherit the binding // context (it may contain patterns, e.g. a switch // expression), so translate it inside a `ctx.scoped` @@ -313,7 +313,7 @@ fn translation_rules() -> Vec> { result.extend(ctx.translate(obs)?); } result - }} + } ), // property_binding with any pattern name (identifier or // destructuring). Reads outer modifiers / chained tag from `ctx`. @@ -351,7 +351,7 @@ fn translation_rules() -> Vec> { declarator: _* @@decls (modifiers)* @mods) => - {{ + member* { let binding_text = ctx.ast.source_text(binding_kind); let binding = ctx.literal("modifier", &binding_text); // The `let`/`var` binding modifier leads the declaration's @@ -365,7 +365,7 @@ fn translation_rules() -> Vec> { result.extend(ctx.translate(decl)?); } result - }} + } ), // ---- Enums ---- // enum_type_parameter → parameter (with optional name as pattern). @@ -425,7 +425,7 @@ fn translation_rules() -> Vec> { rule!( (enum_entry case: _+ @@cases (modifiers)* @mods) => - {{ + member* { ctx.outer_modifiers = mods; let mut result = Vec::new(); @@ -434,7 +434,7 @@ fn translation_rules() -> Vec> { result.extend(ctx.translate(case)?); } result - }} + } ), // Plain assignment: `x = expr` rule!( @@ -449,9 +449,9 @@ fn translation_rules() -> Vec> { (compound_assign_expr target: {target} operator: (infix_operator #{op}) value: {value}) ), // Unwrap `type` wrapper node - rule!((type name: @inner) => {inner}), + rule!((type name: @inner) => type_expr { inner }), // `directly_assignable_expression` is just a wrapper; unwrap it - rule!((directly_assignable_expression expr: @inner) => {inner}), + rule!((directly_assignable_expression expr: @inner) => expr { inner }), // Pattern with bound_identifier → name_pattern. rule!( (pattern bound_identifier: @name) @@ -463,12 +463,12 @@ fn translation_rules() -> Vec> { rule!( (pattern kind: (binding_pattern binding: (value_binding_pattern mutability: @@binding_kind) pattern: @@pattern)) => - {{ + pattern* { let binding_text = ctx.ast.source_text(binding_kind); let binding = ctx.literal("modifier", &binding_text); ctx.outer_modifiers = vec![binding]; ctx.translate(pattern)? - }} + } ), // case T.foo(x,y) pattern rule!( @@ -501,7 +501,7 @@ fn translation_rules() -> Vec> { rule!( (pattern kind: (simple_identifier) @name) => - { + pattern { if ctx.in_binding_pattern() { tree!((name_pattern identifier: (identifier #{name}))) } else { @@ -537,10 +537,10 @@ fn translation_rules() -> Vec> { rule!( (function_parameter parameter: @@p default_value: _? @def) => - {{ + parameter* { ctx.default_value = def; ctx.translate(p)? - }} + } ), // Parameter with external name and type rule!( @@ -763,7 +763,7 @@ fn translation_rules() -> Vec> { element: (pattern_element pattern: (name_pattern identifier: (identifier #{name}))))) ), // If-condition — unwrap (pass through the inner expression/pattern) - rule!((if_condition kind: @inner) => {inner}), + rule!((if_condition kind: @inner) => expr_or_pattern { inner }), // ---- Loops ---- // For-in loop with optional where-clause guard. rule!( @@ -796,7 +796,7 @@ fn translation_rules() -> Vec> { body: (block stmt: {body})) ), // Labeled statement (e.g. `outer: for ...`). Strip the trailing ':' from the label token. - rule!((labeled_statement label: (statement_label) @lbl statement: @stmt) => { + rule!((labeled_statement label: (statement_label) @lbl statement: @stmt) => labeled_stmt { let text = ctx.ast.source_text(lbl); let name = &text[..text.len() - 1]; tree!((labeled_stmt label: (identifier #{name}) stmt: {stmt})) @@ -818,7 +818,7 @@ fn translation_rules() -> Vec> { rule!((dictionary_literal_item key: @k value: @v) => (key_value_pair key: {k} value: {v})), // ---- Optionals and errors ---- // Optional chaining — unwrap the marker - rule!((optional_chain_marker expr: @inner) => {inner}), + rule!((optional_chain_marker expr: @inner) => expr { inner }), // try/try?/try! expr → unary_expr with operator "try", "try?" or "try!" rule!((try_expression (try_operator) @op expr: @inner) => (unary_expr operator: (prefix_operator #{op}) operand: {inner})), rule!((try_expression operator: (try_operator) @op expr: @inner) => (unary_expr operator: (prefix_operator #{op}) operand: {inner})), @@ -874,7 +874,7 @@ fn translation_rules() -> Vec> { rule!( (identifier part: _+ @parts) => - {member_chain(&mut ctx, parts)} + expr { member_chain(&mut ctx, parts) } ), // Scoped import declaration (for example `import struct Foo.Bar`): // flatten the identifier parts into a member_access_expr and bind the @@ -905,7 +905,7 @@ fn translation_rules() -> Vec> { // Super expression → super_expr rule!((super_expression) => (super_expr)), // Modifiers — unwrap to individual modifier children - rule!((modifiers _* @mods) => {mods}), + rule!((modifiers _* @mods) => modifier* { mods }), rule!((attribute) @m => (modifier #{m})), rule!((visibility_modifier) @m => (modifier #{m})), rule!((function_modifier) @m => (modifier #{m})), @@ -917,7 +917,7 @@ fn translation_rules() -> Vec> { rule!((inheritance_modifier) @m => (modifier #{m})), rule!((property_behavior_modifier) @m => (modifier #{m})), // Type annotations — unwrap - rule!((type_annotation type: @inner) => {inner}), + rule!((type_annotation type: @inner) => type_expr { inner }), // user_type is split into simple_user_type parts. // Keep a conservative textual fallback to avoid dropping type information. rule!((user_type) @ty => (named_type_expr name: (identifier #{ty}))), @@ -1092,7 +1092,7 @@ fn translation_rules() -> Vec> { type: _? @ty (modifiers)* @mods) => - {{ + accessor_declaration* { ctx.property_name = Some(tree!((identifier #{name}))); ctx.property_type = ty; ctx.outer_modifiers = mods; @@ -1103,7 +1103,7 @@ fn translation_rules() -> Vec> { result.extend(ctx.translate(acc)?); } result - }} + } ), // getter_specifier / setter_specifier → bodyless accessor_declaration // getter_specifier / setter_specifier → bodyless @@ -1130,7 +1130,7 @@ fn translation_rules() -> Vec> { modifier: {chained_modifier(&mut ctx)}) ), // protocol_property_requirements wrapper — should be consumed by above; fallback - rule!((protocol_property_requirements accessor: _* @accs) => {accs}), + rule!((protocol_property_requirements accessor: _* @accs) => accessor_declaration* { accs }), // Computed getter → accessor_declaration (body optional). // Reads property name/type from the outer property_binding rule // and binding/outer modifiers + chained tag from the outer @@ -1186,7 +1186,7 @@ fn translation_rules() -> Vec> { // willset/didset block — spread to children (only reachable as a // fallback; the outer property_binding manual rule normally // captures the willset/didset clauses directly). - rule!((willset_didset_block _* @clauses) => {clauses}), + rule!((willset_didset_block _* @clauses) => accessor_declaration* { clauses }), // willset clause → accessor_declaration (body optional). Reads // `ctx.property_name` set by the outer property_binding rule and // binding/outer modifiers + chained tag from the outer @@ -1220,11 +1220,6 @@ fn translation_rules() -> Vec> { => (unsupported_node) ), - rule!( - _ @node - => - {node} - ), ] } From 11afcce8b32be1ef6624e61665bc60955f8bc329 Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 2 Jul 2026 12:57:43 +0000 Subject: [PATCH 19/90] yeast: Fix bug in matching `(_)` Turns out, `(_)` would match both named and unnamed nodes, as we never checked the value of the `match_unnamed` field. This is the real reason why the final catch-all rule we removed in the last commit was superfluous -- unnamed nodes were being caught by the penultimate rule instead (and mapped to `unsupported_node`). Having fixed the bug, we now (correctly) get errors due to unmatched unnamed nodes in the input. To fix this, we change the catch-all rule to match unnamed nodes as well. This restores the previous behaviour exactly. At some point, we should find a better way to handle unnamed nodes, as it seems wasteful to map these to `unsupported_node` (since we in practice only use them for their string content). Perhaps we should not attempt to translate unnamed nodes at all? --- shared/yeast/src/query.rs | 12 +++++++++++- unified/extractor/src/languages/swift/swift.rs | 9 ++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/shared/yeast/src/query.rs b/shared/yeast/src/query.rs index bcf0f7facab..3e61ac60b2b 100644 --- a/shared/yeast/src/query.rs +++ b/shared/yeast/src/query.rs @@ -66,7 +66,17 @@ impl QueryNode { pub fn do_match(&self, ast: &Ast, node: Id, matches: &mut Captures) -> Result { match self { - QueryNode::Any { .. } => Ok(true), + QueryNode::Any { match_unnamed } => { + if *match_unnamed { + Ok(true) + } else { + // `(_)` only matches named nodes, matching tree-sitter + // semantics. Bare `_` (with `match_unnamed = true`) + // matches any node. + let n = ast.get_node(node).unwrap(); + Ok(n.is_named()) + } + } QueryNode::Node { kind, children } => { let node = ast.get_node(node).unwrap(); let target_kind = ast diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index cb3fdff8875..74809ea18e4 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -1215,8 +1215,15 @@ fn translation_rules() -> Vec> { // Preprocessor conditionals — unsupported rule!((diagnostic) => (unsupported_node)), // ---- Fallbacks ---- + // Bare `_` (rather than `(_)`) so this matches both named nodes + // and unnamed tokens. Any unnamed token that escapes the + // input-schema-specific rules (e.g. captured operators in + // `additive_expression op: @op`) has its auto-translated value + // replaced with an `unsupported_node` whose source range is + // inherited from the original token, so `#{op}` still reads the + // original text. rule!( - (_) + _ => (unsupported_node) ), From d609680083f078bdfec7f1acbed55ed9c79c27d2 Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 2 Jul 2026 21:03:59 +0000 Subject: [PATCH 20/90] yeast: Fix escaping bug in yeast-macros Happily, it turned out that there was already a library function for handling this case. --- shared/yeast-macros/src/parse.rs | 37 +++++--------------------------- 1 file changed, 5 insertions(+), 32 deletions(-) diff --git a/shared/yeast-macros/src/parse.rs b/shared/yeast-macros/src/parse.rs index 986a9bac641..01c0b574b1c 100644 --- a/shared/yeast-macros/src/parse.rs +++ b/shared/yeast-macros/src/parse.rs @@ -1183,39 +1183,12 @@ fn parse_named_string_arg(tokens: &mut Tokens, expected_name: &str) -> Result Option { - let raw = lit.to_string(); - let bytes = raw.as_bytes(); - // Match plain `"..."` literals; reject byte strings, raw strings (for - // simplicity), char literals, numbers, etc. - if bytes.first() != Some(&b'"') || bytes.last() != Some(&b'"') { - return None; - } - let mut out = String::with_capacity(raw.len()); - let mut chars = raw[1..raw.len() - 1].chars(); - while let Some(c) = chars.next() { - if c != '\\' { - out.push(c); - continue; - } - match chars.next()? { - 'n' => out.push('\n'), - 't' => out.push('\t'), - 'r' => out.push('\r'), - '\\' => out.push('\\'), - '\'' => out.push('\''), - '"' => out.push('"'), - '0' => out.push('\0'), - other => { - // Unknown escape — give up rather than silently mis-parse. - out.push('\\'); - out.push(other); - } - } - } - Some(out) + let tokens = TokenStream::from(TokenTree::Literal(lit.clone())); + syn::parse2::(tokens).ok().map(|s| s.value()) } /// Split a token stream into top-level comma-separated items. Commas inside From 36d0ceb292b0ebadd12a1f91bd511add42afcdbe Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 9 Jul 2026 14:29:47 +0000 Subject: [PATCH 21/90] update codeql documentation --- .../codeql-changelog/codeql-cli-2.26.0.rst | 177 ++++++++++++++++++ .../codeql-changelog/index.rst | 1 + 2 files changed, 178 insertions(+) create mode 100644 docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.26.0.rst diff --git a/docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.26.0.rst b/docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.26.0.rst new file mode 100644 index 00000000000..5b5746bf26c --- /dev/null +++ b/docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.26.0.rst @@ -0,0 +1,177 @@ +.. _codeql-cli-2.26.0: + +========================== +CodeQL 2.26.0 (2026-07-08) +========================== + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: none + +This is an overview of changes in the CodeQL CLI and relevant CodeQL query and library packs. For additional updates on changes to the CodeQL code scanning experience, check out the `code scanning section on the GitHub blog `__, `relevant GitHub Changelog updates `__, `changes in the CodeQL extension for Visual Studio Code `__, and the `CodeQL Action changelog `__. + +Security Coverage +----------------- + +CodeQL 2.26.0 runs a total of 497 security queries when configured with the Default suite (covering 170 CWE). The Extended suite enables an additional 131 queries (covering 32 more CWE). 1 security query has been added with this release. + +CodeQL CLI +---------- + +Improvements +~~~~~~~~~~~~ + +* Improved the performance of commands that interact with Git repositories by checking whether the :code:`git` command-line tool is available at most once per CodeQL CLI invocation. + +Query Packs +----------- + +Minor Analysis Improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Golang +"""""" + +* The query :code:`go/unhandled-writable-file-close` ("Writable file handle closed without error handling") now produces fewer false positives. A deferred call to :code:`Close` that is preceded on every execution path by a handled call to :code:`Sync` on the same file handle is no longer flagged. + +Python +"""""" + +* The :code:`py/modification-of-locals` query no longer flags modifications of a :code:`locals()` dictionary that has been passed out of the scope in which :code:`locals()` was called (for example, by passing it to another function or storing it in an instance attribute). In such cases the dictionary is used as an ordinary mapping and modifying it is meaningful, so these were false positives. The "modification has no effect" claim only applies within the scope that called :code:`locals()`, which is now the only case reported. + +Swift +""""" + +* Fixed an issue where common usage patterns for :code:`CryptoKit` weren't being recognized as hashing sinks for the :code:`swift/weak-sensitive-data-hashing` and :code:`swift/weak-password-hashing` queries. These queries may find additional results after this change. + +New Queries +~~~~~~~~~~~ + +JavaScript/TypeScript +""""""""""""""""""""" + +* Added a new query, :code:`js/system-prompt-injection`, to detect cases where untrusted, user-provided values flow into the system prompt of an AI model, allowing an attacker to manipulate the model's behavior. +* Added a new experimental query, :code:`javascript/ssrf-ipv6-transition-incomplete-guard`, to detect SSRF host-validation guards that reject private IPv4 ranges but fail to unwrap IPv6-transition forms (IPv4-mapped :code:`::ffff:`, NAT64 :code:`64:ff9b::`, 6to4 :code:`2002::`), allowing the guard to be bypassed by wrapping an internal IPv4 address in a transition literal. + +Query Metadata Changes +~~~~~~~~~~~~~~~~~~~~~~ + +GitHub Actions +"""""""""""""" + +* The name, description, and alert message of :code:`actions/untrusted-checkout/medium` have been corrected to describe a non-privileged context. + +Language Libraries +------------------ + +Bug Fixes +~~~~~~~~~ + +GitHub Actions +"""""""""""""" + +* GitHub Actions queries now better account for permission checks on jobs that call reusable workflows. +* The query :code:`actions/pr-on-self-hosted-runner` was updated to the latest standard runner labels reducing false positive results. + +Breaking Changes +~~~~~~~~~~~~~~~~ + +C/C++ +""""" + +* Removed the deprecated :code:`overrideReturnsNull` predicate from :code:`Options.qll`. Use :code:`CustomOptions.overrideReturnsNull` instead. +* Removed the deprecated :code:`returnsNull` predicate from :code:`Options.qll`. Use :code:`CustomOptions.returnsNull` instead. +* Removed the deprecated :code:`exits` predicate from :code:`Options.qll`. Use :code:`CustomOptions.exits` instead. +* Removed the deprecated :code:`exprExits` predicate from :code:`Options.qll`. Use :code:`CustomOptions.exprExits` instead. +* Removed the deprecated :code:`alwaysCheckReturnValue` predicate from :code:`Options.qll`. Use :code:`CustomOptions.alwaysCheckReturnValue` instead. +* Removed the deprecated :code:`okToIgnoreReturnValue` predicate from :code:`Options.qll`. Use :code:`CustomOptions.okToIgnoreReturnValue` instead. +* Removed the deprecated :code:`semmle.code.cpp.Member`. Import :code:`semmle.code.cpp.Element` and/or :code:`semmle.code.cpp.Type` directly. +* Removed the deprecated :code:`UnknownDefaultLocation` class. Use :code:`UnknownLocation` instead. +* Removed the deprecated :code:`UnknownExprLocation` class. Use :code:`UnknownLocation` instead. +* Removed the deprecated :code:`UnknownStmtLocation` class. Use :code:`UnknownLocation` instead. +* Removed the deprecated :code:`TemplateParameter` class. Use :code:`TypeTemplateParameter` instead. +* Support for class resolution across link targets has been removed for databases which were created with CodeQL versions before 1.23.0. + +C# +"" + +* Renamed types related to *operation* expressions. The QL classes :code:`BinaryArithmeticOperation`, :code:`BinaryBitwiseOperation`, and :code:`BinaryLogicalOperation` now include compound assignments; for example, :code:`BinaryArithmeticOperation` now includes :code:`a += b`. + +Ruby +"""" + +* The :code:`else` branch of a :code:`case` expression is no longer represented as a :code:`StmtSequence` directly. Instead, a new :code:`CaseElseBranch` AST node wraps the body (a :code:`StmtSequence`). :code:`CaseExpr.getElseBranch()` now returns a :code:`CaseElseBranch`, and the body of the else branch can be accessed via :code:`CaseElseBranch.getBody()`. + +Major Analysis Improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +C# +"" + +* Added Razor Page handler method parameters (e.g., :code:`OnGet`, :code:`OnPost`, :code:`OnPostAsync`) as remote flow sources, enabling security queries such as :code:`cs/sql-injection` to detect vulnerabilities in :code:`PageModel` subclasses. + +Minor Analysis Improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +C# +"" + +* Improved property and indexer call target resolution for partially overridden properties and indexers. +* Improved extraction of range-access expressions on spans and strings (for example, :code:`a[0..3]`). These expressions are now extracted as :code:`Slice` (span) or :code:`Substring` (string) calls. +* Improved call target resolution for ref-return properties and indexers. + +Golang +"""""" + +* Added models for the :code:`log/slog` package (Go 1.21+). Its logging functions and + :code:`*slog.Logger` methods (:code:`Debug`\ /\ :code:`Info`\ /\ :code:`Warn`\ /\ :code:`Error`, their :code:`Context` variants, and :code:`Log`\ /\ :code:`LogAttrs`) are now recognized as logging sinks, so the + :code:`go/log-injection` and :code:`go/clear-text-logging` queries cover code that logs through :code:`slog`. +* :code:`DataFlow::ResultNode`\ s are no longer created for returned expressions in functions with named result parameters. In this case there are already result nodes corresponding to :code:`IR::ReadResultInstruction`\ s at the end of the function body. +* :code:`FuncTypeExpr.getNumResult()` now gets the number of result parameters. It previously got the number of result declarations, which is different when one result declaration declares more than one variable, as in :code:`x, y int`. All uses of it expected the number of result parameters. Its QLDoc has been updated. +* More logging functions are now recognized as not returning or panicking. + +Java/Kotlin +""""""""""" + +* Improved modeling of Apache HttpClient :code:`execute` method sinks for :code:`java/ssrf` and :code:`java/non-https-url`. + +JavaScript/TypeScript +""""""""""""""""""""" + +* Added more prompt-injection sinks for the OpenAI, Anthropic, and Google GenAI SDKs: OpenAI :code:`videos.create`\ /\ :code:`edit`\ /\ :code:`extend`\ /\ :code:`remix` (Sora) prompts and :code:`beta.realtime.sessions.create` instructions, Anthropic legacy :code:`completions.create` prompts, and Google GenAI :code:`caches.create` cached contents and system instructions. +* The OpenAI legacy :code:`completions.create` prompt is now treated as a user-prompt-injection sink instead of a system-prompt-injection sink, since the legacy :code:`/v1/completions` endpoint takes a single free-form prompt with no role separation. + +Python +"""""" + +* Python type tracking now follows values stored in instance attributes such as :code:`self.attr` across instance methods, including across a class hierarchy (for example, a value stored on :code:`self.attr` in a base class and read in a subclass, or vice versa). As a result, analysis is more likely to recognize user-defined objects that are stored on :code:`self` and used later in other methods, which may produce additional results. +* Simplified the internal predicates that detect :code:`@staticmethod`, :code:`@classmethod` and :code:`@property` decorators to match the decorator's AST :code:`Name` directly, rather than going through the CFG and requiring the name to resolve globally. Code that shadows these three builtin decorators at the module-scope will now be classified by the decorator name alone; in practice, shadowing these names is extremely rare and the call-graph results are unchanged. +* Python taint tracking is now more precise for values flowing through container contents, such as list, set, tuple, and dictionary elements. This may remove some false positive alerts. + +Deprecated APIs +~~~~~~~~~~~~~~~ + +Golang +"""""" + +* :code:`FuncTypeExpr.getResultDecl()` has been deprecated. Use :code:`FuncTypeExpr.getResultDecl(int i)` instead. + +Python +"""""" + +* The :code:`Function.getAReturnValueFlowNode()` predicate has been deprecated. Bind a :code:`Return` node explicitly instead — :code:`exists(Return ret | ret.getScope() = f and n.getNode() = ret.getValue())`. This is a preparatory step towards migrating the dataflow library off the legacy CFG; it has no semantic effect. +* The :code:`AstNode.getAFlowNode()` predicate has been deprecated. Use :code:`ControlFlowNode.getNode()` from the other direction instead: replace :code:`e.getAFlowNode() = n` with :code:`n.getNode() = e`. This is a preparatory step towards migrating the dataflow library off the legacy CFG; it has no semantic effect. + +New Features +~~~~~~~~~~~~ + +Java/Kotlin +""""""""""" + +* Kotlin 2.4.0 can now be analysed. + +JavaScript/TypeScript +""""""""""""""""""""" + +* Added :code:`UseMemoDirective` and :code:`UseNoMemoDirective` classes to model the React compiler directives :code:`"use memo"` and :code:`"use no memo"`. diff --git a/docs/codeql/codeql-overview/codeql-changelog/index.rst b/docs/codeql/codeql-overview/codeql-changelog/index.rst index ac4a8041faa..301adddeedb 100644 --- a/docs/codeql/codeql-overview/codeql-changelog/index.rst +++ b/docs/codeql/codeql-overview/codeql-changelog/index.rst @@ -11,6 +11,7 @@ A list of queries for each suite and language `is available here Date: Thu, 9 Jul 2026 17:09:07 +0200 Subject: [PATCH 22/90] C#: Remove misplaced change note After almost 4 years it does not seem to be relevant anymore to propagate this to the change log. --- .../change-notes/released/2022-11-07-constant-expression.md | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 csharp/ql/src/change-notes/released/2022-11-07-constant-expression.md diff --git a/csharp/ql/src/change-notes/released/2022-11-07-constant-expression.md b/csharp/ql/src/change-notes/released/2022-11-07-constant-expression.md deleted file mode 100644 index 9e2d667d2eb..00000000000 --- a/csharp/ql/src/change-notes/released/2022-11-07-constant-expression.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -category: minorAnalysis ---- -* The `Constant Condition` query was extended to catch cases when `String.IsNullOrEmpty` returns a constant value. \ No newline at end of file From 66fea698a20b62b1c1afb867d6bfea9400710397 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 9 Jul 2026 17:10:13 +0200 Subject: [PATCH 23/90] C#: Add CWE-73 tag to `cs/assembly-path-injection` Use of CWE-114 is discoraged, and CWE-73 seems appropriate as the assembly path is user controlled here. --- .../ql/src/Security Features/CWE-114/AssemblyPathInjection.ql | 1 + csharp/ql/src/change-notes/2026-07-09-CWE-114.md | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 csharp/ql/src/change-notes/2026-07-09-CWE-114.md diff --git a/csharp/ql/src/Security Features/CWE-114/AssemblyPathInjection.ql b/csharp/ql/src/Security Features/CWE-114/AssemblyPathInjection.ql index 9cd6fc68b4c..a17e23fc37d 100644 --- a/csharp/ql/src/Security Features/CWE-114/AssemblyPathInjection.ql +++ b/csharp/ql/src/Security Features/CWE-114/AssemblyPathInjection.ql @@ -9,6 +9,7 @@ * @security-severity 8.2 * @precision high * @tags security + * external/cwe/cwe-073 * external/cwe/cwe-114 */ diff --git a/csharp/ql/src/change-notes/2026-07-09-CWE-114.md b/csharp/ql/src/change-notes/2026-07-09-CWE-114.md new file mode 100644 index 00000000000..928a4cc497a --- /dev/null +++ b/csharp/ql/src/change-notes/2026-07-09-CWE-114.md @@ -0,0 +1,4 @@ +--- +category: queryMetadata +--- +* Added the tag `external/cwe/cwe-073` to `cs/assembly-path-injection`. From 72efb6a0586e860727fe39598ccc931f24895ea9 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 9 Jul 2026 17:11:58 +0200 Subject: [PATCH 24/90] C++: Add CWE 73 and CWE 78 tags to `cpp/uncontrolled-process-operation` User controlled data here either leads to the loading of an uncontrolled library (CWE 73) or the execution of an uncontrolled command (CWE 78). --- .../src/Security/CWE/CWE-114/UncontrolledProcessOperation.ql | 2 ++ cpp/ql/src/change-notes/2026-07-09-CWE-114.md | 4 ++++ 2 files changed, 6 insertions(+) create mode 100644 cpp/ql/src/change-notes/2026-07-09-CWE-114.md diff --git a/cpp/ql/src/Security/CWE/CWE-114/UncontrolledProcessOperation.ql b/cpp/ql/src/Security/CWE/CWE-114/UncontrolledProcessOperation.ql index 7d2513d25e3..f321fc37c28 100644 --- a/cpp/ql/src/Security/CWE/CWE-114/UncontrolledProcessOperation.ql +++ b/cpp/ql/src/Security/CWE/CWE-114/UncontrolledProcessOperation.ql @@ -9,6 +9,8 @@ * @precision medium * @id cpp/uncontrolled-process-operation * @tags security + * external/cwe/cwe-073 + * external/cwe/cwe-078 * external/cwe/cwe-114 */ diff --git a/cpp/ql/src/change-notes/2026-07-09-CWE-114.md b/cpp/ql/src/change-notes/2026-07-09-CWE-114.md new file mode 100644 index 00000000000..f766678232e --- /dev/null +++ b/cpp/ql/src/change-notes/2026-07-09-CWE-114.md @@ -0,0 +1,4 @@ +--- +category: queryMetadata +--- +* Added the tags `external/cwe/cwe-073` and `external/cwe/cwe-078` to `cpp/uncontrolled-process-operation`. From 233c3ce30d70aba4d450ee280adc95b8cfe6f64d Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Thu, 9 Jul 2026 16:47:49 +0100 Subject: [PATCH 25/90] Fix incorrect sanitizers --- ...6-07-09-remove-incorrect-path-sanitizer.md | 4 ++ go/ql/lib/ext/path.filepath.model.yml | 5 --- .../go/security/TaintedPathCustomizations.qll | 37 ------------------- .../Security/CWE-022/TaintedPath.expected | 15 +++++++- .../Security/CWE-022/TaintedPath.go | 8 ++-- .../Security/CWE-022/ZipSlip.expected | 17 ++++++++- .../test/query-tests/Security/CWE-022/tst.go | 8 ++-- 7 files changed, 40 insertions(+), 54 deletions(-) create mode 100644 go/ql/lib/change-notes/2026-07-09-remove-incorrect-path-sanitizer.md diff --git a/go/ql/lib/change-notes/2026-07-09-remove-incorrect-path-sanitizer.md b/go/ql/lib/change-notes/2026-07-09-remove-incorrect-path-sanitizer.md new file mode 100644 index 00000000000..4e7a4be2caf --- /dev/null +++ b/go/ql/lib/change-notes/2026-07-09-remove-incorrect-path-sanitizer.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* The function `Rel` in `path/filepath` was incorrectly considered a sanitizer for `go/path-injection` and `go/zipslip`. This has now been fixed, which may lead to more results for those queries. diff --git a/go/ql/lib/ext/path.filepath.model.yml b/go/ql/lib/ext/path.filepath.model.yml index d450e2bbc56..15bcb7d386d 100644 --- a/go/ql/lib/ext/path.filepath.model.yml +++ b/go/ql/lib/ext/path.filepath.model.yml @@ -1,9 +1,4 @@ extensions: - - addsTo: - pack: codeql/go-all - extensible: barrierModel - data: - - ["path/filepath", "", False, "Rel", "", "", "ReturnValue", "path-injection", "manual"] - addsTo: pack: codeql/go-all extensible: summaryModel diff --git a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll index 20341159c64..d4936553737 100644 --- a/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll +++ b/go/ql/lib/semmle/go/security/TaintedPathCustomizations.qll @@ -129,43 +129,6 @@ module TaintedPath { DotDotReplaceAll() { this.getReplacedString() = ["..", "."] } } - /** - * A node `nd` guarded by a check that ensures it is contained within some root folder, - * considered as a sanitizer for path traversal. - * - * We currently recognize checks of the following form: - * - * ``` - * ..., err := filepath.Rel(base, path) - * if err == nil { - * // path is known to be contained in base - * } - * ``` - */ - class PathContainmentCheck extends SanitizerGuard, DataFlow::EqualityTestNode { - DataFlow::Node path; - boolean outcome; - - PathContainmentCheck() { - exists(Function f, FunctionInput inp, FunctionOutput outp, DataFlow::Property p | - f.hasQualifiedName("path/filepath", "Rel") and - inp.isParameter(1) and - outp.isResult(1) and - p.isNil() - | - exists(DataFlow::Node call, DataFlow::Node res | - call = f.getACall() and - DataFlow::localFlow(outp.getNode(call), res) - | - p.checkOn(this, outcome, res) and - path = inp.getNode(call) - ) - ) - } - - override predicate checks(Expr e, boolean branch) { e = path.asExpr() and branch = outcome } - } - /** * A call of the form `strings.HasPrefix(path, ...)` considered as a sanitizer guard * for `path`. diff --git a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected index c95fa5e7af6..851ed3533ae 100644 --- a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected +++ b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.expected @@ -1,25 +1,36 @@ #select | TaintedPath.go:18:29:18:40 | tainted_path | TaintedPath.go:15:18:15:22 | selection of URL | TaintedPath.go:18:29:18:40 | tainted_path | This path depends on a $@. | TaintedPath.go:15:18:15:22 | selection of URL | user-provided value | | TaintedPath.go:22:28:22:69 | call to Join | TaintedPath.go:15:18:15:22 | selection of URL | TaintedPath.go:22:28:22:69 | call to Join | This path depends on a $@. | TaintedPath.go:15:18:15:22 | selection of URL | user-provided value | +| TaintedPath.go:27:28:27:45 | sanitized_filepath | TaintedPath.go:15:18:15:22 | selection of URL | TaintedPath.go:27:28:27:45 | sanitized_filepath | This path depends on a $@. | TaintedPath.go:15:18:15:22 | selection of URL | user-provided value | +| TaintedPath.go:43:29:43:40 | tainted_path | TaintedPath.go:15:18:15:22 | selection of URL | TaintedPath.go:43:29:43:40 | tainted_path | This path depends on a $@. | TaintedPath.go:15:18:15:22 | selection of URL | user-provided value | | TaintedPath.go:74:28:74:57 | call to Clean | TaintedPath.go:15:18:15:22 | selection of URL | TaintedPath.go:74:28:74:57 | call to Clean | This path depends on a $@. | TaintedPath.go:15:18:15:22 | selection of URL | user-provided value | edges | TaintedPath.go:15:18:15:22 | selection of URL | TaintedPath.go:15:18:15:30 | call to Query | provenance | Src:MaD:2 MaD:3 | | TaintedPath.go:15:18:15:30 | call to Query | TaintedPath.go:18:29:18:40 | tainted_path | provenance | Sink:MaD:1 | | TaintedPath.go:15:18:15:30 | call to Query | TaintedPath.go:22:57:22:68 | tainted_path | provenance | | | TaintedPath.go:22:57:22:68 | tainted_path | TaintedPath.go:22:28:22:69 | call to Join | provenance | FunctionModel Sink:MaD:1 | +| TaintedPath.go:22:57:22:68 | tainted_path | TaintedPath.go:26:63:26:74 | tainted_path | provenance | | +| TaintedPath.go:22:57:22:68 | tainted_path | TaintedPath.go:43:29:43:40 | tainted_path | provenance | Sink:MaD:1 | | TaintedPath.go:22:57:22:68 | tainted_path | TaintedPath.go:74:39:74:56 | ...+... | provenance | | -| TaintedPath.go:74:39:74:56 | ...+... | TaintedPath.go:74:28:74:57 | call to Clean | provenance | MaD:4 Sink:MaD:1 | +| TaintedPath.go:26:2:26:75 | ... := ...[0] | TaintedPath.go:27:28:27:45 | sanitized_filepath | provenance | Sink:MaD:1 | +| TaintedPath.go:26:63:26:74 | tainted_path | TaintedPath.go:26:2:26:75 | ... := ...[0] | provenance | MaD:4 | +| TaintedPath.go:74:39:74:56 | ...+... | TaintedPath.go:74:28:74:57 | call to Clean | provenance | MaD:5 Sink:MaD:1 | models | 1 | Sink: io/ioutil; ; false; ReadFile; ; ; Argument[0]; path-injection; manual | | 2 | Source: net/http; Request; true; URL; ; ; ; remote; manual | | 3 | Summary: net/url; URL; true; Query; ; ; Argument[receiver]; ReturnValue; taint; manual | -| 4 | Summary: path; ; false; Clean; ; ; Argument[0]; ReturnValue; taint; manual | +| 4 | Summary: path/filepath; ; false; Rel; ; ; Argument[0..1]; ReturnValue[0]; taint; manual | +| 5 | Summary: path; ; false; Clean; ; ; Argument[0]; ReturnValue; taint; manual | nodes | TaintedPath.go:15:18:15:22 | selection of URL | semmle.label | selection of URL | | TaintedPath.go:15:18:15:30 | call to Query | semmle.label | call to Query | | TaintedPath.go:18:29:18:40 | tainted_path | semmle.label | tainted_path | | TaintedPath.go:22:28:22:69 | call to Join | semmle.label | call to Join | | TaintedPath.go:22:57:22:68 | tainted_path | semmle.label | tainted_path | +| TaintedPath.go:26:2:26:75 | ... := ...[0] | semmle.label | ... := ...[0] | +| TaintedPath.go:26:63:26:74 | tainted_path | semmle.label | tainted_path | +| TaintedPath.go:27:28:27:45 | sanitized_filepath | semmle.label | sanitized_filepath | +| TaintedPath.go:43:29:43:40 | tainted_path | semmle.label | tainted_path | | TaintedPath.go:74:28:74:57 | call to Clean | semmle.label | call to Clean | | TaintedPath.go:74:39:74:56 | ...+... | semmle.label | ...+... | subpaths diff --git a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go index 812b56f7c94..680c108d056 100644 --- a/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go +++ b/go/ql/test/query-tests/Security/CWE-022/TaintedPath.go @@ -22,9 +22,9 @@ func handler(w http.ResponseWriter, r *http.Request) { data, _ = ioutil.ReadFile(filepath.Join("/home/user/", tainted_path)) // $ Alert[go/path-injection] w.Write(data) - // GOOD: This can only read inside the provided safe path + // BAD: This could still read any file on the file system sanitized_filepath, _ := filepath.Rel("/home/user/safepath", tainted_path) - data, _ = ioutil.ReadFile(sanitized_filepath) + data, _ = ioutil.ReadFile(sanitized_filepath) // $ Alert[go/path-injection] w.Write(data) // GOOD: This can only read inside the provided safe path @@ -37,10 +37,10 @@ func handler(w http.ResponseWriter, r *http.Request) { data, _ = ioutil.ReadFile(strings.ReplaceAll(tainted_path, "..", "")) w.Write(data) - // GOOD: This can only read inside the provided safe path + // BAD: This could still read any file on the file system _, err := filepath.Rel("/home/user/safepath", tainted_path) if err == nil { - data, _ = ioutil.ReadFile(tainted_path) + data, _ = ioutil.ReadFile(tainted_path) // $ Alert[go/path-injection] w.Write(data) } diff --git a/go/ql/test/query-tests/Security/CWE-022/ZipSlip.expected b/go/ql/test/query-tests/Security/CWE-022/ZipSlip.expected index 3bfd80a120c..88f1666af3b 100644 --- a/go/ql/test/query-tests/Security/CWE-022/ZipSlip.expected +++ b/go/ql/test/query-tests/Security/CWE-022/ZipSlip.expected @@ -2,7 +2,9 @@ | UnsafeUnzipSymlinkGood.go:72:3:72:25 | ... := ...[0] | UnsafeUnzipSymlinkGood.go:72:3:72:25 | ... := ...[0] | UnsafeUnzipSymlinkGood.go:61:31:61:62 | call to Join | Unsanitized archive entry, which may contain '..', is used in a $@. | UnsafeUnzipSymlinkGood.go:61:31:61:62 | call to Join | file system operation | | ZipSlip.go:11:2:15:2 | range statement[1] | ZipSlip.go:11:2:15:2 | range statement[1] | ZipSlip.go:14:20:14:20 | p | Unsanitized archive entry, which may contain '..', is used in a $@. | ZipSlip.go:14:20:14:20 | p | file system operation | | tarslip.go:15:2:15:30 | ... := ...[0] | tarslip.go:15:2:15:30 | ... := ...[0] | tarslip.go:16:14:16:34 | call to Dir | Unsanitized archive entry, which may contain '..', is used in a $@. | tarslip.go:16:14:16:34 | call to Dir | file system operation | +| tst.go:23:2:43:2 | range statement[1] | tst.go:23:2:43:2 | range statement[1] | tst.go:27:21:27:48 | call to Join | Unsanitized archive entry, which may contain '..', is used in a $@. | tst.go:27:21:27:48 | call to Join | file system operation | | tst.go:23:2:43:2 | range statement[1] | tst.go:23:2:43:2 | range statement[1] | tst.go:29:20:29:23 | path | Unsanitized archive entry, which may contain '..', is used in a $@. | tst.go:29:20:29:23 | path | file system operation | +| tst.go:23:2:43:2 | range statement[1] | tst.go:23:2:43:2 | range statement[1] | tst.go:31:21:31:24 | path | Unsanitized archive entry, which may contain '..', is used in a $@. | tst.go:31:21:31:24 | path | file system operation | edges | UnsafeUnzipSymlinkGood.go:52:24:52:32 | SSA def(candidate) | UnsafeUnzipSymlinkGood.go:61:53:61:61 | candidate | provenance | | | UnsafeUnzipSymlinkGood.go:61:53:61:61 | candidate | UnsafeUnzipSymlinkGood.go:61:31:61:62 | call to Join | provenance | FunctionModel Sink:MaD:3 | @@ -14,14 +16,20 @@ edges | ZipSlip.go:12:3:12:30 | ... := ...[0] | ZipSlip.go:14:20:14:20 | p | provenance | Sink:MaD:1 | | ZipSlip.go:12:24:12:29 | selection of Name | ZipSlip.go:12:3:12:30 | ... := ...[0] | provenance | MaD:4 | | tarslip.go:15:2:15:30 | ... := ...[0] | tarslip.go:16:23:16:33 | selection of Name | provenance | | -| tarslip.go:16:23:16:33 | selection of Name | tarslip.go:16:14:16:34 | call to Dir | provenance | MaD:5 Sink:MaD:2 | +| tarslip.go:16:23:16:33 | selection of Name | tarslip.go:16:14:16:34 | call to Dir | provenance | MaD:6 Sink:MaD:2 | +| tst.go:23:2:43:2 | range statement[1] | tst.go:25:38:25:41 | path | provenance | | | tst.go:23:2:43:2 | range statement[1] | tst.go:29:20:29:23 | path | provenance | Sink:MaD:1 | +| tst.go:23:2:43:2 | range statement[1] | tst.go:31:21:31:24 | path | provenance | Sink:MaD:1 | +| tst.go:25:3:25:42 | ... := ...[0] | tst.go:27:41:27:47 | relpath | provenance | | +| tst.go:25:38:25:41 | path | tst.go:25:3:25:42 | ... := ...[0] | provenance | MaD:5 | +| tst.go:27:41:27:47 | relpath | tst.go:27:21:27:48 | call to Join | provenance | FunctionModel Sink:MaD:1 | models | 1 | Sink: io/ioutil; ; false; WriteFile; ; ; Argument[0]; path-injection; manual | | 2 | Sink: os; ; false; MkdirAll; ; ; Argument[0]; path-injection; manual | | 3 | Sink: os; ; false; Readlink; ; ; Argument[0]; path-injection; manual | | 4 | Summary: path/filepath; ; false; Abs; ; ; Argument[0]; ReturnValue[0]; taint; manual | -| 5 | Summary: path; ; false; Dir; ; ; Argument[0]; ReturnValue; taint; manual | +| 5 | Summary: path/filepath; ; false; Rel; ; ; Argument[0..1]; ReturnValue[0]; taint; manual | +| 6 | Summary: path; ; false; Dir; ; ; Argument[0]; ReturnValue; taint; manual | nodes | UnsafeUnzipSymlinkGood.go:52:24:52:32 | SSA def(candidate) | semmle.label | SSA def(candidate) | | UnsafeUnzipSymlinkGood.go:61:31:61:62 | call to Join | semmle.label | call to Join | @@ -37,5 +45,10 @@ nodes | tarslip.go:16:14:16:34 | call to Dir | semmle.label | call to Dir | | tarslip.go:16:23:16:33 | selection of Name | semmle.label | selection of Name | | tst.go:23:2:43:2 | range statement[1] | semmle.label | range statement[1] | +| tst.go:25:3:25:42 | ... := ...[0] | semmle.label | ... := ...[0] | +| tst.go:25:38:25:41 | path | semmle.label | path | +| tst.go:27:21:27:48 | call to Join | semmle.label | call to Join | +| tst.go:27:41:27:47 | relpath | semmle.label | relpath | | tst.go:29:20:29:23 | path | semmle.label | path | +| tst.go:31:21:31:24 | path | semmle.label | path | subpaths diff --git a/go/ql/test/query-tests/Security/CWE-022/tst.go b/go/ql/test/query-tests/Security/CWE-022/tst.go index 4cf3a77c4c8..33b2aa072c0 100644 --- a/go/ql/test/query-tests/Security/CWE-022/tst.go +++ b/go/ql/test/query-tests/Security/CWE-022/tst.go @@ -14,7 +14,7 @@ func uploadFile(w http.ResponseWriter, r *http.Request) { file, handler, _ := r.FormFile("file") // err handling defer file.Close() - tempFile, _ := ioutil.TempFile("/tmp", handler.Filename) // NOT OK + tempFile, _ := ioutil.TempFile("/tmp", handler.Filename) // OK use(tempFile) } @@ -24,11 +24,11 @@ func unzip2(f string, root string) { path := f.Name relpath, err := filepath.Rel(root, path) if err == nil { - ioutil.WriteFile(filepath.Join(root, relpath), []byte("present"), 0666) // OK + ioutil.WriteFile(filepath.Join(root, relpath), []byte("present"), 0666) // $ Sink[go/zipslip] } - ioutil.WriteFile(path, []byte("present"), 0666) // $ Sink[go/zipslip] // NOT OK + ioutil.WriteFile(path, []byte("present"), 0666) // $ Sink[go/zipslip] if containedIn(path, root) { - ioutil.WriteFile(path, []byte("present"), 0666) // OK + ioutil.WriteFile(path, []byte("present"), 0666) // $ Sink[go/zipslip] } if ok, _ := regexp.MatchString("^[a-z]*$", path); ok { ioutil.WriteFile(path, []byte("present"), 0666) // OK From 9acbbb80493c04391ce06c9ecc2bc1b645602208 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 9 Jul 2026 16:48:01 +0000 Subject: [PATCH 26/90] Release preparation for version 2.26.1 --- actions/ql/lib/CHANGELOG.md | 4 +++ .../ql/lib/change-notes/released/0.4.39.md | 3 ++ actions/ql/lib/codeql-pack.release.yml | 2 +- actions/ql/lib/qlpack.yml | 2 +- actions/ql/src/CHANGELOG.md | 4 +++ .../ql/src/change-notes/released/0.6.31.md | 3 ++ actions/ql/src/codeql-pack.release.yml | 2 +- actions/ql/src/qlpack.yml | 2 +- cpp/ql/lib/CHANGELOG.md | 10 +++++++ ...6-30-variables-as-mad-sources-and-sinks.md | 4 --- .../12.0.0.md} | 13 ++++++--- cpp/ql/lib/codeql-pack.release.yml | 2 +- cpp/ql/lib/qlpack.yml | 2 +- cpp/ql/src/CHANGELOG.md | 6 ++++ .../1.7.0.md} | 7 +++-- cpp/ql/src/codeql-pack.release.yml | 2 +- cpp/ql/src/qlpack.yml | 2 +- .../ql/campaigns/Solorigate/lib/CHANGELOG.md | 4 +++ .../lib/change-notes/released/1.7.70.md | 3 ++ .../Solorigate/lib/codeql-pack.release.yml | 2 +- csharp/ql/campaigns/Solorigate/lib/qlpack.yml | 2 +- .../ql/campaigns/Solorigate/src/CHANGELOG.md | 4 +++ .../src/change-notes/released/1.7.70.md | 3 ++ .../Solorigate/src/codeql-pack.release.yml | 2 +- csharp/ql/campaigns/Solorigate/src/qlpack.yml | 2 +- csharp/ql/lib/CHANGELOG.md | 6 ++++ .../7.1.0.md} | 7 +++-- csharp/ql/lib/codeql-pack.release.yml | 2 +- csharp/ql/lib/qlpack.yml | 2 +- csharp/ql/src/CHANGELOG.md | 6 ++++ .../1.8.0.md} | 7 +++-- csharp/ql/src/codeql-pack.release.yml | 2 +- csharp/ql/src/qlpack.yml | 2 +- go/ql/consistency-queries/CHANGELOG.md | 4 +++ .../change-notes/released/1.0.53.md | 3 ++ .../codeql-pack.release.yml | 2 +- go/ql/consistency-queries/qlpack.yml | 2 +- go/ql/lib/CHANGELOG.md | 6 ++++ .../7.2.1.md} | 7 +++-- go/ql/lib/codeql-pack.release.yml | 2 +- go/ql/lib/qlpack.yml | 2 +- go/ql/src/CHANGELOG.md | 4 +++ go/ql/src/change-notes/released/1.6.6.md | 3 ++ go/ql/src/codeql-pack.release.yml | 2 +- go/ql/src/qlpack.yml | 2 +- java/ql/lib/CHANGELOG.md | 8 ++++++ .../2026-07-01-org.apache.poi-mads.md | 4 --- ...7-03-fix-pattern-sanitizer-tainted-path.md | 4 --- .../9.2.1.md} | 9 ++++-- java/ql/lib/codeql-pack.release.yml | 2 +- java/ql/lib/qlpack.yml | 2 +- java/ql/src/CHANGELOG.md | 4 +++ java/ql/src/change-notes/released/1.11.6.md | 3 ++ java/ql/src/codeql-pack.release.yml | 2 +- java/ql/src/qlpack.yml | 2 +- javascript/ql/lib/CHANGELOG.md | 6 ++++ .../2.8.1.md} | 7 +++-- javascript/ql/lib/codeql-pack.release.yml | 2 +- javascript/ql/lib/qlpack.yml | 2 +- javascript/ql/src/CHANGELOG.md | 4 +++ .../ql/src/change-notes/released/2.4.1.md | 3 ++ javascript/ql/src/codeql-pack.release.yml | 2 +- javascript/ql/src/qlpack.yml | 2 +- misc/suite-helpers/CHANGELOG.md | 4 +++ .../change-notes/released/1.0.53.md | 3 ++ misc/suite-helpers/codeql-pack.release.yml | 2 +- misc/suite-helpers/qlpack.yml | 2 +- python/ql/lib/CHANGELOG.md | 28 +++++++++++-------- .../7.2.1.md} | 9 +++--- python/ql/lib/codeql-pack.release.yml | 2 +- python/ql/lib/qlpack.yml | 2 +- python/ql/src/CHANGELOG.md | 4 +++ python/ql/src/change-notes/released/1.8.6.md | 3 ++ python/ql/src/codeql-pack.release.yml | 2 +- python/ql/src/qlpack.yml | 2 +- ruby/ql/lib/CHANGELOG.md | 4 +++ ruby/ql/lib/change-notes/released/6.0.1.md | 3 ++ ruby/ql/lib/codeql-pack.release.yml | 2 +- ruby/ql/lib/qlpack.yml | 2 +- ruby/ql/src/CHANGELOG.md | 4 +++ ruby/ql/src/change-notes/released/1.6.6.md | 3 ++ ruby/ql/src/codeql-pack.release.yml | 2 +- ruby/ql/src/qlpack.yml | 2 +- rust/ql/lib/CHANGELOG.md | 4 +++ rust/ql/lib/change-notes/released/0.2.17.md | 3 ++ rust/ql/lib/codeql-pack.release.yml | 2 +- rust/ql/lib/qlpack.yml | 2 +- rust/ql/src/CHANGELOG.md | 6 ++++ .../0.1.38.md} | 7 +++-- rust/ql/src/codeql-pack.release.yml | 2 +- rust/ql/src/qlpack.yml | 2 +- shared/concepts/CHANGELOG.md | 4 +++ .../concepts/change-notes/released/0.0.27.md | 3 ++ shared/concepts/codeql-pack.release.yml | 2 +- shared/concepts/qlpack.yml | 2 +- shared/controlflow/CHANGELOG.md | 4 +++ .../change-notes/released/2.0.37.md | 3 ++ shared/controlflow/codeql-pack.release.yml | 2 +- shared/controlflow/qlpack.yml | 2 +- shared/dataflow/CHANGELOG.md | 4 +++ .../dataflow/change-notes/released/2.1.9.md | 3 ++ shared/dataflow/codeql-pack.release.yml | 2 +- shared/dataflow/qlpack.yml | 2 +- shared/mad/CHANGELOG.md | 4 +++ shared/mad/change-notes/released/1.0.53.md | 3 ++ shared/mad/codeql-pack.release.yml | 2 +- shared/mad/qlpack.yml | 2 +- shared/namebinding/CHANGELOG.md | 4 +++ .../change-notes/released/0.0.2.md | 3 ++ shared/namebinding/codeql-pack.release.yml | 2 +- shared/namebinding/qlpack.yml | 2 +- shared/quantum/CHANGELOG.md | 4 +++ .../quantum/change-notes/released/0.0.31.md | 3 ++ shared/quantum/codeql-pack.release.yml | 2 +- shared/quantum/qlpack.yml | 2 +- shared/rangeanalysis/CHANGELOG.md | 4 +++ .../change-notes/released/1.0.53.md | 3 ++ shared/rangeanalysis/codeql-pack.release.yml | 2 +- shared/rangeanalysis/qlpack.yml | 2 +- shared/regex/CHANGELOG.md | 4 +++ shared/regex/change-notes/released/1.0.53.md | 3 ++ shared/regex/codeql-pack.release.yml | 2 +- shared/regex/qlpack.yml | 2 +- shared/ssa/CHANGELOG.md | 4 +++ shared/ssa/change-notes/released/2.0.29.md | 3 ++ shared/ssa/codeql-pack.release.yml | 2 +- shared/ssa/qlpack.yml | 2 +- shared/threat-models/CHANGELOG.md | 4 +++ .../change-notes/released/1.0.53.md | 3 ++ shared/threat-models/codeql-pack.release.yml | 2 +- shared/threat-models/qlpack.yml | 2 +- shared/tutorial/CHANGELOG.md | 4 +++ .../tutorial/change-notes/released/1.0.53.md | 3 ++ shared/tutorial/codeql-pack.release.yml | 2 +- shared/tutorial/qlpack.yml | 2 +- shared/typeflow/CHANGELOG.md | 4 +++ .../typeflow/change-notes/released/1.0.53.md | 3 ++ shared/typeflow/codeql-pack.release.yml | 2 +- shared/typeflow/qlpack.yml | 2 +- shared/typeinference/CHANGELOG.md | 4 +++ .../change-notes/released/0.0.34.md | 3 ++ shared/typeinference/codeql-pack.release.yml | 2 +- shared/typeinference/qlpack.yml | 2 +- shared/typetracking/CHANGELOG.md | 4 +++ .../change-notes/released/2.0.37.md | 3 ++ shared/typetracking/codeql-pack.release.yml | 2 +- shared/typetracking/qlpack.yml | 2 +- shared/typos/CHANGELOG.md | 4 +++ shared/typos/change-notes/released/1.0.53.md | 3 ++ shared/typos/codeql-pack.release.yml | 2 +- shared/typos/qlpack.yml | 2 +- shared/util/CHANGELOG.md | 4 +++ shared/util/change-notes/released/2.0.40.md | 3 ++ shared/util/codeql-pack.release.yml | 2 +- shared/util/qlpack.yml | 2 +- shared/xml/CHANGELOG.md | 4 +++ shared/xml/change-notes/released/1.0.53.md | 3 ++ shared/xml/codeql-pack.release.yml | 2 +- shared/xml/qlpack.yml | 2 +- shared/yaml/CHANGELOG.md | 4 +++ shared/yaml/change-notes/released/1.0.53.md | 3 ++ shared/yaml/codeql-pack.release.yml | 2 +- shared/yaml/qlpack.yml | 2 +- swift/ql/lib/CHANGELOG.md | 4 +++ swift/ql/lib/change-notes/released/6.7.2.md | 3 ++ swift/ql/lib/codeql-pack.release.yml | 2 +- swift/ql/lib/qlpack.yml | 2 +- swift/ql/src/CHANGELOG.md | 4 +++ swift/ql/src/change-notes/released/1.3.6.md | 3 ++ swift/ql/src/codeql-pack.release.yml | 2 +- swift/ql/src/qlpack.yml | 2 +- 171 files changed, 430 insertions(+), 136 deletions(-) create mode 100644 actions/ql/lib/change-notes/released/0.4.39.md create mode 100644 actions/ql/src/change-notes/released/0.6.31.md delete mode 100644 cpp/ql/lib/change-notes/2026-06-30-variables-as-mad-sources-and-sinks.md rename cpp/ql/lib/change-notes/{2026-06-30-mad-qualified-field-names.md => released/12.0.0.md} (54%) rename cpp/ql/src/change-notes/{2026-07-09-CWE-114.md => released/1.7.0.md} (73%) create mode 100644 csharp/ql/campaigns/Solorigate/lib/change-notes/released/1.7.70.md create mode 100644 csharp/ql/campaigns/Solorigate/src/change-notes/released/1.7.70.md rename csharp/ql/lib/change-notes/{2026-06-24-nuget-packages-config.md => released/7.1.0.md} (90%) rename csharp/ql/src/change-notes/{2026-07-09-CWE-114.md => released/1.8.0.md} (65%) create mode 100644 go/ql/consistency-queries/change-notes/released/1.0.53.md rename go/ql/lib/change-notes/{2026-06-30-model-log-slog.md => released/7.2.1.md} (83%) create mode 100644 go/ql/src/change-notes/released/1.6.6.md delete mode 100644 java/ql/lib/change-notes/2026-07-01-org.apache.poi-mads.md delete mode 100644 java/ql/lib/change-notes/2026-07-03-fix-pattern-sanitizer-tainted-path.md rename java/ql/lib/change-notes/{2026-06-30-spring-webclient-uri-ssrf.md => released/9.2.1.md} (57%) create mode 100644 java/ql/src/change-notes/released/1.11.6.md rename javascript/ql/lib/change-notes/{2026-06-22-angular-hostlistener-postmessage.md => released/2.8.1.md} (87%) create mode 100644 javascript/ql/src/change-notes/released/2.4.1.md create mode 100644 misc/suite-helpers/change-notes/released/1.0.53.md rename python/ql/lib/change-notes/{2026-05-19-flask-subclasses.md => released/7.2.1.md} (79%) create mode 100644 python/ql/src/change-notes/released/1.8.6.md create mode 100644 ruby/ql/lib/change-notes/released/6.0.1.md create mode 100644 ruby/ql/src/change-notes/released/1.6.6.md create mode 100644 rust/ql/lib/change-notes/released/0.2.17.md rename rust/ql/src/change-notes/{2026-06-25-hard-coded-cryptographic-value-arithmetic-barrier.md => released/0.1.38.md} (88%) create mode 100644 shared/concepts/change-notes/released/0.0.27.md create mode 100644 shared/controlflow/change-notes/released/2.0.37.md create mode 100644 shared/dataflow/change-notes/released/2.1.9.md create mode 100644 shared/mad/change-notes/released/1.0.53.md create mode 100644 shared/namebinding/change-notes/released/0.0.2.md create mode 100644 shared/quantum/change-notes/released/0.0.31.md create mode 100644 shared/rangeanalysis/change-notes/released/1.0.53.md create mode 100644 shared/regex/change-notes/released/1.0.53.md create mode 100644 shared/ssa/change-notes/released/2.0.29.md create mode 100644 shared/threat-models/change-notes/released/1.0.53.md create mode 100644 shared/tutorial/change-notes/released/1.0.53.md create mode 100644 shared/typeflow/change-notes/released/1.0.53.md create mode 100644 shared/typeinference/change-notes/released/0.0.34.md create mode 100644 shared/typetracking/change-notes/released/2.0.37.md create mode 100644 shared/typos/change-notes/released/1.0.53.md create mode 100644 shared/util/change-notes/released/2.0.40.md create mode 100644 shared/xml/change-notes/released/1.0.53.md create mode 100644 shared/yaml/change-notes/released/1.0.53.md create mode 100644 swift/ql/lib/change-notes/released/6.7.2.md create mode 100644 swift/ql/src/change-notes/released/1.3.6.md diff --git a/actions/ql/lib/CHANGELOG.md b/actions/ql/lib/CHANGELOG.md index f677e631b4b..c00be74a85e 100644 --- a/actions/ql/lib/CHANGELOG.md +++ b/actions/ql/lib/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.4.39 + +No user-facing changes. + ## 0.4.38 ### Bug Fixes diff --git a/actions/ql/lib/change-notes/released/0.4.39.md b/actions/ql/lib/change-notes/released/0.4.39.md new file mode 100644 index 00000000000..12a9766b4a8 --- /dev/null +++ b/actions/ql/lib/change-notes/released/0.4.39.md @@ -0,0 +1,3 @@ +## 0.4.39 + +No user-facing changes. diff --git a/actions/ql/lib/codeql-pack.release.yml b/actions/ql/lib/codeql-pack.release.yml index 5b7b7bb1f33..6c1c9d9a739 100644 --- a/actions/ql/lib/codeql-pack.release.yml +++ b/actions/ql/lib/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 0.4.38 +lastReleaseVersion: 0.4.39 diff --git a/actions/ql/lib/qlpack.yml b/actions/ql/lib/qlpack.yml index 33b0c790dd6..77ecbfaa8f1 100644 --- a/actions/ql/lib/qlpack.yml +++ b/actions/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/actions-all -version: 0.4.39-dev +version: 0.4.39 library: true warnOnImplicitThis: true dependencies: diff --git a/actions/ql/src/CHANGELOG.md b/actions/ql/src/CHANGELOG.md index d05f3336c09..502c9aefe9d 100644 --- a/actions/ql/src/CHANGELOG.md +++ b/actions/ql/src/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.6.31 + +No user-facing changes. + ## 0.6.30 ### Query Metadata Changes diff --git a/actions/ql/src/change-notes/released/0.6.31.md b/actions/ql/src/change-notes/released/0.6.31.md new file mode 100644 index 00000000000..f408f5c2241 --- /dev/null +++ b/actions/ql/src/change-notes/released/0.6.31.md @@ -0,0 +1,3 @@ +## 0.6.31 + +No user-facing changes. diff --git a/actions/ql/src/codeql-pack.release.yml b/actions/ql/src/codeql-pack.release.yml index 14436232c24..565446e48f2 100644 --- a/actions/ql/src/codeql-pack.release.yml +++ b/actions/ql/src/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 0.6.30 +lastReleaseVersion: 0.6.31 diff --git a/actions/ql/src/qlpack.yml b/actions/ql/src/qlpack.yml index fb617417c3c..b623ff9b903 100644 --- a/actions/ql/src/qlpack.yml +++ b/actions/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/actions-queries -version: 0.6.31-dev +version: 0.6.31 library: false warnOnImplicitThis: true groups: [actions, queries] diff --git a/cpp/ql/lib/CHANGELOG.md b/cpp/ql/lib/CHANGELOG.md index fd08c4404b0..daaa1e03fcb 100644 --- a/cpp/ql/lib/CHANGELOG.md +++ b/cpp/ql/lib/CHANGELOG.md @@ -1,3 +1,13 @@ +## 12.0.0 + +### Breaking Changes + +* Removed support for using variables as sources and sinks in models-as-data. Users of this feature should convert such sources and sinks to models defined using the QL language. + +### Deprecated APIs + +* Models-as-data flow summaries now use fully qualified field names (for example, `MyNamespace::MyStruct::myField`) instead of unqualified field names such as `myField`. We recommend updating existing flow summaries to use fully qualified field names. Unqualified field names are still supported, but that support will be removed in a future release. + ## 11.0.0 ### Breaking Changes diff --git a/cpp/ql/lib/change-notes/2026-06-30-variables-as-mad-sources-and-sinks.md b/cpp/ql/lib/change-notes/2026-06-30-variables-as-mad-sources-and-sinks.md deleted file mode 100644 index fc93dbf303e..00000000000 --- a/cpp/ql/lib/change-notes/2026-06-30-variables-as-mad-sources-and-sinks.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -category: breaking ---- -* Removed support for using variables as sources and sinks in models-as-data. Users of this feature should convert such sources and sinks to models defined using the QL language. \ No newline at end of file diff --git a/cpp/ql/lib/change-notes/2026-06-30-mad-qualified-field-names.md b/cpp/ql/lib/change-notes/released/12.0.0.md similarity index 54% rename from cpp/ql/lib/change-notes/2026-06-30-mad-qualified-field-names.md rename to cpp/ql/lib/change-notes/released/12.0.0.md index f31fcd6490c..69f3fd9220f 100644 --- a/cpp/ql/lib/change-notes/2026-06-30-mad-qualified-field-names.md +++ b/cpp/ql/lib/change-notes/released/12.0.0.md @@ -1,4 +1,9 @@ ---- -category: deprecated ---- -* Models-as-data flow summaries now use fully qualified field names (for example, `MyNamespace::MyStruct::myField`) instead of unqualified field names such as `myField`. We recommend updating existing flow summaries to use fully qualified field names. Unqualified field names are still supported, but that support will be removed in a future release. \ No newline at end of file +## 12.0.0 + +### Breaking Changes + +* Removed support for using variables as sources and sinks in models-as-data. Users of this feature should convert such sources and sinks to models defined using the QL language. + +### Deprecated APIs + +* Models-as-data flow summaries now use fully qualified field names (for example, `MyNamespace::MyStruct::myField`) instead of unqualified field names such as `myField`. We recommend updating existing flow summaries to use fully qualified field names. Unqualified field names are still supported, but that support will be removed in a future release. diff --git a/cpp/ql/lib/codeql-pack.release.yml b/cpp/ql/lib/codeql-pack.release.yml index e9866a9ab38..5bc9fd40be3 100644 --- a/cpp/ql/lib/codeql-pack.release.yml +++ b/cpp/ql/lib/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 11.0.0 +lastReleaseVersion: 12.0.0 diff --git a/cpp/ql/lib/qlpack.yml b/cpp/ql/lib/qlpack.yml index 04f66548112..5f4b92a191f 100644 --- a/cpp/ql/lib/qlpack.yml +++ b/cpp/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/cpp-all -version: 11.0.1-dev +version: 12.0.0 groups: cpp dbscheme: semmlecode.cpp.dbscheme extractor: cpp diff --git a/cpp/ql/src/CHANGELOG.md b/cpp/ql/src/CHANGELOG.md index 9d8877f2181..2fe576d89c7 100644 --- a/cpp/ql/src/CHANGELOG.md +++ b/cpp/ql/src/CHANGELOG.md @@ -1,3 +1,9 @@ +## 1.7.0 + +### Query Metadata Changes + +* Added the tags `external/cwe/cwe-073` and `external/cwe/cwe-078` to `cpp/uncontrolled-process-operation`. + ## 1.6.5 No user-facing changes. diff --git a/cpp/ql/src/change-notes/2026-07-09-CWE-114.md b/cpp/ql/src/change-notes/released/1.7.0.md similarity index 73% rename from cpp/ql/src/change-notes/2026-07-09-CWE-114.md rename to cpp/ql/src/change-notes/released/1.7.0.md index f766678232e..4b53916571b 100644 --- a/cpp/ql/src/change-notes/2026-07-09-CWE-114.md +++ b/cpp/ql/src/change-notes/released/1.7.0.md @@ -1,4 +1,5 @@ ---- -category: queryMetadata ---- +## 1.7.0 + +### Query Metadata Changes + * Added the tags `external/cwe/cwe-073` and `external/cwe/cwe-078` to `cpp/uncontrolled-process-operation`. diff --git a/cpp/ql/src/codeql-pack.release.yml b/cpp/ql/src/codeql-pack.release.yml index 03153270557..d1184cc6750 100644 --- a/cpp/ql/src/codeql-pack.release.yml +++ b/cpp/ql/src/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.6.5 +lastReleaseVersion: 1.7.0 diff --git a/cpp/ql/src/qlpack.yml b/cpp/ql/src/qlpack.yml index 3b6365f29c6..1d1a9206a28 100644 --- a/cpp/ql/src/qlpack.yml +++ b/cpp/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/cpp-queries -version: 1.6.6-dev +version: 1.7.0 groups: - cpp - queries diff --git a/csharp/ql/campaigns/Solorigate/lib/CHANGELOG.md b/csharp/ql/campaigns/Solorigate/lib/CHANGELOG.md index e1fbde4a626..69004620f59 100644 --- a/csharp/ql/campaigns/Solorigate/lib/CHANGELOG.md +++ b/csharp/ql/campaigns/Solorigate/lib/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.7.70 + +No user-facing changes. + ## 1.7.69 No user-facing changes. diff --git a/csharp/ql/campaigns/Solorigate/lib/change-notes/released/1.7.70.md b/csharp/ql/campaigns/Solorigate/lib/change-notes/released/1.7.70.md new file mode 100644 index 00000000000..8b0555eb6b0 --- /dev/null +++ b/csharp/ql/campaigns/Solorigate/lib/change-notes/released/1.7.70.md @@ -0,0 +1,3 @@ +## 1.7.70 + +No user-facing changes. diff --git a/csharp/ql/campaigns/Solorigate/lib/codeql-pack.release.yml b/csharp/ql/campaigns/Solorigate/lib/codeql-pack.release.yml index 711f9a5b58f..af6d36d8096 100644 --- a/csharp/ql/campaigns/Solorigate/lib/codeql-pack.release.yml +++ b/csharp/ql/campaigns/Solorigate/lib/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.7.69 +lastReleaseVersion: 1.7.70 diff --git a/csharp/ql/campaigns/Solorigate/lib/qlpack.yml b/csharp/ql/campaigns/Solorigate/lib/qlpack.yml index 88080d5df9a..9fb3427e685 100644 --- a/csharp/ql/campaigns/Solorigate/lib/qlpack.yml +++ b/csharp/ql/campaigns/Solorigate/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/csharp-solorigate-all -version: 1.7.70-dev +version: 1.7.70 groups: - csharp - solorigate diff --git a/csharp/ql/campaigns/Solorigate/src/CHANGELOG.md b/csharp/ql/campaigns/Solorigate/src/CHANGELOG.md index e1fbde4a626..69004620f59 100644 --- a/csharp/ql/campaigns/Solorigate/src/CHANGELOG.md +++ b/csharp/ql/campaigns/Solorigate/src/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.7.70 + +No user-facing changes. + ## 1.7.69 No user-facing changes. diff --git a/csharp/ql/campaigns/Solorigate/src/change-notes/released/1.7.70.md b/csharp/ql/campaigns/Solorigate/src/change-notes/released/1.7.70.md new file mode 100644 index 00000000000..8b0555eb6b0 --- /dev/null +++ b/csharp/ql/campaigns/Solorigate/src/change-notes/released/1.7.70.md @@ -0,0 +1,3 @@ +## 1.7.70 + +No user-facing changes. diff --git a/csharp/ql/campaigns/Solorigate/src/codeql-pack.release.yml b/csharp/ql/campaigns/Solorigate/src/codeql-pack.release.yml index 711f9a5b58f..af6d36d8096 100644 --- a/csharp/ql/campaigns/Solorigate/src/codeql-pack.release.yml +++ b/csharp/ql/campaigns/Solorigate/src/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.7.69 +lastReleaseVersion: 1.7.70 diff --git a/csharp/ql/campaigns/Solorigate/src/qlpack.yml b/csharp/ql/campaigns/Solorigate/src/qlpack.yml index effa1c940c0..4b55c941207 100644 --- a/csharp/ql/campaigns/Solorigate/src/qlpack.yml +++ b/csharp/ql/campaigns/Solorigate/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/csharp-solorigate-queries -version: 1.7.70-dev +version: 1.7.70 groups: - csharp - solorigate diff --git a/csharp/ql/lib/CHANGELOG.md b/csharp/ql/lib/CHANGELOG.md index 7987a729ec6..3f8f2d91d17 100644 --- a/csharp/ql/lib/CHANGELOG.md +++ b/csharp/ql/lib/CHANGELOG.md @@ -1,3 +1,9 @@ +## 7.1.0 + +### Major Analysis Improvements + +* Simplified and streamlined the use of NuGet sources when downloading dependencies via `[mono] nuget.exe` in `build-mode: none`: NuGet sources are now supplied via the `-Source` flag instead of moving or creating `nuget.config` files in the checked-out repository, private registries are used if configured, and only reachable feeds are used when NuGet feed checking is enabled (the default). + ## 7.0.0 ### Breaking Changes diff --git a/csharp/ql/lib/change-notes/2026-06-24-nuget-packages-config.md b/csharp/ql/lib/change-notes/released/7.1.0.md similarity index 90% rename from csharp/ql/lib/change-notes/2026-06-24-nuget-packages-config.md rename to csharp/ql/lib/change-notes/released/7.1.0.md index 5b236a118da..6a164c9ebb4 100644 --- a/csharp/ql/lib/change-notes/2026-06-24-nuget-packages-config.md +++ b/csharp/ql/lib/change-notes/released/7.1.0.md @@ -1,4 +1,5 @@ ---- -category: majorAnalysis ---- +## 7.1.0 + +### Major Analysis Improvements + * Simplified and streamlined the use of NuGet sources when downloading dependencies via `[mono] nuget.exe` in `build-mode: none`: NuGet sources are now supplied via the `-Source` flag instead of moving or creating `nuget.config` files in the checked-out repository, private registries are used if configured, and only reachable feeds are used when NuGet feed checking is enabled (the default). diff --git a/csharp/ql/lib/codeql-pack.release.yml b/csharp/ql/lib/codeql-pack.release.yml index e0db21c7869..dcaaa76112a 100644 --- a/csharp/ql/lib/codeql-pack.release.yml +++ b/csharp/ql/lib/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 7.0.0 +lastReleaseVersion: 7.1.0 diff --git a/csharp/ql/lib/qlpack.yml b/csharp/ql/lib/qlpack.yml index 0749eea574d..336a7c6be73 100644 --- a/csharp/ql/lib/qlpack.yml +++ b/csharp/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/csharp-all -version: 7.0.1-dev +version: 7.1.0 groups: csharp dbscheme: semmlecode.csharp.dbscheme extractor: csharp diff --git a/csharp/ql/src/CHANGELOG.md b/csharp/ql/src/CHANGELOG.md index 2e316088da5..931690b863e 100644 --- a/csharp/ql/src/CHANGELOG.md +++ b/csharp/ql/src/CHANGELOG.md @@ -1,3 +1,9 @@ +## 1.8.0 + +### Query Metadata Changes + +* Added the tag `external/cwe/cwe-073` to `cs/assembly-path-injection`. + ## 1.7.5 No user-facing changes. diff --git a/csharp/ql/src/change-notes/2026-07-09-CWE-114.md b/csharp/ql/src/change-notes/released/1.8.0.md similarity index 65% rename from csharp/ql/src/change-notes/2026-07-09-CWE-114.md rename to csharp/ql/src/change-notes/released/1.8.0.md index 928a4cc497a..b7c4ca4cf22 100644 --- a/csharp/ql/src/change-notes/2026-07-09-CWE-114.md +++ b/csharp/ql/src/change-notes/released/1.8.0.md @@ -1,4 +1,5 @@ ---- -category: queryMetadata ---- +## 1.8.0 + +### Query Metadata Changes + * Added the tag `external/cwe/cwe-073` to `cs/assembly-path-injection`. diff --git a/csharp/ql/src/codeql-pack.release.yml b/csharp/ql/src/codeql-pack.release.yml index 83aebd7c12a..dc8a37cc443 100644 --- a/csharp/ql/src/codeql-pack.release.yml +++ b/csharp/ql/src/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.7.5 +lastReleaseVersion: 1.8.0 diff --git a/csharp/ql/src/qlpack.yml b/csharp/ql/src/qlpack.yml index 9110c334a2e..b243e8fb928 100644 --- a/csharp/ql/src/qlpack.yml +++ b/csharp/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/csharp-queries -version: 1.7.6-dev +version: 1.8.0 groups: - csharp - queries diff --git a/go/ql/consistency-queries/CHANGELOG.md b/go/ql/consistency-queries/CHANGELOG.md index 1b79dbf69e2..676a2dc0189 100644 --- a/go/ql/consistency-queries/CHANGELOG.md +++ b/go/ql/consistency-queries/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.0.53 + +No user-facing changes. + ## 1.0.52 No user-facing changes. diff --git a/go/ql/consistency-queries/change-notes/released/1.0.53.md b/go/ql/consistency-queries/change-notes/released/1.0.53.md new file mode 100644 index 00000000000..0421d6f4cad --- /dev/null +++ b/go/ql/consistency-queries/change-notes/released/1.0.53.md @@ -0,0 +1,3 @@ +## 1.0.53 + +No user-facing changes. diff --git a/go/ql/consistency-queries/codeql-pack.release.yml b/go/ql/consistency-queries/codeql-pack.release.yml index ea1d2eed4d2..fb44bfeae13 100644 --- a/go/ql/consistency-queries/codeql-pack.release.yml +++ b/go/ql/consistency-queries/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.0.52 +lastReleaseVersion: 1.0.53 diff --git a/go/ql/consistency-queries/qlpack.yml b/go/ql/consistency-queries/qlpack.yml index 486dcf5c9f8..d7752bb1bf6 100644 --- a/go/ql/consistency-queries/qlpack.yml +++ b/go/ql/consistency-queries/qlpack.yml @@ -1,5 +1,5 @@ name: codeql-go-consistency-queries -version: 1.0.53-dev +version: 1.0.53 groups: - go - queries diff --git a/go/ql/lib/CHANGELOG.md b/go/ql/lib/CHANGELOG.md index 29a5bfbf178..72566ce22c0 100644 --- a/go/ql/lib/CHANGELOG.md +++ b/go/ql/lib/CHANGELOG.md @@ -1,3 +1,9 @@ +## 7.2.1 + +### Minor Analysis Improvements + + * Improved models for the `log/slog` package (Go 1.21+), including `*slog.Logger` methods, `With`/`WithGroup`, and `Attr`/`Value` helpers, improving coverage for the `go/log-injection` and `go/clear-text-logging` queries. + ## 7.2.0 ### Deprecated APIs diff --git a/go/ql/lib/change-notes/2026-06-30-model-log-slog.md b/go/ql/lib/change-notes/released/7.2.1.md similarity index 83% rename from go/ql/lib/change-notes/2026-06-30-model-log-slog.md rename to go/ql/lib/change-notes/released/7.2.1.md index 836678f1ac9..20e73beaddc 100644 --- a/go/ql/lib/change-notes/2026-06-30-model-log-slog.md +++ b/go/ql/lib/change-notes/released/7.2.1.md @@ -1,4 +1,5 @@ ---- -category: minorAnalysis ---- +## 7.2.1 + +### Minor Analysis Improvements + * Improved models for the `log/slog` package (Go 1.21+), including `*slog.Logger` methods, `With`/`WithGroup`, and `Attr`/`Value` helpers, improving coverage for the `go/log-injection` and `go/clear-text-logging` queries. diff --git a/go/ql/lib/codeql-pack.release.yml b/go/ql/lib/codeql-pack.release.yml index fda9ea165fc..09f680074c4 100644 --- a/go/ql/lib/codeql-pack.release.yml +++ b/go/ql/lib/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 7.2.0 +lastReleaseVersion: 7.2.1 diff --git a/go/ql/lib/qlpack.yml b/go/ql/lib/qlpack.yml index f65b3855cf7..1573f5b2ad8 100644 --- a/go/ql/lib/qlpack.yml +++ b/go/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/go-all -version: 7.2.1-dev +version: 7.2.1 groups: go dbscheme: go.dbscheme extractor: go diff --git a/go/ql/src/CHANGELOG.md b/go/ql/src/CHANGELOG.md index b74b08295b2..85425d0c223 100644 --- a/go/ql/src/CHANGELOG.md +++ b/go/ql/src/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.6.6 + +No user-facing changes. + ## 1.6.5 ### Minor Analysis Improvements diff --git a/go/ql/src/change-notes/released/1.6.6.md b/go/ql/src/change-notes/released/1.6.6.md new file mode 100644 index 00000000000..14281d9101b --- /dev/null +++ b/go/ql/src/change-notes/released/1.6.6.md @@ -0,0 +1,3 @@ +## 1.6.6 + +No user-facing changes. diff --git a/go/ql/src/codeql-pack.release.yml b/go/ql/src/codeql-pack.release.yml index 03153270557..f8e54f30a67 100644 --- a/go/ql/src/codeql-pack.release.yml +++ b/go/ql/src/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.6.5 +lastReleaseVersion: 1.6.6 diff --git a/go/ql/src/qlpack.yml b/go/ql/src/qlpack.yml index 2db1c639026..98b0c2cf12c 100644 --- a/go/ql/src/qlpack.yml +++ b/go/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/go-queries -version: 1.6.6-dev +version: 1.6.6 groups: - go - queries diff --git a/java/ql/lib/CHANGELOG.md b/java/ql/lib/CHANGELOG.md index 9a60d9f070e..cb3200a4ef4 100644 --- a/java/ql/lib/CHANGELOG.md +++ b/java/ql/lib/CHANGELOG.md @@ -1,3 +1,11 @@ +## 9.2.1 + +### Minor Analysis Improvements + +* Regular expression checks via annotation with `@javax.validation.constraints.Pattern` are now recognized as sanitizers for `java/path-injection`. +* Added summary and LLM-generated source and sink models for `org.apache.poi`. +* The first argument of the `uri` method of `WebClient$UriSpec` in `org.springframework.web.reactive.function.client` is now considered a request forgery sink. Previously only the first arguments of the `WebClient.create` and `WebClient$Builder.baseUrl` methods were considered. This may lead to more alerts for the query `java/ssrf` (Server-side request forgery). + ## 9.2.0 ### New Features diff --git a/java/ql/lib/change-notes/2026-07-01-org.apache.poi-mads.md b/java/ql/lib/change-notes/2026-07-01-org.apache.poi-mads.md deleted file mode 100644 index 741372b8e59..00000000000 --- a/java/ql/lib/change-notes/2026-07-01-org.apache.poi-mads.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -category: minorAnalysis ---- -* Added summary and LLM-generated source and sink models for `org.apache.poi`. diff --git a/java/ql/lib/change-notes/2026-07-03-fix-pattern-sanitizer-tainted-path.md b/java/ql/lib/change-notes/2026-07-03-fix-pattern-sanitizer-tainted-path.md deleted file mode 100644 index b04c6594445..00000000000 --- a/java/ql/lib/change-notes/2026-07-03-fix-pattern-sanitizer-tainted-path.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -category: minorAnalysis ---- -* Regular expression checks via annotation with `@javax.validation.constraints.Pattern` are now recognized as sanitizers for `java/path-injection`. diff --git a/java/ql/lib/change-notes/2026-06-30-spring-webclient-uri-ssrf.md b/java/ql/lib/change-notes/released/9.2.1.md similarity index 57% rename from java/ql/lib/change-notes/2026-06-30-spring-webclient-uri-ssrf.md rename to java/ql/lib/change-notes/released/9.2.1.md index d0474959b9c..8292ca72b1e 100644 --- a/java/ql/lib/change-notes/2026-06-30-spring-webclient-uri-ssrf.md +++ b/java/ql/lib/change-notes/released/9.2.1.md @@ -1,4 +1,7 @@ ---- -category: minorAnalysis ---- +## 9.2.1 + +### Minor Analysis Improvements + +* Regular expression checks via annotation with `@javax.validation.constraints.Pattern` are now recognized as sanitizers for `java/path-injection`. +* Added summary and LLM-generated source and sink models for `org.apache.poi`. * The first argument of the `uri` method of `WebClient$UriSpec` in `org.springframework.web.reactive.function.client` is now considered a request forgery sink. Previously only the first arguments of the `WebClient.create` and `WebClient$Builder.baseUrl` methods were considered. This may lead to more alerts for the query `java/ssrf` (Server-side request forgery). diff --git a/java/ql/lib/codeql-pack.release.yml b/java/ql/lib/codeql-pack.release.yml index 8bc32f3e62a..336f8a5aa5b 100644 --- a/java/ql/lib/codeql-pack.release.yml +++ b/java/ql/lib/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 9.2.0 +lastReleaseVersion: 9.2.1 diff --git a/java/ql/lib/qlpack.yml b/java/ql/lib/qlpack.yml index a847cb88c63..a3664696b9d 100644 --- a/java/ql/lib/qlpack.yml +++ b/java/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/java-all -version: 9.2.1-dev +version: 9.2.1 groups: java dbscheme: config/semmlecode.dbscheme extractor: java diff --git a/java/ql/src/CHANGELOG.md b/java/ql/src/CHANGELOG.md index 4e7c1a329c2..234038218cb 100644 --- a/java/ql/src/CHANGELOG.md +++ b/java/ql/src/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.11.6 + +No user-facing changes. + ## 1.11.5 No user-facing changes. diff --git a/java/ql/src/change-notes/released/1.11.6.md b/java/ql/src/change-notes/released/1.11.6.md new file mode 100644 index 00000000000..8d55b7757a2 --- /dev/null +++ b/java/ql/src/change-notes/released/1.11.6.md @@ -0,0 +1,3 @@ +## 1.11.6 + +No user-facing changes. diff --git a/java/ql/src/codeql-pack.release.yml b/java/ql/src/codeql-pack.release.yml index d3dd29373b1..b30a039ff43 100644 --- a/java/ql/src/codeql-pack.release.yml +++ b/java/ql/src/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.11.5 +lastReleaseVersion: 1.11.6 diff --git a/java/ql/src/qlpack.yml b/java/ql/src/qlpack.yml index 6f9c819f109..2c49d49c617 100644 --- a/java/ql/src/qlpack.yml +++ b/java/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/java-queries -version: 1.11.6-dev +version: 1.11.6 groups: - java - queries diff --git a/javascript/ql/lib/CHANGELOG.md b/javascript/ql/lib/CHANGELOG.md index e3802a7686e..3d89e1bca7e 100644 --- a/javascript/ql/lib/CHANGELOG.md +++ b/javascript/ql/lib/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.8.1 + +### Minor Analysis Improvements + +* Added support for Angular's `@HostListener('window:message', ...)` and `@HostListener('document:message', ...)` decorators as `postMessage` event handlers. The decorated method's event parameter is now recognized as a client-side remote flow source, and is considered by the `js/missing-origin-check` query. + ## 2.8.0 ### New Features diff --git a/javascript/ql/lib/change-notes/2026-06-22-angular-hostlistener-postmessage.md b/javascript/ql/lib/change-notes/released/2.8.1.md similarity index 87% rename from javascript/ql/lib/change-notes/2026-06-22-angular-hostlistener-postmessage.md rename to javascript/ql/lib/change-notes/released/2.8.1.md index 686e8ae96da..983d8b8e009 100644 --- a/javascript/ql/lib/change-notes/2026-06-22-angular-hostlistener-postmessage.md +++ b/javascript/ql/lib/change-notes/released/2.8.1.md @@ -1,4 +1,5 @@ ---- -category: minorAnalysis ---- +## 2.8.1 + +### Minor Analysis Improvements + * Added support for Angular's `@HostListener('window:message', ...)` and `@HostListener('document:message', ...)` decorators as `postMessage` event handlers. The decorated method's event parameter is now recognized as a client-side remote flow source, and is considered by the `js/missing-origin-check` query. diff --git a/javascript/ql/lib/codeql-pack.release.yml b/javascript/ql/lib/codeql-pack.release.yml index 8e0a6e07a08..ff1368057c6 100644 --- a/javascript/ql/lib/codeql-pack.release.yml +++ b/javascript/ql/lib/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 2.8.0 +lastReleaseVersion: 2.8.1 diff --git a/javascript/ql/lib/qlpack.yml b/javascript/ql/lib/qlpack.yml index 584f2e135f7..3ffde4f1d31 100644 --- a/javascript/ql/lib/qlpack.yml +++ b/javascript/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/javascript-all -version: 2.8.1-dev +version: 2.8.1 groups: javascript dbscheme: semmlecode.javascript.dbscheme extractor: javascript diff --git a/javascript/ql/src/CHANGELOG.md b/javascript/ql/src/CHANGELOG.md index 3da6a12390e..32c71e4bd81 100644 --- a/javascript/ql/src/CHANGELOG.md +++ b/javascript/ql/src/CHANGELOG.md @@ -1,3 +1,7 @@ +## 2.4.1 + +No user-facing changes. + ## 2.4.0 ### New Queries diff --git a/javascript/ql/src/change-notes/released/2.4.1.md b/javascript/ql/src/change-notes/released/2.4.1.md new file mode 100644 index 00000000000..bd572220957 --- /dev/null +++ b/javascript/ql/src/change-notes/released/2.4.1.md @@ -0,0 +1,3 @@ +## 2.4.1 + +No user-facing changes. diff --git a/javascript/ql/src/codeql-pack.release.yml b/javascript/ql/src/codeql-pack.release.yml index cb0ea3a249a..eead7b212da 100644 --- a/javascript/ql/src/codeql-pack.release.yml +++ b/javascript/ql/src/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 2.4.0 +lastReleaseVersion: 2.4.1 diff --git a/javascript/ql/src/qlpack.yml b/javascript/ql/src/qlpack.yml index b608077e3e0..3a88e0c6b13 100644 --- a/javascript/ql/src/qlpack.yml +++ b/javascript/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/javascript-queries -version: 2.4.1-dev +version: 2.4.1 groups: - javascript - queries diff --git a/misc/suite-helpers/CHANGELOG.md b/misc/suite-helpers/CHANGELOG.md index b73e8234a5b..af850b3d5af 100644 --- a/misc/suite-helpers/CHANGELOG.md +++ b/misc/suite-helpers/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.0.53 + +No user-facing changes. + ## 1.0.52 No user-facing changes. diff --git a/misc/suite-helpers/change-notes/released/1.0.53.md b/misc/suite-helpers/change-notes/released/1.0.53.md new file mode 100644 index 00000000000..0421d6f4cad --- /dev/null +++ b/misc/suite-helpers/change-notes/released/1.0.53.md @@ -0,0 +1,3 @@ +## 1.0.53 + +No user-facing changes. diff --git a/misc/suite-helpers/codeql-pack.release.yml b/misc/suite-helpers/codeql-pack.release.yml index ea1d2eed4d2..fb44bfeae13 100644 --- a/misc/suite-helpers/codeql-pack.release.yml +++ b/misc/suite-helpers/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.0.52 +lastReleaseVersion: 1.0.53 diff --git a/misc/suite-helpers/qlpack.yml b/misc/suite-helpers/qlpack.yml index 0dafb086b74..248b9c7207c 100644 --- a/misc/suite-helpers/qlpack.yml +++ b/misc/suite-helpers/qlpack.yml @@ -1,4 +1,4 @@ name: codeql/suite-helpers -version: 1.0.53-dev +version: 1.0.53 groups: shared warnOnImplicitThis: true diff --git a/python/ql/lib/CHANGELOG.md b/python/ql/lib/CHANGELOG.md index a8122f03eb1..1fd5cec625d 100644 --- a/python/ql/lib/CHANGELOG.md +++ b/python/ql/lib/CHANGELOG.md @@ -1,3 +1,9 @@ +## 7.2.1 + +### Minor Analysis Improvements + +* `Flask::FlaskApp::instance()` will now also return instances of subclasses defined in the source tree. Previously, these were filtered out. `Flask::FlaskApp::classRef()` has been deprecated in favor of `Flask::FlaskApp::subclassRef()` since it already returned some subclasses. + ## 7.2.0 ### Deprecated APIs @@ -70,7 +76,7 @@ No user-facing changes. ### Minor Analysis Improvements -* Added new full SSRF sanitization barrier from the new AntiSSRF library. +* Added new full SSRF sanitization barrier from the new AntiSSRF library. * When a guard such as `isSafe(x)` is defined, we now also automatically handle `isSafe(x) == true` and `isSafe(x) != false`. ## 6.1.1 @@ -169,7 +175,7 @@ No user-facing changes. ### Minor Analysis Improvements - The modelling of Psycopg2 now supports the use of `psycopg2.pool` connection pools for handling database connections. -* Removed `lxml` as an XML bomb sink. The underlying libxml2 library now includes [entity reference loop detection](https://github.com/lxml/lxml/blob/f33ac2c2f5f9c4c4c1fc47f363be96db308f2fa6/doc/FAQ.txt#L1077) that prevents XML bomb attacks. +* Removed `lxml` as an XML bomb sink. The underlying libxml2 library now includes [entity reference loop detection](https://github.com/lxml/lxml/blob/f33ac2c2f5f9c4c4c1fc47f363be96db308f2fa6/doc/FAQ.txt#L1077) that prevents XML bomb attacks. ## 4.0.13 @@ -262,7 +268,7 @@ No user-facing changes. ### Minor Analysis Improvements * The sensitive data library has been improved so that `snake_case` style variable names are recognized more reliably. This may result in more sensitive data being identified, and more results from queries that use the sensitive data library. -- Additional taint steps through methods of `lxml.etree.Element` and `lxml.etree.ElementTree` objects from the `lxml` PyPI package have been modeled. +- Additional taint steps through methods of `lxml.etree.Element` and `lxml.etree.ElementTree` objects from the `lxml` PyPI package have been modeled. ## 3.1.0 @@ -316,7 +322,7 @@ No user-facing changes. ### Minor Analysis Improvements -* The common sanitizer guard `StringConstCompareBarrier` has been renamed to `ConstCompareBarrier` and expanded to cover comparisons with other constant values such as `None`. This may result in fewer false positive results for several queries. +* The common sanitizer guard `StringConstCompareBarrier` has been renamed to `ConstCompareBarrier` and expanded to cover comparisons with other constant values such as `None`. This may result in fewer false positive results for several queries. ## 2.0.0 @@ -545,7 +551,7 @@ No user-facing changes. ### New Features -* The `DataFlow::StateConfigSig` signature module has gained default implementations for `isBarrier/2` and `isAdditionalFlowStep/4`. +* The `DataFlow::StateConfigSig` signature module has gained default implementations for `isBarrier/2` and `isAdditionalFlowStep/4`. Hence it is no longer needed to provide `none()` implementations of these predicates if they are not needed. ### Minor Analysis Improvements @@ -572,7 +578,7 @@ No user-facing changes. * Deleted many deprecated predicates and classes with uppercase `API`, `HTTP`, `XSS`, `SQL`, etc. in their names. Use the PascalCased versions instead. * Deleted the deprecated `getName()` predicate from the `Container` class, use `getAbsolutePath()` instead. * Deleted many deprecated module names that started with a lowercase letter, use the versions that start with an uppercase letter instead. -* Deleted many deprecated predicates in `PointsTo.qll`. +* Deleted many deprecated predicates in `PointsTo.qll`. * Deleted many deprecated files from the `semmle.python.security` package. * Deleted the deprecated `BottleRoutePointToExtension` class from `Extensions.qll`. * Type tracking is now aware of flow summaries. This leads to a richer API graph, and may lead to more results in some queries. @@ -729,7 +735,7 @@ No user-facing changes. ### Deprecated APIs * Some unused predicates in `SsaDefinitions.qll`, `TObject.qll`, `protocols.qll`, and the `pointsto/` folder have been deprecated. -* Some classes/modules with upper-case acronyms in their name have been renamed to follow our style-guide. +* Some classes/modules with upper-case acronyms in their name have been renamed to follow our style-guide. The old name still exists as a deprecated alias. ### Minor Analysis Improvements @@ -748,9 +754,9 @@ No user-facing changes. ### Deprecated APIs -* Many classes/predicates/modules with upper-case acronyms in their name have been renamed to follow our style-guide. +* Many classes/predicates/modules with upper-case acronyms in their name have been renamed to follow our style-guide. The old name still exists as a deprecated alias. -* The utility files previously in the `semmle.python.security.performance` package have been moved to the `semmle.python.security.regexp` package. +* The utility files previously in the `semmle.python.security.performance` package have been moved to the `semmle.python.security.regexp` package. The previous files still exist as deprecated aliases. ### Minor Analysis Improvements @@ -843,9 +849,9 @@ No user-facing changes. ### Deprecated APIs -* Many classes/predicates/modules that had upper-case acronyms have been renamed to follow our style-guide. +* Many classes/predicates/modules that had upper-case acronyms have been renamed to follow our style-guide. The old name still exists as a deprecated alias. -* Some modules that started with a lowercase letter have been renamed to follow our style-guide. +* Some modules that started with a lowercase letter have been renamed to follow our style-guide. The old name still exists as a deprecated alias. ### New Features diff --git a/python/ql/lib/change-notes/2026-05-19-flask-subclasses.md b/python/ql/lib/change-notes/released/7.2.1.md similarity index 79% rename from python/ql/lib/change-notes/2026-05-19-flask-subclasses.md rename to python/ql/lib/change-notes/released/7.2.1.md index e0288351ddc..7f90ffa118c 100644 --- a/python/ql/lib/change-notes/2026-05-19-flask-subclasses.md +++ b/python/ql/lib/change-notes/released/7.2.1.md @@ -1,4 +1,5 @@ ---- -category: minorAnalysis ---- -* `Flask::FlaskApp::instance()` will now also return instances of subclasses defined in the source tree. Previously, these were filtered out. `Flask::FlaskApp::classRef()` has been deprecated in favor of `Flask::FlaskApp::subclassRef()` since it already returned some subclasses. \ No newline at end of file +## 7.2.1 + +### Minor Analysis Improvements + +* `Flask::FlaskApp::instance()` will now also return instances of subclasses defined in the source tree. Previously, these were filtered out. `Flask::FlaskApp::classRef()` has been deprecated in favor of `Flask::FlaskApp::subclassRef()` since it already returned some subclasses. diff --git a/python/ql/lib/codeql-pack.release.yml b/python/ql/lib/codeql-pack.release.yml index fda9ea165fc..09f680074c4 100644 --- a/python/ql/lib/codeql-pack.release.yml +++ b/python/ql/lib/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 7.2.0 +lastReleaseVersion: 7.2.1 diff --git a/python/ql/lib/qlpack.yml b/python/ql/lib/qlpack.yml index 506fd493c79..837f6c437dd 100644 --- a/python/ql/lib/qlpack.yml +++ b/python/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/python-all -version: 7.2.1-dev +version: 7.2.1 groups: python dbscheme: semmlecode.python.dbscheme extractor: python diff --git a/python/ql/src/CHANGELOG.md b/python/ql/src/CHANGELOG.md index 0c9c972e5fa..f5681e69958 100644 --- a/python/ql/src/CHANGELOG.md +++ b/python/ql/src/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.8.6 + +No user-facing changes. + ## 1.8.5 ### Minor Analysis Improvements diff --git a/python/ql/src/change-notes/released/1.8.6.md b/python/ql/src/change-notes/released/1.8.6.md new file mode 100644 index 00000000000..ae534ff6538 --- /dev/null +++ b/python/ql/src/change-notes/released/1.8.6.md @@ -0,0 +1,3 @@ +## 1.8.6 + +No user-facing changes. diff --git a/python/ql/src/codeql-pack.release.yml b/python/ql/src/codeql-pack.release.yml index 75869ad94ec..c0101bd25c9 100644 --- a/python/ql/src/codeql-pack.release.yml +++ b/python/ql/src/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.8.5 +lastReleaseVersion: 1.8.6 diff --git a/python/ql/src/qlpack.yml b/python/ql/src/qlpack.yml index a4a2db0e660..5e0040f834f 100644 --- a/python/ql/src/qlpack.yml +++ b/python/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/python-queries -version: 1.8.6-dev +version: 1.8.6 groups: - python - queries diff --git a/ruby/ql/lib/CHANGELOG.md b/ruby/ql/lib/CHANGELOG.md index 3e1ebc8c712..d05b5a3b36a 100644 --- a/ruby/ql/lib/CHANGELOG.md +++ b/ruby/ql/lib/CHANGELOG.md @@ -1,3 +1,7 @@ +## 6.0.1 + +No user-facing changes. + ## 6.0.0 ### Breaking Changes diff --git a/ruby/ql/lib/change-notes/released/6.0.1.md b/ruby/ql/lib/change-notes/released/6.0.1.md new file mode 100644 index 00000000000..35b17912c81 --- /dev/null +++ b/ruby/ql/lib/change-notes/released/6.0.1.md @@ -0,0 +1,3 @@ +## 6.0.1 + +No user-facing changes. diff --git a/ruby/ql/lib/codeql-pack.release.yml b/ruby/ql/lib/codeql-pack.release.yml index f8c4fa43ccb..d1f3c68c812 100644 --- a/ruby/ql/lib/codeql-pack.release.yml +++ b/ruby/ql/lib/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 6.0.0 +lastReleaseVersion: 6.0.1 diff --git a/ruby/ql/lib/qlpack.yml b/ruby/ql/lib/qlpack.yml index 6957217db6d..ed80e6ead32 100644 --- a/ruby/ql/lib/qlpack.yml +++ b/ruby/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/ruby-all -version: 6.0.1-dev +version: 6.0.1 groups: ruby extractor: ruby dbscheme: ruby.dbscheme diff --git a/ruby/ql/src/CHANGELOG.md b/ruby/ql/src/CHANGELOG.md index 1df5dad19b5..3b31341bbe0 100644 --- a/ruby/ql/src/CHANGELOG.md +++ b/ruby/ql/src/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.6.6 + +No user-facing changes. + ## 1.6.5 No user-facing changes. diff --git a/ruby/ql/src/change-notes/released/1.6.6.md b/ruby/ql/src/change-notes/released/1.6.6.md new file mode 100644 index 00000000000..14281d9101b --- /dev/null +++ b/ruby/ql/src/change-notes/released/1.6.6.md @@ -0,0 +1,3 @@ +## 1.6.6 + +No user-facing changes. diff --git a/ruby/ql/src/codeql-pack.release.yml b/ruby/ql/src/codeql-pack.release.yml index 03153270557..f8e54f30a67 100644 --- a/ruby/ql/src/codeql-pack.release.yml +++ b/ruby/ql/src/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.6.5 +lastReleaseVersion: 1.6.6 diff --git a/ruby/ql/src/qlpack.yml b/ruby/ql/src/qlpack.yml index c34506fd287..1016e0294af 100644 --- a/ruby/ql/src/qlpack.yml +++ b/ruby/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/ruby-queries -version: 1.6.6-dev +version: 1.6.6 groups: - ruby - queries diff --git a/rust/ql/lib/CHANGELOG.md b/rust/ql/lib/CHANGELOG.md index d0ffbecc504..f6d806129b1 100644 --- a/rust/ql/lib/CHANGELOG.md +++ b/rust/ql/lib/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.2.17 + +No user-facing changes. + ## 0.2.16 No user-facing changes. diff --git a/rust/ql/lib/change-notes/released/0.2.17.md b/rust/ql/lib/change-notes/released/0.2.17.md new file mode 100644 index 00000000000..669582f3978 --- /dev/null +++ b/rust/ql/lib/change-notes/released/0.2.17.md @@ -0,0 +1,3 @@ +## 0.2.17 + +No user-facing changes. diff --git a/rust/ql/lib/codeql-pack.release.yml b/rust/ql/lib/codeql-pack.release.yml index 2aa64d9ed07..ba24dcb5a00 100644 --- a/rust/ql/lib/codeql-pack.release.yml +++ b/rust/ql/lib/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 0.2.16 +lastReleaseVersion: 0.2.17 diff --git a/rust/ql/lib/qlpack.yml b/rust/ql/lib/qlpack.yml index 7750d2a6a3d..74edd9e3549 100644 --- a/rust/ql/lib/qlpack.yml +++ b/rust/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/rust-all -version: 0.2.17-dev +version: 0.2.17 groups: rust extractor: rust dbscheme: rust.dbscheme diff --git a/rust/ql/src/CHANGELOG.md b/rust/ql/src/CHANGELOG.md index 5b50934a5fc..c29782a5cf4 100644 --- a/rust/ql/src/CHANGELOG.md +++ b/rust/ql/src/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.1.38 + +### Minor Analysis Improvements + +* The `rust/hard-coded-cryptographic-value` query now treats arithmetic and bitwise operations, including string append operations, as barriers. This addresses false positive results where hard-coded constants are combined with non-constant data, such as incrementing a nonce or appending variable data to a constant prefix. + ## 0.1.37 No user-facing changes. diff --git a/rust/ql/src/change-notes/2026-06-25-hard-coded-cryptographic-value-arithmetic-barrier.md b/rust/ql/src/change-notes/released/0.1.38.md similarity index 88% rename from rust/ql/src/change-notes/2026-06-25-hard-coded-cryptographic-value-arithmetic-barrier.md rename to rust/ql/src/change-notes/released/0.1.38.md index bee0af58314..42bc8be5db5 100644 --- a/rust/ql/src/change-notes/2026-06-25-hard-coded-cryptographic-value-arithmetic-barrier.md +++ b/rust/ql/src/change-notes/released/0.1.38.md @@ -1,4 +1,5 @@ ---- -category: minorAnalysis ---- +## 0.1.38 + +### Minor Analysis Improvements + * The `rust/hard-coded-cryptographic-value` query now treats arithmetic and bitwise operations, including string append operations, as barriers. This addresses false positive results where hard-coded constants are combined with non-constant data, such as incrementing a nonce or appending variable data to a constant prefix. diff --git a/rust/ql/src/codeql-pack.release.yml b/rust/ql/src/codeql-pack.release.yml index 38d6184e74c..107bf68ce69 100644 --- a/rust/ql/src/codeql-pack.release.yml +++ b/rust/ql/src/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 0.1.37 +lastReleaseVersion: 0.1.38 diff --git a/rust/ql/src/qlpack.yml b/rust/ql/src/qlpack.yml index 591c913eb69..ecc76c64f3b 100644 --- a/rust/ql/src/qlpack.yml +++ b/rust/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/rust-queries -version: 0.1.38-dev +version: 0.1.38 groups: - rust - queries diff --git a/shared/concepts/CHANGELOG.md b/shared/concepts/CHANGELOG.md index 5e5a0889e5d..b4fff81bb36 100644 --- a/shared/concepts/CHANGELOG.md +++ b/shared/concepts/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.0.27 + +No user-facing changes. + ## 0.0.26 No user-facing changes. diff --git a/shared/concepts/change-notes/released/0.0.27.md b/shared/concepts/change-notes/released/0.0.27.md new file mode 100644 index 00000000000..ff6e274427b --- /dev/null +++ b/shared/concepts/change-notes/released/0.0.27.md @@ -0,0 +1,3 @@ +## 0.0.27 + +No user-facing changes. diff --git a/shared/concepts/codeql-pack.release.yml b/shared/concepts/codeql-pack.release.yml index c576d2d7db2..dbab90d6989 100644 --- a/shared/concepts/codeql-pack.release.yml +++ b/shared/concepts/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 0.0.26 +lastReleaseVersion: 0.0.27 diff --git a/shared/concepts/qlpack.yml b/shared/concepts/qlpack.yml index d8b7fb5b554..b7f9407b10b 100644 --- a/shared/concepts/qlpack.yml +++ b/shared/concepts/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/concepts -version: 0.0.27-dev +version: 0.0.27 groups: shared library: true dependencies: diff --git a/shared/controlflow/CHANGELOG.md b/shared/controlflow/CHANGELOG.md index 80735c7276d..4d26392ac02 100644 --- a/shared/controlflow/CHANGELOG.md +++ b/shared/controlflow/CHANGELOG.md @@ -1,3 +1,7 @@ +## 2.0.37 + +No user-facing changes. + ## 2.0.36 No user-facing changes. diff --git a/shared/controlflow/change-notes/released/2.0.37.md b/shared/controlflow/change-notes/released/2.0.37.md new file mode 100644 index 00000000000..7d0b67a34af --- /dev/null +++ b/shared/controlflow/change-notes/released/2.0.37.md @@ -0,0 +1,3 @@ +## 2.0.37 + +No user-facing changes. diff --git a/shared/controlflow/codeql-pack.release.yml b/shared/controlflow/codeql-pack.release.yml index 7e4aaa0dd67..108259a7400 100644 --- a/shared/controlflow/codeql-pack.release.yml +++ b/shared/controlflow/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 2.0.36 +lastReleaseVersion: 2.0.37 diff --git a/shared/controlflow/qlpack.yml b/shared/controlflow/qlpack.yml index d14ee7d34d7..4fdac9e1929 100644 --- a/shared/controlflow/qlpack.yml +++ b/shared/controlflow/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/controlflow -version: 2.0.37-dev +version: 2.0.37 groups: shared library: true dependencies: diff --git a/shared/dataflow/CHANGELOG.md b/shared/dataflow/CHANGELOG.md index a1074cfcebb..3ebddaaf364 100644 --- a/shared/dataflow/CHANGELOG.md +++ b/shared/dataflow/CHANGELOG.md @@ -1,3 +1,7 @@ +## 2.1.9 + +No user-facing changes. + ## 2.1.8 No user-facing changes. diff --git a/shared/dataflow/change-notes/released/2.1.9.md b/shared/dataflow/change-notes/released/2.1.9.md new file mode 100644 index 00000000000..28c98cd9d95 --- /dev/null +++ b/shared/dataflow/change-notes/released/2.1.9.md @@ -0,0 +1,3 @@ +## 2.1.9 + +No user-facing changes. diff --git a/shared/dataflow/codeql-pack.release.yml b/shared/dataflow/codeql-pack.release.yml index 93b985f46e1..545d5dc7658 100644 --- a/shared/dataflow/codeql-pack.release.yml +++ b/shared/dataflow/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 2.1.8 +lastReleaseVersion: 2.1.9 diff --git a/shared/dataflow/qlpack.yml b/shared/dataflow/qlpack.yml index ae047432fc5..24de3405f90 100644 --- a/shared/dataflow/qlpack.yml +++ b/shared/dataflow/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/dataflow -version: 2.1.9-dev +version: 2.1.9 groups: shared library: true dependencies: diff --git a/shared/mad/CHANGELOG.md b/shared/mad/CHANGELOG.md index 08494880152..76889f7ca3c 100644 --- a/shared/mad/CHANGELOG.md +++ b/shared/mad/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.0.53 + +No user-facing changes. + ## 1.0.52 No user-facing changes. diff --git a/shared/mad/change-notes/released/1.0.53.md b/shared/mad/change-notes/released/1.0.53.md new file mode 100644 index 00000000000..0421d6f4cad --- /dev/null +++ b/shared/mad/change-notes/released/1.0.53.md @@ -0,0 +1,3 @@ +## 1.0.53 + +No user-facing changes. diff --git a/shared/mad/codeql-pack.release.yml b/shared/mad/codeql-pack.release.yml index ea1d2eed4d2..fb44bfeae13 100644 --- a/shared/mad/codeql-pack.release.yml +++ b/shared/mad/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.0.52 +lastReleaseVersion: 1.0.53 diff --git a/shared/mad/qlpack.yml b/shared/mad/qlpack.yml index 066ccfdf771..44bf6c5c6a6 100644 --- a/shared/mad/qlpack.yml +++ b/shared/mad/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/mad -version: 1.0.53-dev +version: 1.0.53 groups: shared library: true dependencies: diff --git a/shared/namebinding/CHANGELOG.md b/shared/namebinding/CHANGELOG.md index 59b60bad0f3..7668a5ba39d 100644 --- a/shared/namebinding/CHANGELOG.md +++ b/shared/namebinding/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.0.2 + +No user-facing changes. + ## 0.0.1 No user-facing changes. diff --git a/shared/namebinding/change-notes/released/0.0.2.md b/shared/namebinding/change-notes/released/0.0.2.md new file mode 100644 index 00000000000..5ab250998ed --- /dev/null +++ b/shared/namebinding/change-notes/released/0.0.2.md @@ -0,0 +1,3 @@ +## 0.0.2 + +No user-facing changes. diff --git a/shared/namebinding/codeql-pack.release.yml b/shared/namebinding/codeql-pack.release.yml index c6933410b71..55dc06fbd76 100644 --- a/shared/namebinding/codeql-pack.release.yml +++ b/shared/namebinding/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 0.0.1 +lastReleaseVersion: 0.0.2 diff --git a/shared/namebinding/qlpack.yml b/shared/namebinding/qlpack.yml index 15876b50208..9bd12561910 100644 --- a/shared/namebinding/qlpack.yml +++ b/shared/namebinding/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/namebinding -version: 0.0.2-dev +version: 0.0.2 groups: shared library: true dependencies: diff --git a/shared/quantum/CHANGELOG.md b/shared/quantum/CHANGELOG.md index 1652285654a..3bbb96e59a9 100644 --- a/shared/quantum/CHANGELOG.md +++ b/shared/quantum/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.0.31 + +No user-facing changes. + ## 0.0.30 No user-facing changes. diff --git a/shared/quantum/change-notes/released/0.0.31.md b/shared/quantum/change-notes/released/0.0.31.md new file mode 100644 index 00000000000..99e11d52c92 --- /dev/null +++ b/shared/quantum/change-notes/released/0.0.31.md @@ -0,0 +1,3 @@ +## 0.0.31 + +No user-facing changes. diff --git a/shared/quantum/codeql-pack.release.yml b/shared/quantum/codeql-pack.release.yml index 0c61b463bab..54b504d06ec 100644 --- a/shared/quantum/codeql-pack.release.yml +++ b/shared/quantum/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 0.0.30 +lastReleaseVersion: 0.0.31 diff --git a/shared/quantum/qlpack.yml b/shared/quantum/qlpack.yml index 546491e0768..9f8995a2ca6 100644 --- a/shared/quantum/qlpack.yml +++ b/shared/quantum/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/quantum -version: 0.0.31-dev +version: 0.0.31 groups: shared library: true dependencies: diff --git a/shared/rangeanalysis/CHANGELOG.md b/shared/rangeanalysis/CHANGELOG.md index cc127126c92..fd36e1fd7b5 100644 --- a/shared/rangeanalysis/CHANGELOG.md +++ b/shared/rangeanalysis/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.0.53 + +No user-facing changes. + ## 1.0.52 No user-facing changes. diff --git a/shared/rangeanalysis/change-notes/released/1.0.53.md b/shared/rangeanalysis/change-notes/released/1.0.53.md new file mode 100644 index 00000000000..0421d6f4cad --- /dev/null +++ b/shared/rangeanalysis/change-notes/released/1.0.53.md @@ -0,0 +1,3 @@ +## 1.0.53 + +No user-facing changes. diff --git a/shared/rangeanalysis/codeql-pack.release.yml b/shared/rangeanalysis/codeql-pack.release.yml index ea1d2eed4d2..fb44bfeae13 100644 --- a/shared/rangeanalysis/codeql-pack.release.yml +++ b/shared/rangeanalysis/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.0.52 +lastReleaseVersion: 1.0.53 diff --git a/shared/rangeanalysis/qlpack.yml b/shared/rangeanalysis/qlpack.yml index cda17399a57..b934c84900c 100644 --- a/shared/rangeanalysis/qlpack.yml +++ b/shared/rangeanalysis/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/rangeanalysis -version: 1.0.53-dev +version: 1.0.53 groups: shared library: true dependencies: diff --git a/shared/regex/CHANGELOG.md b/shared/regex/CHANGELOG.md index 488896015d6..a6d4cc0a69b 100644 --- a/shared/regex/CHANGELOG.md +++ b/shared/regex/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.0.53 + +No user-facing changes. + ## 1.0.52 No user-facing changes. diff --git a/shared/regex/change-notes/released/1.0.53.md b/shared/regex/change-notes/released/1.0.53.md new file mode 100644 index 00000000000..0421d6f4cad --- /dev/null +++ b/shared/regex/change-notes/released/1.0.53.md @@ -0,0 +1,3 @@ +## 1.0.53 + +No user-facing changes. diff --git a/shared/regex/codeql-pack.release.yml b/shared/regex/codeql-pack.release.yml index ea1d2eed4d2..fb44bfeae13 100644 --- a/shared/regex/codeql-pack.release.yml +++ b/shared/regex/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.0.52 +lastReleaseVersion: 1.0.53 diff --git a/shared/regex/qlpack.yml b/shared/regex/qlpack.yml index de6b49e8483..c64f4833d95 100644 --- a/shared/regex/qlpack.yml +++ b/shared/regex/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/regex -version: 1.0.53-dev +version: 1.0.53 groups: shared library: true dependencies: diff --git a/shared/ssa/CHANGELOG.md b/shared/ssa/CHANGELOG.md index 2348e9a484f..94914d7b024 100644 --- a/shared/ssa/CHANGELOG.md +++ b/shared/ssa/CHANGELOG.md @@ -1,3 +1,7 @@ +## 2.0.29 + +No user-facing changes. + ## 2.0.28 No user-facing changes. diff --git a/shared/ssa/change-notes/released/2.0.29.md b/shared/ssa/change-notes/released/2.0.29.md new file mode 100644 index 00000000000..c8e5d5c3d05 --- /dev/null +++ b/shared/ssa/change-notes/released/2.0.29.md @@ -0,0 +1,3 @@ +## 2.0.29 + +No user-facing changes. diff --git a/shared/ssa/codeql-pack.release.yml b/shared/ssa/codeql-pack.release.yml index ec5bd6ba369..1425cb159e4 100644 --- a/shared/ssa/codeql-pack.release.yml +++ b/shared/ssa/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 2.0.28 +lastReleaseVersion: 2.0.29 diff --git a/shared/ssa/qlpack.yml b/shared/ssa/qlpack.yml index 67bed21c679..d92b095c581 100644 --- a/shared/ssa/qlpack.yml +++ b/shared/ssa/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/ssa -version: 2.0.29-dev +version: 2.0.29 groups: shared library: true dependencies: diff --git a/shared/threat-models/CHANGELOG.md b/shared/threat-models/CHANGELOG.md index 1b79dbf69e2..676a2dc0189 100644 --- a/shared/threat-models/CHANGELOG.md +++ b/shared/threat-models/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.0.53 + +No user-facing changes. + ## 1.0.52 No user-facing changes. diff --git a/shared/threat-models/change-notes/released/1.0.53.md b/shared/threat-models/change-notes/released/1.0.53.md new file mode 100644 index 00000000000..0421d6f4cad --- /dev/null +++ b/shared/threat-models/change-notes/released/1.0.53.md @@ -0,0 +1,3 @@ +## 1.0.53 + +No user-facing changes. diff --git a/shared/threat-models/codeql-pack.release.yml b/shared/threat-models/codeql-pack.release.yml index ea1d2eed4d2..fb44bfeae13 100644 --- a/shared/threat-models/codeql-pack.release.yml +++ b/shared/threat-models/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.0.52 +lastReleaseVersion: 1.0.53 diff --git a/shared/threat-models/qlpack.yml b/shared/threat-models/qlpack.yml index 9dd6aaa670a..782bde4ee5c 100644 --- a/shared/threat-models/qlpack.yml +++ b/shared/threat-models/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/threat-models -version: 1.0.53-dev +version: 1.0.53 library: true groups: shared dataExtensions: diff --git a/shared/tutorial/CHANGELOG.md b/shared/tutorial/CHANGELOG.md index cb1a4642f73..ebf4a35d7c2 100644 --- a/shared/tutorial/CHANGELOG.md +++ b/shared/tutorial/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.0.53 + +No user-facing changes. + ## 1.0.52 No user-facing changes. diff --git a/shared/tutorial/change-notes/released/1.0.53.md b/shared/tutorial/change-notes/released/1.0.53.md new file mode 100644 index 00000000000..0421d6f4cad --- /dev/null +++ b/shared/tutorial/change-notes/released/1.0.53.md @@ -0,0 +1,3 @@ +## 1.0.53 + +No user-facing changes. diff --git a/shared/tutorial/codeql-pack.release.yml b/shared/tutorial/codeql-pack.release.yml index ea1d2eed4d2..fb44bfeae13 100644 --- a/shared/tutorial/codeql-pack.release.yml +++ b/shared/tutorial/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.0.52 +lastReleaseVersion: 1.0.53 diff --git a/shared/tutorial/qlpack.yml b/shared/tutorial/qlpack.yml index db557278bd8..44f487b4b39 100644 --- a/shared/tutorial/qlpack.yml +++ b/shared/tutorial/qlpack.yml @@ -1,7 +1,7 @@ name: codeql/tutorial description: Library for the CodeQL detective tutorials, helping new users learn to write CodeQL queries. -version: 1.0.53-dev +version: 1.0.53 groups: shared library: true warnOnImplicitThis: true diff --git a/shared/typeflow/CHANGELOG.md b/shared/typeflow/CHANGELOG.md index 6e1c15f6a2a..c48ddc3d6c7 100644 --- a/shared/typeflow/CHANGELOG.md +++ b/shared/typeflow/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.0.53 + +No user-facing changes. + ## 1.0.52 No user-facing changes. diff --git a/shared/typeflow/change-notes/released/1.0.53.md b/shared/typeflow/change-notes/released/1.0.53.md new file mode 100644 index 00000000000..0421d6f4cad --- /dev/null +++ b/shared/typeflow/change-notes/released/1.0.53.md @@ -0,0 +1,3 @@ +## 1.0.53 + +No user-facing changes. diff --git a/shared/typeflow/codeql-pack.release.yml b/shared/typeflow/codeql-pack.release.yml index ea1d2eed4d2..fb44bfeae13 100644 --- a/shared/typeflow/codeql-pack.release.yml +++ b/shared/typeflow/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.0.52 +lastReleaseVersion: 1.0.53 diff --git a/shared/typeflow/qlpack.yml b/shared/typeflow/qlpack.yml index 3e904af63e3..5545bfb4422 100644 --- a/shared/typeflow/qlpack.yml +++ b/shared/typeflow/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/typeflow -version: 1.0.53-dev +version: 1.0.53 groups: shared library: true dependencies: diff --git a/shared/typeinference/CHANGELOG.md b/shared/typeinference/CHANGELOG.md index 66b8fa3444b..fd3064b1a94 100644 --- a/shared/typeinference/CHANGELOG.md +++ b/shared/typeinference/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.0.34 + +No user-facing changes. + ## 0.0.33 No user-facing changes. diff --git a/shared/typeinference/change-notes/released/0.0.34.md b/shared/typeinference/change-notes/released/0.0.34.md new file mode 100644 index 00000000000..d58affae389 --- /dev/null +++ b/shared/typeinference/change-notes/released/0.0.34.md @@ -0,0 +1,3 @@ +## 0.0.34 + +No user-facing changes. diff --git a/shared/typeinference/codeql-pack.release.yml b/shared/typeinference/codeql-pack.release.yml index dff9e7f6ea9..6fb3b6253bb 100644 --- a/shared/typeinference/codeql-pack.release.yml +++ b/shared/typeinference/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 0.0.33 +lastReleaseVersion: 0.0.34 diff --git a/shared/typeinference/qlpack.yml b/shared/typeinference/qlpack.yml index f25557f4f13..b9625bdbc6d 100644 --- a/shared/typeinference/qlpack.yml +++ b/shared/typeinference/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/typeinference -version: 0.0.34-dev +version: 0.0.34 groups: shared library: true dependencies: diff --git a/shared/typetracking/CHANGELOG.md b/shared/typetracking/CHANGELOG.md index 8a7f7ab7014..8f0a97e4a40 100644 --- a/shared/typetracking/CHANGELOG.md +++ b/shared/typetracking/CHANGELOG.md @@ -1,3 +1,7 @@ +## 2.0.37 + +No user-facing changes. + ## 2.0.36 No user-facing changes. diff --git a/shared/typetracking/change-notes/released/2.0.37.md b/shared/typetracking/change-notes/released/2.0.37.md new file mode 100644 index 00000000000..7d0b67a34af --- /dev/null +++ b/shared/typetracking/change-notes/released/2.0.37.md @@ -0,0 +1,3 @@ +## 2.0.37 + +No user-facing changes. diff --git a/shared/typetracking/codeql-pack.release.yml b/shared/typetracking/codeql-pack.release.yml index 7e4aaa0dd67..108259a7400 100644 --- a/shared/typetracking/codeql-pack.release.yml +++ b/shared/typetracking/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 2.0.36 +lastReleaseVersion: 2.0.37 diff --git a/shared/typetracking/qlpack.yml b/shared/typetracking/qlpack.yml index fd9fa8ec813..2f007b6795d 100644 --- a/shared/typetracking/qlpack.yml +++ b/shared/typetracking/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/typetracking -version: 2.0.37-dev +version: 2.0.37 groups: shared library: true dependencies: diff --git a/shared/typos/CHANGELOG.md b/shared/typos/CHANGELOG.md index 738e64b021c..72bc8c938a9 100644 --- a/shared/typos/CHANGELOG.md +++ b/shared/typos/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.0.53 + +No user-facing changes. + ## 1.0.52 No user-facing changes. diff --git a/shared/typos/change-notes/released/1.0.53.md b/shared/typos/change-notes/released/1.0.53.md new file mode 100644 index 00000000000..0421d6f4cad --- /dev/null +++ b/shared/typos/change-notes/released/1.0.53.md @@ -0,0 +1,3 @@ +## 1.0.53 + +No user-facing changes. diff --git a/shared/typos/codeql-pack.release.yml b/shared/typos/codeql-pack.release.yml index ea1d2eed4d2..fb44bfeae13 100644 --- a/shared/typos/codeql-pack.release.yml +++ b/shared/typos/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.0.52 +lastReleaseVersion: 1.0.53 diff --git a/shared/typos/qlpack.yml b/shared/typos/qlpack.yml index 9e8d3b21c01..3b6940512a6 100644 --- a/shared/typos/qlpack.yml +++ b/shared/typos/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/typos -version: 1.0.53-dev +version: 1.0.53 groups: shared library: true warnOnImplicitThis: true diff --git a/shared/util/CHANGELOG.md b/shared/util/CHANGELOG.md index 10b02218c5f..f079131d61f 100644 --- a/shared/util/CHANGELOG.md +++ b/shared/util/CHANGELOG.md @@ -1,3 +1,7 @@ +## 2.0.40 + +No user-facing changes. + ## 2.0.39 No user-facing changes. diff --git a/shared/util/change-notes/released/2.0.40.md b/shared/util/change-notes/released/2.0.40.md new file mode 100644 index 00000000000..889c7f38559 --- /dev/null +++ b/shared/util/change-notes/released/2.0.40.md @@ -0,0 +1,3 @@ +## 2.0.40 + +No user-facing changes. diff --git a/shared/util/codeql-pack.release.yml b/shared/util/codeql-pack.release.yml index 063a268e5f9..df31666e556 100644 --- a/shared/util/codeql-pack.release.yml +++ b/shared/util/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 2.0.39 +lastReleaseVersion: 2.0.40 diff --git a/shared/util/qlpack.yml b/shared/util/qlpack.yml index 2ab432b4e47..44ea768668f 100644 --- a/shared/util/qlpack.yml +++ b/shared/util/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/util -version: 2.0.40-dev +version: 2.0.40 groups: shared library: true dependencies: null diff --git a/shared/xml/CHANGELOG.md b/shared/xml/CHANGELOG.md index 4a639c1f50f..f857db7bb80 100644 --- a/shared/xml/CHANGELOG.md +++ b/shared/xml/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.0.53 + +No user-facing changes. + ## 1.0.52 No user-facing changes. diff --git a/shared/xml/change-notes/released/1.0.53.md b/shared/xml/change-notes/released/1.0.53.md new file mode 100644 index 00000000000..0421d6f4cad --- /dev/null +++ b/shared/xml/change-notes/released/1.0.53.md @@ -0,0 +1,3 @@ +## 1.0.53 + +No user-facing changes. diff --git a/shared/xml/codeql-pack.release.yml b/shared/xml/codeql-pack.release.yml index ea1d2eed4d2..fb44bfeae13 100644 --- a/shared/xml/codeql-pack.release.yml +++ b/shared/xml/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.0.52 +lastReleaseVersion: 1.0.53 diff --git a/shared/xml/qlpack.yml b/shared/xml/qlpack.yml index 37565835712..6c506e42987 100644 --- a/shared/xml/qlpack.yml +++ b/shared/xml/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/xml -version: 1.0.53-dev +version: 1.0.53 groups: shared library: true dependencies: diff --git a/shared/yaml/CHANGELOG.md b/shared/yaml/CHANGELOG.md index 69f699d7847..a674b0b998b 100644 --- a/shared/yaml/CHANGELOG.md +++ b/shared/yaml/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.0.53 + +No user-facing changes. + ## 1.0.52 No user-facing changes. diff --git a/shared/yaml/change-notes/released/1.0.53.md b/shared/yaml/change-notes/released/1.0.53.md new file mode 100644 index 00000000000..0421d6f4cad --- /dev/null +++ b/shared/yaml/change-notes/released/1.0.53.md @@ -0,0 +1,3 @@ +## 1.0.53 + +No user-facing changes. diff --git a/shared/yaml/codeql-pack.release.yml b/shared/yaml/codeql-pack.release.yml index ea1d2eed4d2..fb44bfeae13 100644 --- a/shared/yaml/codeql-pack.release.yml +++ b/shared/yaml/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.0.52 +lastReleaseVersion: 1.0.53 diff --git a/shared/yaml/qlpack.yml b/shared/yaml/qlpack.yml index 795bbb1b1a7..a9b004ad8ff 100644 --- a/shared/yaml/qlpack.yml +++ b/shared/yaml/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/yaml -version: 1.0.53-dev +version: 1.0.53 groups: shared library: true warnOnImplicitThis: true diff --git a/swift/ql/lib/CHANGELOG.md b/swift/ql/lib/CHANGELOG.md index 1d75e0d4eb1..a590169625d 100644 --- a/swift/ql/lib/CHANGELOG.md +++ b/swift/ql/lib/CHANGELOG.md @@ -1,3 +1,7 @@ +## 6.7.2 + +No user-facing changes. + ## 6.7.1 No user-facing changes. diff --git a/swift/ql/lib/change-notes/released/6.7.2.md b/swift/ql/lib/change-notes/released/6.7.2.md new file mode 100644 index 00000000000..0f421a7c8fd --- /dev/null +++ b/swift/ql/lib/change-notes/released/6.7.2.md @@ -0,0 +1,3 @@ +## 6.7.2 + +No user-facing changes. diff --git a/swift/ql/lib/codeql-pack.release.yml b/swift/ql/lib/codeql-pack.release.yml index 9512a723a32..c29ce074416 100644 --- a/swift/ql/lib/codeql-pack.release.yml +++ b/swift/ql/lib/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 6.7.1 +lastReleaseVersion: 6.7.2 diff --git a/swift/ql/lib/qlpack.yml b/swift/ql/lib/qlpack.yml index 1000e5b25b9..f4160734f51 100644 --- a/swift/ql/lib/qlpack.yml +++ b/swift/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/swift-all -version: 6.7.2-dev +version: 6.7.2 groups: swift extractor: swift dbscheme: swift.dbscheme diff --git a/swift/ql/src/CHANGELOG.md b/swift/ql/src/CHANGELOG.md index d185e3d5428..8457bde2f42 100644 --- a/swift/ql/src/CHANGELOG.md +++ b/swift/ql/src/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.3.6 + +No user-facing changes. + ## 1.3.5 ### Minor Analysis Improvements diff --git a/swift/ql/src/change-notes/released/1.3.6.md b/swift/ql/src/change-notes/released/1.3.6.md new file mode 100644 index 00000000000..ce7baecf210 --- /dev/null +++ b/swift/ql/src/change-notes/released/1.3.6.md @@ -0,0 +1,3 @@ +## 1.3.6 + +No user-facing changes. diff --git a/swift/ql/src/codeql-pack.release.yml b/swift/ql/src/codeql-pack.release.yml index 1e1845ea66d..0a0b0986311 100644 --- a/swift/ql/src/codeql-pack.release.yml +++ b/swift/ql/src/codeql-pack.release.yml @@ -1,2 +1,2 @@ --- -lastReleaseVersion: 1.3.5 +lastReleaseVersion: 1.3.6 diff --git a/swift/ql/src/qlpack.yml b/swift/ql/src/qlpack.yml index de366deabb7..b1fd127caaa 100644 --- a/swift/ql/src/qlpack.yml +++ b/swift/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/swift-queries -version: 1.3.6-dev +version: 1.3.6 groups: - swift - queries From ec83894d48760db35e3f6cc47c239880fcaedc5d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 9 Jul 2026 17:25:35 +0000 Subject: [PATCH 27/90] Post-release preparation for codeql-cli-2.26.1 --- actions/ql/lib/qlpack.yml | 2 +- actions/ql/src/qlpack.yml | 2 +- cpp/ql/lib/qlpack.yml | 2 +- cpp/ql/src/qlpack.yml | 2 +- csharp/ql/campaigns/Solorigate/lib/qlpack.yml | 2 +- csharp/ql/campaigns/Solorigate/src/qlpack.yml | 2 +- csharp/ql/lib/qlpack.yml | 2 +- csharp/ql/src/qlpack.yml | 2 +- go/ql/consistency-queries/qlpack.yml | 2 +- go/ql/lib/qlpack.yml | 2 +- go/ql/src/qlpack.yml | 2 +- java/ql/lib/qlpack.yml | 2 +- java/ql/src/qlpack.yml | 2 +- javascript/ql/lib/qlpack.yml | 2 +- javascript/ql/src/qlpack.yml | 2 +- misc/suite-helpers/qlpack.yml | 2 +- python/ql/lib/qlpack.yml | 2 +- python/ql/src/qlpack.yml | 2 +- ruby/ql/lib/qlpack.yml | 2 +- ruby/ql/src/qlpack.yml | 2 +- rust/ql/lib/qlpack.yml | 2 +- rust/ql/src/qlpack.yml | 2 +- shared/concepts/qlpack.yml | 2 +- shared/controlflow/qlpack.yml | 2 +- shared/dataflow/qlpack.yml | 2 +- shared/mad/qlpack.yml | 2 +- shared/namebinding/qlpack.yml | 2 +- shared/quantum/qlpack.yml | 2 +- shared/rangeanalysis/qlpack.yml | 2 +- shared/regex/qlpack.yml | 2 +- shared/ssa/qlpack.yml | 2 +- shared/threat-models/qlpack.yml | 2 +- shared/tutorial/qlpack.yml | 2 +- shared/typeflow/qlpack.yml | 2 +- shared/typeinference/qlpack.yml | 2 +- shared/typetracking/qlpack.yml | 2 +- shared/typos/qlpack.yml | 2 +- shared/util/qlpack.yml | 2 +- shared/xml/qlpack.yml | 2 +- shared/yaml/qlpack.yml | 2 +- swift/ql/lib/qlpack.yml | 2 +- swift/ql/src/qlpack.yml | 2 +- 42 files changed, 42 insertions(+), 42 deletions(-) diff --git a/actions/ql/lib/qlpack.yml b/actions/ql/lib/qlpack.yml index 77ecbfaa8f1..fec6a9446d0 100644 --- a/actions/ql/lib/qlpack.yml +++ b/actions/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/actions-all -version: 0.4.39 +version: 0.4.40-dev library: true warnOnImplicitThis: true dependencies: diff --git a/actions/ql/src/qlpack.yml b/actions/ql/src/qlpack.yml index b623ff9b903..2d53972a2ea 100644 --- a/actions/ql/src/qlpack.yml +++ b/actions/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/actions-queries -version: 0.6.31 +version: 0.6.32-dev library: false warnOnImplicitThis: true groups: [actions, queries] diff --git a/cpp/ql/lib/qlpack.yml b/cpp/ql/lib/qlpack.yml index 5f4b92a191f..73226a267c9 100644 --- a/cpp/ql/lib/qlpack.yml +++ b/cpp/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/cpp-all -version: 12.0.0 +version: 12.0.1-dev groups: cpp dbscheme: semmlecode.cpp.dbscheme extractor: cpp diff --git a/cpp/ql/src/qlpack.yml b/cpp/ql/src/qlpack.yml index 1d1a9206a28..b193e3ea360 100644 --- a/cpp/ql/src/qlpack.yml +++ b/cpp/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/cpp-queries -version: 1.7.0 +version: 1.7.1-dev groups: - cpp - queries diff --git a/csharp/ql/campaigns/Solorigate/lib/qlpack.yml b/csharp/ql/campaigns/Solorigate/lib/qlpack.yml index 9fb3427e685..e33ebfaa5cc 100644 --- a/csharp/ql/campaigns/Solorigate/lib/qlpack.yml +++ b/csharp/ql/campaigns/Solorigate/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/csharp-solorigate-all -version: 1.7.70 +version: 1.7.71-dev groups: - csharp - solorigate diff --git a/csharp/ql/campaigns/Solorigate/src/qlpack.yml b/csharp/ql/campaigns/Solorigate/src/qlpack.yml index 4b55c941207..b25077c1667 100644 --- a/csharp/ql/campaigns/Solorigate/src/qlpack.yml +++ b/csharp/ql/campaigns/Solorigate/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/csharp-solorigate-queries -version: 1.7.70 +version: 1.7.71-dev groups: - csharp - solorigate diff --git a/csharp/ql/lib/qlpack.yml b/csharp/ql/lib/qlpack.yml index 336a7c6be73..0a034f90073 100644 --- a/csharp/ql/lib/qlpack.yml +++ b/csharp/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/csharp-all -version: 7.1.0 +version: 7.1.1-dev groups: csharp dbscheme: semmlecode.csharp.dbscheme extractor: csharp diff --git a/csharp/ql/src/qlpack.yml b/csharp/ql/src/qlpack.yml index b243e8fb928..ee88fc20f70 100644 --- a/csharp/ql/src/qlpack.yml +++ b/csharp/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/csharp-queries -version: 1.8.0 +version: 1.8.1-dev groups: - csharp - queries diff --git a/go/ql/consistency-queries/qlpack.yml b/go/ql/consistency-queries/qlpack.yml index d7752bb1bf6..d5e7e5af81e 100644 --- a/go/ql/consistency-queries/qlpack.yml +++ b/go/ql/consistency-queries/qlpack.yml @@ -1,5 +1,5 @@ name: codeql-go-consistency-queries -version: 1.0.53 +version: 1.0.54-dev groups: - go - queries diff --git a/go/ql/lib/qlpack.yml b/go/ql/lib/qlpack.yml index 1573f5b2ad8..dd21ea23eec 100644 --- a/go/ql/lib/qlpack.yml +++ b/go/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/go-all -version: 7.2.1 +version: 7.2.2-dev groups: go dbscheme: go.dbscheme extractor: go diff --git a/go/ql/src/qlpack.yml b/go/ql/src/qlpack.yml index 98b0c2cf12c..68359b55c92 100644 --- a/go/ql/src/qlpack.yml +++ b/go/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/go-queries -version: 1.6.6 +version: 1.6.7-dev groups: - go - queries diff --git a/java/ql/lib/qlpack.yml b/java/ql/lib/qlpack.yml index a3664696b9d..09b4eb797c7 100644 --- a/java/ql/lib/qlpack.yml +++ b/java/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/java-all -version: 9.2.1 +version: 9.2.2-dev groups: java dbscheme: config/semmlecode.dbscheme extractor: java diff --git a/java/ql/src/qlpack.yml b/java/ql/src/qlpack.yml index 2c49d49c617..4dc88f1fa1c 100644 --- a/java/ql/src/qlpack.yml +++ b/java/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/java-queries -version: 1.11.6 +version: 1.11.7-dev groups: - java - queries diff --git a/javascript/ql/lib/qlpack.yml b/javascript/ql/lib/qlpack.yml index 3ffde4f1d31..9ffe3dc1a05 100644 --- a/javascript/ql/lib/qlpack.yml +++ b/javascript/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/javascript-all -version: 2.8.1 +version: 2.8.2-dev groups: javascript dbscheme: semmlecode.javascript.dbscheme extractor: javascript diff --git a/javascript/ql/src/qlpack.yml b/javascript/ql/src/qlpack.yml index 3a88e0c6b13..56c7b6675e1 100644 --- a/javascript/ql/src/qlpack.yml +++ b/javascript/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/javascript-queries -version: 2.4.1 +version: 2.4.2-dev groups: - javascript - queries diff --git a/misc/suite-helpers/qlpack.yml b/misc/suite-helpers/qlpack.yml index 248b9c7207c..38df6eb9512 100644 --- a/misc/suite-helpers/qlpack.yml +++ b/misc/suite-helpers/qlpack.yml @@ -1,4 +1,4 @@ name: codeql/suite-helpers -version: 1.0.53 +version: 1.0.54-dev groups: shared warnOnImplicitThis: true diff --git a/python/ql/lib/qlpack.yml b/python/ql/lib/qlpack.yml index 837f6c437dd..415f397df57 100644 --- a/python/ql/lib/qlpack.yml +++ b/python/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/python-all -version: 7.2.1 +version: 7.2.2-dev groups: python dbscheme: semmlecode.python.dbscheme extractor: python diff --git a/python/ql/src/qlpack.yml b/python/ql/src/qlpack.yml index 5e0040f834f..703a6ecb6f5 100644 --- a/python/ql/src/qlpack.yml +++ b/python/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/python-queries -version: 1.8.6 +version: 1.8.7-dev groups: - python - queries diff --git a/ruby/ql/lib/qlpack.yml b/ruby/ql/lib/qlpack.yml index ed80e6ead32..4ec86b3f63b 100644 --- a/ruby/ql/lib/qlpack.yml +++ b/ruby/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/ruby-all -version: 6.0.1 +version: 6.0.2-dev groups: ruby extractor: ruby dbscheme: ruby.dbscheme diff --git a/ruby/ql/src/qlpack.yml b/ruby/ql/src/qlpack.yml index 1016e0294af..2b01f60bc65 100644 --- a/ruby/ql/src/qlpack.yml +++ b/ruby/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/ruby-queries -version: 1.6.6 +version: 1.6.7-dev groups: - ruby - queries diff --git a/rust/ql/lib/qlpack.yml b/rust/ql/lib/qlpack.yml index 74edd9e3549..94135af275f 100644 --- a/rust/ql/lib/qlpack.yml +++ b/rust/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/rust-all -version: 0.2.17 +version: 0.2.18-dev groups: rust extractor: rust dbscheme: rust.dbscheme diff --git a/rust/ql/src/qlpack.yml b/rust/ql/src/qlpack.yml index ecc76c64f3b..7fdf25e7122 100644 --- a/rust/ql/src/qlpack.yml +++ b/rust/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/rust-queries -version: 0.1.38 +version: 0.1.39-dev groups: - rust - queries diff --git a/shared/concepts/qlpack.yml b/shared/concepts/qlpack.yml index b7f9407b10b..fae2ee68178 100644 --- a/shared/concepts/qlpack.yml +++ b/shared/concepts/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/concepts -version: 0.0.27 +version: 0.0.28-dev groups: shared library: true dependencies: diff --git a/shared/controlflow/qlpack.yml b/shared/controlflow/qlpack.yml index 4fdac9e1929..f18fc3f099b 100644 --- a/shared/controlflow/qlpack.yml +++ b/shared/controlflow/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/controlflow -version: 2.0.37 +version: 2.0.38-dev groups: shared library: true dependencies: diff --git a/shared/dataflow/qlpack.yml b/shared/dataflow/qlpack.yml index 24de3405f90..4c5e24dafcb 100644 --- a/shared/dataflow/qlpack.yml +++ b/shared/dataflow/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/dataflow -version: 2.1.9 +version: 2.1.10-dev groups: shared library: true dependencies: diff --git a/shared/mad/qlpack.yml b/shared/mad/qlpack.yml index 44bf6c5c6a6..79f075ca1f0 100644 --- a/shared/mad/qlpack.yml +++ b/shared/mad/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/mad -version: 1.0.53 +version: 1.0.54-dev groups: shared library: true dependencies: diff --git a/shared/namebinding/qlpack.yml b/shared/namebinding/qlpack.yml index 9bd12561910..2e7acee2ef8 100644 --- a/shared/namebinding/qlpack.yml +++ b/shared/namebinding/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/namebinding -version: 0.0.2 +version: 0.0.3-dev groups: shared library: true dependencies: diff --git a/shared/quantum/qlpack.yml b/shared/quantum/qlpack.yml index 9f8995a2ca6..2b3c84cacee 100644 --- a/shared/quantum/qlpack.yml +++ b/shared/quantum/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/quantum -version: 0.0.31 +version: 0.0.32-dev groups: shared library: true dependencies: diff --git a/shared/rangeanalysis/qlpack.yml b/shared/rangeanalysis/qlpack.yml index b934c84900c..e1368aa2345 100644 --- a/shared/rangeanalysis/qlpack.yml +++ b/shared/rangeanalysis/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/rangeanalysis -version: 1.0.53 +version: 1.0.54-dev groups: shared library: true dependencies: diff --git a/shared/regex/qlpack.yml b/shared/regex/qlpack.yml index c64f4833d95..3de28f64f28 100644 --- a/shared/regex/qlpack.yml +++ b/shared/regex/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/regex -version: 1.0.53 +version: 1.0.54-dev groups: shared library: true dependencies: diff --git a/shared/ssa/qlpack.yml b/shared/ssa/qlpack.yml index d92b095c581..be9a88a001a 100644 --- a/shared/ssa/qlpack.yml +++ b/shared/ssa/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/ssa -version: 2.0.29 +version: 2.0.30-dev groups: shared library: true dependencies: diff --git a/shared/threat-models/qlpack.yml b/shared/threat-models/qlpack.yml index 782bde4ee5c..b6341595742 100644 --- a/shared/threat-models/qlpack.yml +++ b/shared/threat-models/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/threat-models -version: 1.0.53 +version: 1.0.54-dev library: true groups: shared dataExtensions: diff --git a/shared/tutorial/qlpack.yml b/shared/tutorial/qlpack.yml index 44f487b4b39..f1cc9f375cf 100644 --- a/shared/tutorial/qlpack.yml +++ b/shared/tutorial/qlpack.yml @@ -1,7 +1,7 @@ name: codeql/tutorial description: Library for the CodeQL detective tutorials, helping new users learn to write CodeQL queries. -version: 1.0.53 +version: 1.0.54-dev groups: shared library: true warnOnImplicitThis: true diff --git a/shared/typeflow/qlpack.yml b/shared/typeflow/qlpack.yml index 5545bfb4422..c69f8203a95 100644 --- a/shared/typeflow/qlpack.yml +++ b/shared/typeflow/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/typeflow -version: 1.0.53 +version: 1.0.54-dev groups: shared library: true dependencies: diff --git a/shared/typeinference/qlpack.yml b/shared/typeinference/qlpack.yml index b9625bdbc6d..0d181eacd02 100644 --- a/shared/typeinference/qlpack.yml +++ b/shared/typeinference/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/typeinference -version: 0.0.34 +version: 0.0.35-dev groups: shared library: true dependencies: diff --git a/shared/typetracking/qlpack.yml b/shared/typetracking/qlpack.yml index 2f007b6795d..fe58d417556 100644 --- a/shared/typetracking/qlpack.yml +++ b/shared/typetracking/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/typetracking -version: 2.0.37 +version: 2.0.38-dev groups: shared library: true dependencies: diff --git a/shared/typos/qlpack.yml b/shared/typos/qlpack.yml index 3b6940512a6..e6465e71734 100644 --- a/shared/typos/qlpack.yml +++ b/shared/typos/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/typos -version: 1.0.53 +version: 1.0.54-dev groups: shared library: true warnOnImplicitThis: true diff --git a/shared/util/qlpack.yml b/shared/util/qlpack.yml index 44ea768668f..7c1589ce7c6 100644 --- a/shared/util/qlpack.yml +++ b/shared/util/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/util -version: 2.0.40 +version: 2.0.41-dev groups: shared library: true dependencies: null diff --git a/shared/xml/qlpack.yml b/shared/xml/qlpack.yml index 6c506e42987..bb3f29d1718 100644 --- a/shared/xml/qlpack.yml +++ b/shared/xml/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/xml -version: 1.0.53 +version: 1.0.54-dev groups: shared library: true dependencies: diff --git a/shared/yaml/qlpack.yml b/shared/yaml/qlpack.yml index a9b004ad8ff..e169087c624 100644 --- a/shared/yaml/qlpack.yml +++ b/shared/yaml/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/yaml -version: 1.0.53 +version: 1.0.54-dev groups: shared library: true warnOnImplicitThis: true diff --git a/swift/ql/lib/qlpack.yml b/swift/ql/lib/qlpack.yml index f4160734f51..807adcb196f 100644 --- a/swift/ql/lib/qlpack.yml +++ b/swift/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/swift-all -version: 6.7.2 +version: 6.7.3-dev groups: swift extractor: swift dbscheme: swift.dbscheme diff --git a/swift/ql/src/qlpack.yml b/swift/ql/src/qlpack.yml index b1fd127caaa..050ce125a8e 100644 --- a/swift/ql/src/qlpack.yml +++ b/swift/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/swift-queries -version: 1.3.6 +version: 1.3.7-dev groups: - swift - queries From 643080208b38f793b75d3d390280f26b57b2f8f3 Mon Sep 17 00:00:00 2001 From: Kristen Newbury Date: Thu, 9 Jul 2026 17:01:29 -0400 Subject: [PATCH 28/90] Add UntrustedCheckoutTOCTOUX ControlCheck model fix --- actions/ql/lib/codeql/actions/security/ControlChecks.qll | 4 +--- .../src/Security/CWE-367/UntrustedCheckoutTOCTOUCritical.ql | 3 ++- .../Security/CWE-367/UntrustedCheckoutTOCTOUCritical.expected | 2 ++ 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/actions/ql/lib/codeql/actions/security/ControlChecks.qll b/actions/ql/lib/codeql/actions/security/ControlChecks.qll index 41f512abbc3..900f62556b0 100644 --- a/actions/ql/lib/codeql/actions/security/ControlChecks.qll +++ b/actions/ql/lib/codeql/actions/security/ControlChecks.qll @@ -144,9 +144,7 @@ class EnvironmentCheck extends ControlCheck instanceof Environment { // Environment checks are not effective against any mutable attacks // they do actually protect against untrusted code execution (sha) override predicate protectsCategoryAndEvent(string category, string event) { - event = actor_is_attacker_event() and category = any_category() - or - event = actor_not_attacker_event() and category = non_toctou_category() + event = [actor_is_attacker_event(), actor_not_attacker_event()] and category = non_toctou_category() } } diff --git a/actions/ql/src/Security/CWE-367/UntrustedCheckoutTOCTOUCritical.ql b/actions/ql/src/Security/CWE-367/UntrustedCheckoutTOCTOUCritical.ql index 2aacf20b35f..14a6c9a4c56 100644 --- a/actions/ql/src/Security/CWE-367/UntrustedCheckoutTOCTOUCritical.ql +++ b/actions/ql/src/Security/CWE-367/UntrustedCheckoutTOCTOUCritical.ql @@ -23,7 +23,8 @@ where // the checked-out code may lead to arbitrary code execution checkout.getAFollowingStep() = step and // the checkout occurs in a privileged context - inPrivilegedContext(checkout, event) and + inPrivilegedContext(checkout, event) + and // the mutable checkout step is protected by an Insufficient access check exists(ControlCheck check1 | check1.protects(checkout, event, "untrusted-checkout")) and not exists(ControlCheck check2 | check2.protects(checkout, event, "untrusted-checkout-toctou")) diff --git a/actions/ql/test/query-tests/Security/CWE-367/UntrustedCheckoutTOCTOUCritical.expected b/actions/ql/test/query-tests/Security/CWE-367/UntrustedCheckoutTOCTOUCritical.expected index da66ff822a3..4f0bd967c2b 100644 --- a/actions/ql/test/query-tests/Security/CWE-367/UntrustedCheckoutTOCTOUCritical.expected +++ b/actions/ql/test/query-tests/Security/CWE-367/UntrustedCheckoutTOCTOUCritical.expected @@ -99,6 +99,8 @@ edges #select | .github/workflows/comment.yml:58:9:60:2 | Run Step | .github/workflows/comment.yml:54:9:58:6 | Uses Step | .github/workflows/comment.yml:58:9:60:2 | Run Step | Insufficient protection against execution of untrusted code on a privileged workflow ($@). | .github/workflows/comment.yml:4:3:4:15 | issue_comment | issue_comment | | .github/workflows/comment.yml:68:9:68:43 | Run Step | .github/workflows/comment.yml:64:9:68:6 | Uses Step | .github/workflows/comment.yml:68:9:68:43 | Run Step | Insufficient protection against execution of untrusted code on a privileged workflow ($@). | .github/workflows/comment.yml:4:3:4:15 | issue_comment | issue_comment | +| .github/workflows/deployment1.yml:27:10:30:7 | Run Step | .github/workflows/deployment1.yml:16:10:22:7 | Uses Step | .github/workflows/deployment1.yml:27:10:30:7 | Run Step | Insufficient protection against execution of untrusted code on a privileged workflow ($@). | .github/workflows/deployment1.yml:5:3:5:21 | pull_request_target | pull_request_target | +| .github/workflows/deployment1.yml:30:10:31:53 | Run Step | .github/workflows/deployment1.yml:16:10:22:7 | Uses Step | .github/workflows/deployment1.yml:30:10:31:53 | Run Step | Insufficient protection against execution of untrusted code on a privileged workflow ($@). | .github/workflows/deployment1.yml:5:3:5:21 | pull_request_target | pull_request_target | | .github/workflows/test0.yml:58:9:60:2 | Run Step | .github/workflows/test0.yml:54:9:58:6 | Uses Step | .github/workflows/test0.yml:58:9:60:2 | Run Step | Insufficient protection against execution of untrusted code on a privileged workflow ($@). | .github/workflows/test0.yml:4:3:4:15 | issue_comment | issue_comment | | .github/workflows/test0.yml:68:9:68:43 | Run Step | .github/workflows/test0.yml:64:9:68:6 | Uses Step | .github/workflows/test0.yml:68:9:68:43 | Run Step | Insufficient protection against execution of untrusted code on a privileged workflow ($@). | .github/workflows/test0.yml:4:3:4:15 | issue_comment | issue_comment | | .github/workflows/test4.yml:85:7:88:54 | Uses Step | .github/workflows/test4.yml:79:7:85:4 | Uses Step | .github/workflows/test4.yml:85:7:88:54 | Uses Step | Insufficient protection against execution of untrusted code on a privileged workflow ($@). | .github/workflows/test4.yml:5:3:5:15 | issue_comment | issue_comment | From 8ff671f99ddbc6bb6d0ad27c998b75cf80468035 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 00:44:25 +0000 Subject: [PATCH 29/90] Add changed framework coverage reports --- .../library-coverage/coverage.csv | 599 +++++++++--------- .../library-coverage/coverage.rst | 4 +- 2 files changed, 302 insertions(+), 301 deletions(-) diff --git a/java/documentation/library-coverage/coverage.csv b/java/documentation/library-coverage/coverage.csv index 75c6ca1da78..39bb4b37e59 100644 --- a/java/documentation/library-coverage/coverage.csv +++ b/java/documentation/library-coverage/coverage.csv @@ -1,299 +1,300 @@ -package,sink,source,summary,sink:bean-validation,sink:command-injection,sink:credentials-key,sink:credentials-password,sink:credentials-username,sink:encryption-iv,sink:encryption-salt,sink:environment-injection,sink:file-content-store,sink:fragment-injection,sink:groovy-injection,sink:hostname-verification,sink:html-injection,sink:information-leak,sink:intent-redirection,sink:jexl-injection,sink:jndi-injection,sink:js-injection,sink:ldap-injection,sink:log-injection,sink:mvel-injection,sink:notification,sink:ognl-injection,sink:path-injection,sink:path-injection[read],sink:pending-intents,sink:regex-use,sink:regex-use[-1],sink:regex-use[0],sink:regex-use[],sink:regex-use[f-1],sink:regex-use[f1],sink:regex-use[f],sink:request-forgery,sink:response-splitting,sink:sql-injection,sink:template-injection,sink:trust-boundary-violation,sink:unsafe-deserialization,sink:url-forward,sink:url-redirection,sink:xpath-injection,sink:xslt-injection,source:android-external-storage-dir,source:commandargs,source:contentprovider,source:database,source:environment,source:file,source:remote,summary:taint,summary:value -actions.osgi,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6, -android.app,77,,103,,,,,,,,,,11,,,,,7,,,,,,,42,,,,17,,,,,,,,,,,,,,,,,,,,,,,,,18,85 -android.content,24,31,154,,,,,,,,,,,,,,,16,,,,,,,,,,,,,,,,,,,,,8,,,,,,,,4,,27,,,,,63,91 -android.database,59,,41,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,59,,,,,,,,,,,,,,,41, -android.net,,,60,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,45,15 -android.os,1,2,122,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,2,,,,,,,41,81 -android.support.v4.app,11,,,,,,,,,,,,11,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -android.util,6,16,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,16,, -android.webkit,3,2,,,,,,,,,,,,,,2,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,, -android.widget,,1,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1, -androidx.core.app,47,,95,,,,,,,,,,,,,,,,,,,,,,41,,,,6,,,,,,,,,,,,,,,,,,,,,,,,,12,83 -androidx.fragment.app,11,,,,,,,,,,,,11,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -androidx.slice,2,5,88,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,,,5,,,,,27,61 -antlr,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, -ch.ethz.ssh2,2,,,,,,1,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -cn.hutool.core.codec,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, -com.alibaba.com.caucho.hessian.io,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,, -com.alibaba.druid.sql,1,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,1, -com.alibaba.fastjson2,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, -com.amazonaws.auth,2,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -com.auth0.jwt.algorithms,6,,,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -com.azure.identity,3,,,,,1,1,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -com.caucho.burlap.io,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,, -com.caucho.hessian.io,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,, -com.cedarsoftware.util.io,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,, -com.couchbase.client.core.env,15,,1,,,,9,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, -com.couchbase.client.java,10,,,,,,2,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,, -com.esotericsoftware.kryo.io,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, -com.esotericsoftware.kryo5.io,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, -com.esotericsoftware.yamlbeans,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,, -com.fasterxml.jackson.core,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, -com.fasterxml.jackson.databind,2,,8,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,8, -com.google.common.base,4,,87,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,1,,,,,,,,,,,,,,,,,,,,,63,24 -com.google.common.cache,,,17,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,17 -com.google.common.collect,,,553,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,551 -com.google.common.flogger,29,,,,,,,,,,,,,,,,,,,,,,29,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -com.google.common.io,10,,73,,,,,,,,,1,,,,,,,,,,,,,,,4,5,,,,,,,,,,,,,,,,,,,,,,,,,,72,1 -com.google.gson,,,52,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,38,14 -com.hubspot.jinjava,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,, -com.jcraft.jsch,5,,1,,,,2,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,1, -com.microsoft.sqlserver.jdbc,4,,,,,,2,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -com.mitchellbosecke.pebble,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,, -com.mongodb,10,,,,,,4,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -com.opensymphony.xwork2,56,,961,,,,,,,,,,,,,,,,,,,,,,,56,,,,,,,,,,,,,,,,,,,,,,,,,,,,867,94 -com.rabbitmq.client,,21,7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,21,7, -com.sshtools.j2ssh.authentication,3,,,,,,1,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -com.sun.crypto.provider,19,,,,,17,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -com.sun.jndi.ldap,4,,,,,,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -com.sun.net.httpserver,3,,,,,,1,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -com.sun.net.ssl,3,,,,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -com.sun.rowset,3,,,,,,2,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -com.sun.security.auth.module,2,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -com.sun.security.ntlm,5,,,,,,3,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -com.sun.security.sasl.digest,3,,,,,,2,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -com.thoughtworks.xstream,1,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,, -com.trilead.ssh2,13,,,,,2,4,7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -com.unboundid.ldap.sdk,17,,,,,,,,,,,,,,,,,,,,,17,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -com.zaxxer.hikari,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,, -flexjson,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 -freemarker.cache,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,, -freemarker.template,7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,7,,,,,,,,,,,,,,, -groovy.lang,26,,,,,,,,,,,,,26,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -groovy.text,1,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -groovy.util,5,,,,,,,,,,,,,5,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -hudson,75,9,2648,,4,,,,,,3,2,,,,4,,,,,,,,,,,39,17,,,,,,,,,6,,,,,,,,,,,,,,,5,4,2572,76 -io.jsonwebtoken,,2,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,4, -io.netty.bootstrap,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,,,,,,,,,,,,,,,,,, -io.netty.buffer,,,207,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,130,77 -io.netty.channel,9,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9,,,,,,,,,,,,,,,,2,, -io.netty.handler.codec,4,13,259,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,3,,,,,,,,,,,,,,,,13,143,116 -io.netty.handler.ssl,4,,,,,,,,,,,,,,,,,,,,,,,,,,2,2,,,,,,,,,,,,,,,,,,,,,,,,,,, -io.netty.handler.stream,1,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,, -io.netty.resolver,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, -io.netty.util,2,,23,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,1,,,,,,,,,,,,,,,,,21,2 -io.undertow.server.handlers.resource,1,,3,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,3, -jakarta.activation,2,,2,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,1,,,,,,,,,,,,,,,,,2, -jakarta.faces.context,4,7,,,,,,,,,,,,,,2,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,7,, -jakarta.json,,,123,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100,23 -jakarta.persistence,2,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,1, -jakarta.servlet,2,26,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,26,, -jakarta.ws.rs.client,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,, -jakarta.ws.rs.container,,9,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9,, -jakarta.ws.rs.core,2,,149,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,94,55 -jakarta.xml.bind.attachment,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,, -java.applet,,,11,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,11, -java.awt,1,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,3 -java.beans,1,,177,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,82,95 -java.io,66,1,225,,,,,,,,,22,,,,,,,,,,,,,,,29,15,,,,,,,,,,,,,,,,,,,,,,,,1,,202,23 -java.lang,38,3,790,,13,,,,,,1,,,,,,,,,,,,8,,,,2,9,,,4,,,1,,,,,,,,,,,,,,,,,3,,,510,280 -java.math,,,9,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9 -java.net,23,3,347,,,,1,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,21,,,,,,,,,,,,,,,,3,248,99 -java.nio,47,,499,,,,,,,,,5,,,,,,,,,,,,,,,25,16,,,,,,,,,1,,,,,,,,,,,,,,,,,302,197 -java.rmi,,,68,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,45,23 -java.security,21,,583,,,11,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,285,298 -java.sql,15,1,292,,,,1,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,,9,,,,,,,,,,,1,,,,274,18 -java.text,,,154,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,72,82 -java.time,,,131,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,27,104 -java.util,48,2,1340,,,,,,,,,1,,,,,,,,,,,34,,,,3,,,,,5,2,,1,2,,,,,,,,,,,,,,,2,,,558,782 -javafx.scene.web,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,, -javax.accessibility,,,63,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,28,35 -javax.activation,2,,7,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,1,,,,,,,,,,,,,,,,,7, -javax.annotation.processing,,,28,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,25,3 -javax.crypto,19,,140,,,12,3,,2,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,76,64 -javax.faces.context,4,7,,,,,,,,,,,,,,2,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,7,, -javax.imageio,1,,304,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,138,166 -javax.jms,,9,57,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9,57, -javax.json,,,123,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100,23 -javax.lang.model,,,277,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,217,60 -javax.management,2,,766,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,363,403 -javax.naming,7,,341,,,,,,,,,,,,,,,,,6,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,191,150 -javax.net,4,,136,,,,2,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,87,49 -javax.portlet,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,, -javax.print,2,,133,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,102,31 -javax.rmi.ssl,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6 -javax.script,1,,50,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,14,36 -javax.security.auth,7,,147,,,4,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,50,97 -javax.security.cert,,,5,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,5, -javax.security.sasl,,,49,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,42,7 -javax.servlet,10,29,3,,,,,,,,,,,,,,1,,,,,,,,,,,2,,,,,,,,,,3,,,2,,2,,,,,,,,,,29,3, -javax.smartcardio,,,34,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,24,10 -javax.sound.midi,,,60,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,51,9 -javax.sound.sampled,,,90,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,53,37 -javax.sql,7,,126,,,,4,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,68,58 -javax.tools,,,66,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,62,4 -javax.transaction.xa,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, -javax.validation,1,1,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,, -javax.ws.rs.client,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,, -javax.ws.rs.container,,9,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9,, -javax.ws.rs.core,3,,149,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,2,,,,,,,,,,94,55 -javax.xml.bind.attachment,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,, -javax.xml.catalog,,,12,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,11,1 -javax.xml.crypto,,,269,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,172,97 -javax.xml.datatype,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,5,1 -javax.xml.namespace,,,15,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,5,10 -javax.xml.parsers,,,37,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,35,2 -javax.xml.stream,,,221,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,201,20 -javax.xml.transform,2,,134,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,1,,,,,,,,72,62 -javax.xml.validation,,,29,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,29, -javax.xml.xpath,3,,26,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,,,,,,,,,26, -jenkins,,,523,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,500,23 -jodd.json,,,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10 -kotlin,16,,1849,,,,,,,,,,,,,,,,,,,,,,,,11,3,,,,,,,,,2,,,,,,,,,,,,,,,,,1836,13 -liquibase.database.jvm,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,, -liquibase.statement.core,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,, -net.lingala.zip4j,2,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,, -net.schmizz.sshj,4,,,,,,2,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -net.sf.json,2,,338,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,321,17 -net.sf.saxon.s9api,5,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,5,,,,,,,,, -ognl,6,,,,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -okhttp3,4,,50,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,,,,,,,,,,,,,,,,,23,27 -org.acegisecurity,,,49,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,49, -org.antlr.runtime,1,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.avro,18,19,,,,,,,,,,,,,,,,,,,,,,,,,17,,,,,,,,,,1,,,,,,,,,,,1,,,,17,1,, -org.apache.commons.codec,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6, -org.apache.commons.collections,,,800,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,17,783 -org.apache.commons.collections4,,,806,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,17,789 -org.apache.commons.compress.archivers.tar,,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4, -org.apache.commons.exec,10,,,,6,,,,,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.commons.fileupload,,11,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,11,4, -org.apache.commons.httpclient.util,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, -org.apache.commons.io,124,,570,,,,,,,,,4,,,,,,,,,,,,,,,102,3,,,,,,,,,15,,,,,,,,,,,,,,,,,556,14 -org.apache.commons.jelly,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,, -org.apache.commons.jexl2,15,,,,,,,,,,,,,,,,,,15,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.commons.jexl3,15,,,,,,,,,,,,,,,,,,15,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.commons.lang,1,,767,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,596,171 -org.apache.commons.lang3,7,,425,,,,,,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,1,,,,,,,,,,,,294,131 -org.apache.commons.logging,6,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.commons.net,13,12,,,,,2,2,,,,,,,,,,,,,,,,,,,,3,,,,,,,,,6,,,,,,,,,,,,,,,,12,, -org.apache.commons.ognl,6,,,,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.commons.text,,,272,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,220,52 -org.apache.cxf.catalog,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,, -org.apache.cxf.common.classloader,3,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,2,,,,,,,,,,,,,,,,,, -org.apache.cxf.common.jaxb,1,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.cxf.common.logging,6,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.cxf.configuration.jsse,2,,,,,,,,,,,,,,1,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.cxf.helpers,10,,,,,,,,,,,,,,,,,,,,,,,,,,5,,,,,,,,,,,,,,,,,,5,,,,,,,,,, -org.apache.cxf.resource,9,,,,,,,,,,,,,,,,,,,,,,,,,,4,,,,,,,,,,5,,,,,,,,,,,,,,,,,, -org.apache.cxf.staxutils,1,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.cxf.tools.corba.utils,4,,,,,,,,,,,,,,,,,,,,,,,,,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.cxf.tools.util,10,,,,,,,,,,,,,,,,,,,,,,,,,,10,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.cxf.transform,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,,,,,,,,, -org.apache.directory.ldap.client.api,1,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.hadoop.fs,3,,11,,,,,,,,,,,,,,,,,,,,,,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,11, -org.apache.hadoop.hive.metastore,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,,,,,,,,,,,,,,,, -org.apache.hadoop.hive.ql.exec,1,,1,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,1, -org.apache.hadoop.hive.ql.metadata,1,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.hc.client5.http.async.methods,84,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,84,,,,,,,,,,,,,,,,,, -org.apache.hc.client5.http.classic.methods,37,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,37,,,,,,,,,,,,,,,,,, -org.apache.hc.client5.http.fluent,19,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,19,,,,,,,,,,,,,,,,,, -org.apache.hc.client5.http.protocol,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 -org.apache.hc.client5.http.utils,,,7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,7, -org.apache.hc.core5.benchmark,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,, -org.apache.hc.core5.function,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, -org.apache.hc.core5.http,73,2,45,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,72,,,,,,,,,,,,,,,,2,45, -org.apache.hc.core5.net,,,18,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,18, -org.apache.hc.core5.util,,,24,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,18,6 -org.apache.hive.hcatalog.templeton,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,, -org.apache.http,53,3,117,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,51,,,,,,,,,,,,,,,,3,108,9 -org.apache.ibatis.jdbc,6,,57,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,57, -org.apache.ibatis.mapping,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, -org.apache.log4j,11,,,,,,,,,,,,,,,,,,,,,,11,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.logging.log4j,359,,8,,,,,,,,,,,,,,,,,,,,359,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,4 -org.apache.shiro.authc,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,, -org.apache.shiro.codec,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, -org.apache.shiro.jndi,1,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.shiro.mgt,1,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.sshd.client.session,3,,,,,,1,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.struts.beanvalidation.validation.interceptor,,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4, -org.apache.struts2,14,,3873,,,,,,,,,,,,,,,,,,,,,,,11,,,,,,,,,,,,,,,3,,,,,,,,,,,,,3839,34 -org.apache.tools.ant,14,,,,1,,,,,,,,,,,,,,,,,,,,,,5,8,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.apache.tools.zip,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, -org.apache.velocity.app,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,,,,,,,,,,,,,,, -org.apache.velocity.runtime,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,,,,,,,,,,,,,,, -org.codehaus.cargo.container.installer,3,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,1,,,,,,,,,,,,,,,,,, -org.codehaus.groovy.control,1,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.dom4j,20,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,20,,,,,,,,,, -org.eclipse.jetty.client,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,, -org.exolab.castor.xml,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,, -org.fusesource.leveldbjni,1,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.geogebra.web.full.main,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,, -org.gradle.api.file,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3, -org.hibernate,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10,,,,,,,,,,,,,,,, -org.ho.yaml,8,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,8,,,,,,,,,,,,, -org.influxdb,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,, -org.jabsorb,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,, -org.jboss.logging,324,,,,,,,,,,,,,,,,,,,,,,324,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.jboss.vfs,1,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.jdbi.v3.core,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,, -org.jenkins.ui.icon,,,49,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,48,1 -org.jenkins.ui.symbol,,,33,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,25,8 -org.jooq,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,, -org.json,,,236,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,198,38 -org.keycloak.models.map.storage,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,, -org.kohsuke.stapler,20,24,363,,,,,,,,,,,,,2,,,,,,,,,,,8,1,,,,,,,,,3,,,,,,1,5,,,,,,,,,24,352,11 -org.lastaflute.web,,1,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,4, -org.mvel2,16,,,,,,,,,,,,,,,,,,,,,,,16,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.openjdk.jmh.runner.options,1,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.owasp.esapi,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, -org.pac4j.jwt.config.encryption,4,,,,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.pac4j.jwt.config.signature,4,,,,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.scijava.log,13,,,,,,,,,,,,,,,,,,,,,,13,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.slf4j,55,,6,,,,,,,,,,,,,,,,,,,,55,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,4 -org.springframework.beans,,,30,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,30 -org.springframework.boot.jdbc,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,, -org.springframework.cache,,,13,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,13 -org.springframework.context,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3, -org.springframework.core.io,17,,6,,,,,,,,,,,,,,,,,,,,,,,,16,,,,,,,,,,1,,,,,,,,,,,,,,,,,6, -org.springframework.data.repository,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 -org.springframework.http,14,,77,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,14,,,,,,,,,,,,,,,,,67,10 -org.springframework.jdbc.core,19,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,19,,,,,,,,,,,,,,,, -org.springframework.jdbc.datasource,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,,,,,,,,,,,,,,,,,, -org.springframework.jdbc.object,9,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9,,,,,,,,,,,,,,,, -org.springframework.jndi,1,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.springframework.ldap,47,,,,,,,,,,,,,,,,,,,33,,14,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.springframework.security.core.userdetails,2,,,,,,1,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -org.springframework.security.web.savedrequest,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,, -org.springframework.ui,,,32,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,32 -org.springframework.util,10,,142,,,,,,,,,,,,,,,,,,,,,,,,9,1,,,,,,,,,,,,,,,,,,,,,,,,,,90,52 -org.springframework.validation,,,13,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,13, -org.springframework.web.client,13,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,13,,,,,,,,,,,,,,,,3,, -org.springframework.web.context.request,,8,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,8,, -org.springframework.web.multipart,,12,12,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,12,12, -org.springframework.web.portlet,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,, -org.springframework.web.reactive.function.client,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,,,,,,,,,,,,,,,,,, -org.springframework.web.servlet,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,, -org.springframework.web.socket,,8,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,8,6, -org.springframework.web.util,,9,159,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9,134,25 -org.thymeleaf,2,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,2, -org.xml.sax,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, -org.xmlpull.v1,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,, -org.yaml.snakeyaml,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, -play.libs.ws,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,, -play.mvc,1,13,24,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,13,24, -ratpack.core.form,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3, -ratpack.core.handling,,6,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,4, -ratpack.core.http,,10,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10,10, -ratpack.exec,,,48,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,48 -ratpack.form,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3, -ratpack.func,,,35,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,35 -ratpack.handling,,6,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,4, -ratpack.http,,10,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10,10, -ratpack.util,,,35,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,35 -retrofit2,1,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,1, -software.amazon.awssdk.transfer.s3.model,8,,,,,,,,,,,,,,,,,,,,,,,,,,8,,,,,,,,,,,,,,,,,,,,,,,,,,,, -sun.jvmstat.perfdata.monitor.protocol.local,3,,,,,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -sun.jvmstat.perfdata.monitor.protocol.rmi,1,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -sun.misc,3,,,,,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -sun.net.ftp,5,,,,,,2,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -sun.net.www.protocol.http,3,,,,,,2,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -sun.security.acl,1,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -sun.security.jgss.krb5,2,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -sun.security.krb5,9,,,,,3,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -sun.security.pkcs,4,,,,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -sun.security.pkcs11,3,,,,,1,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -sun.security.provider,2,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -sun.security.ssl,3,,,,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -sun.security.x509,1,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -sun.tools.jconsole,28,,,,,,13,15,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +package,sink,source,summary,sink:bean-validation,sink:command-injection,sink:credentials-key,sink:credentials-password,sink:credentials-username,sink:encryption-iv,sink:encryption-salt,sink:environment-injection,sink:file-content-store,sink:fragment-injection,sink:groovy-injection,sink:hostname-verification,sink:html-injection,sink:information-leak,sink:intent-redirection,sink:jexl-injection,sink:jndi-injection,sink:js-injection,sink:ldap-injection,sink:log-injection,sink:mvel-injection,sink:notification,sink:ognl-injection,sink:path-injection,sink:path-injection[read],sink:pending-intents,sink:regex-use,sink:regex-use[-1],sink:regex-use[0],sink:regex-use[],sink:regex-use[f-1],sink:regex-use[f1],sink:regex-use[f],sink:request-forgery,sink:response-splitting,sink:sql-injection,sink:template-injection,sink:trust-boundary-violation,sink:unsafe-deserialization,sink:url-forward,sink:url-redirection,sink:xpath-injection,sink:xslt-injection,sink:xxe,source:android-external-storage-dir,source:commandargs,source:contentprovider,source:database,source:environment,source:file,source:remote,summary:taint,summary:value +actions.osgi,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6, +android.app,77,,103,,,,,,,,,,11,,,,,7,,,,,,,42,,,,17,,,,,,,,,,,,,,,,,,,,,,,,,,18,85 +android.content,24,31,154,,,,,,,,,,,,,,,16,,,,,,,,,,,,,,,,,,,,,8,,,,,,,,,4,,27,,,,,63,91 +android.database,59,,41,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,59,,,,,,,,,,,,,,,,41, +android.net,,,60,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,45,15 +android.os,1,2,122,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,2,,,,,,,41,81 +android.support.v4.app,11,,,,,,,,,,,,11,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +android.util,6,16,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,16,, +android.webkit,3,2,,,,,,,,,,,,,,2,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,, +android.widget,,1,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1, +androidx.core.app,47,,95,,,,,,,,,,,,,,,,,,,,,,41,,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,12,83 +androidx.fragment.app,11,,,,,,,,,,,,11,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +androidx.slice,2,5,88,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,5,,,,,27,61 +antlr,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, +ch.ethz.ssh2,2,,,,,,1,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +cn.hutool.core.codec,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, +com.alibaba.com.caucho.hessian.io,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,, +com.alibaba.druid.sql,1,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,1, +com.alibaba.fastjson2,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, +com.amazonaws.auth,2,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +com.auth0.jwt.algorithms,6,,,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +com.azure.identity,3,,,,,1,1,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +com.caucho.burlap.io,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,, +com.caucho.hessian.io,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,, +com.cedarsoftware.util.io,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,, +com.couchbase.client.core.env,15,,1,,,,9,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, +com.couchbase.client.java,10,,,,,,2,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,, +com.esotericsoftware.kryo.io,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, +com.esotericsoftware.kryo5.io,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, +com.esotericsoftware.yamlbeans,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,, +com.fasterxml.jackson.core,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, +com.fasterxml.jackson.databind,2,,8,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,8, +com.google.common.base,4,,87,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,1,,,,,,,,,,,,,,,,,,,,,,63,24 +com.google.common.cache,,,17,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,17 +com.google.common.collect,,,553,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,551 +com.google.common.flogger,29,,,,,,,,,,,,,,,,,,,,,,29,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +com.google.common.io,10,,73,,,,,,,,,1,,,,,,,,,,,,,,,4,5,,,,,,,,,,,,,,,,,,,,,,,,,,,72,1 +com.google.gson,,,52,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,38,14 +com.hubspot.jinjava,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,, +com.jcraft.jsch,5,,1,,,,2,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,1, +com.microsoft.sqlserver.jdbc,4,,,,,,2,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +com.mitchellbosecke.pebble,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,, +com.mongodb,10,,,,,,4,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +com.opensymphony.xwork2,56,,961,,,,,,,,,,,,,,,,,,,,,,,56,,,,,,,,,,,,,,,,,,,,,,,,,,,,,867,94 +com.rabbitmq.client,,21,7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,21,7, +com.sshtools.j2ssh.authentication,3,,,,,,1,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +com.sun.crypto.provider,19,,,,,17,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +com.sun.jndi.ldap,4,,,,,,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +com.sun.net.httpserver,3,,,,,,1,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +com.sun.net.ssl,3,,,,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +com.sun.rowset,3,,,,,,2,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +com.sun.security.auth.module,2,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +com.sun.security.ntlm,5,,,,,,3,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +com.sun.security.sasl.digest,3,,,,,,2,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +com.thoughtworks.xstream,1,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,, +com.trilead.ssh2,13,,,,,2,4,7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +com.unboundid.ldap.sdk,17,,,,,,,,,,,,,,,,,,,,,17,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +com.zaxxer.hikari,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,, +flexjson,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 +freemarker.cache,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,, +freemarker.template,7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,7,,,,,,,,,,,,,,,, +groovy.lang,26,,,,,,,,,,,,,26,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +groovy.text,1,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +groovy.util,5,,,,,,,,,,,,,5,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +hudson,75,9,2648,,4,,,,,,3,2,,,,4,,,,,,,,,,,39,17,,,,,,,,,6,,,,,,,,,,,,,,,,5,4,2572,76 +io.jsonwebtoken,,2,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,4, +io.netty.bootstrap,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,,,,,,,,,,,,,,,,,,, +io.netty.buffer,,,207,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,130,77 +io.netty.channel,9,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9,,,,,,,,,,,,,,,,,2,, +io.netty.handler.codec,4,13,259,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,3,,,,,,,,,,,,,,,,,13,143,116 +io.netty.handler.ssl,4,,,,,,,,,,,,,,,,,,,,,,,,,,2,2,,,,,,,,,,,,,,,,,,,,,,,,,,,, +io.netty.handler.stream,1,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,, +io.netty.resolver,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, +io.netty.util,2,,23,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,1,,,,,,,,,,,,,,,,,,21,2 +io.undertow.server.handlers.resource,1,,3,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,3, +jakarta.activation,2,,2,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,1,,,,,,,,,,,,,,,,,,2, +jakarta.faces.context,4,7,,,,,,,,,,,,,,2,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,7,, +jakarta.json,,,123,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100,23 +jakarta.persistence,2,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,1, +jakarta.servlet,2,26,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,26,, +jakarta.ws.rs.client,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,, +jakarta.ws.rs.container,,9,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9,, +jakarta.ws.rs.core,2,,149,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,94,55 +jakarta.xml.bind.attachment,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,, +java.applet,,,11,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,11, +java.awt,1,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,3 +java.beans,1,,177,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,82,95 +java.io,66,1,225,,,,,,,,,22,,,,,,,,,,,,,,,29,15,,,,,,,,,,,,,,,,,,,,,,,,,1,,202,23 +java.lang,38,3,790,,13,,,,,,1,,,,,,,,,,,,8,,,,2,9,,,4,,,1,,,,,,,,,,,,,,,,,,3,,,510,280 +java.math,,,9,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9 +java.net,23,3,347,,,,1,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,21,,,,,,,,,,,,,,,,,3,248,99 +java.nio,47,,499,,,,,,,,,5,,,,,,,,,,,,,,,25,16,,,,,,,,,1,,,,,,,,,,,,,,,,,,302,197 +java.rmi,,,68,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,45,23 +java.security,21,,583,,,11,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,285,298 +java.sql,15,1,292,,,,1,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,,9,,,,,,,,,,,,1,,,,274,18 +java.text,,,154,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,72,82 +java.time,,,131,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,27,104 +java.util,48,2,1340,,,,,,,,,1,,,,,,,,,,,34,,,,3,,,,,5,2,,1,2,,,,,,,,,,,,,,,,2,,,558,782 +javafx.scene.web,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,, +javax.accessibility,,,63,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,28,35 +javax.activation,2,,7,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,1,,,,,,,,,,,,,,,,,,7, +javax.annotation.processing,,,28,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,25,3 +javax.crypto,19,,140,,,12,3,,2,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,76,64 +javax.faces.context,4,7,,,,,,,,,,,,,,2,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,7,, +javax.imageio,1,,304,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,138,166 +javax.jms,,9,57,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9,57, +javax.json,,,123,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,100,23 +javax.lang.model,,,277,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,217,60 +javax.management,2,,766,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,363,403 +javax.naming,7,,341,,,,,,,,,,,,,,,,,6,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,191,150 +javax.net,4,,136,,,,2,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,87,49 +javax.portlet,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,, +javax.print,2,,133,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,102,31 +javax.rmi.ssl,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6 +javax.script,1,,50,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,14,36 +javax.security.auth,7,,147,,,4,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,50,97 +javax.security.cert,,,5,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,5, +javax.security.sasl,,,49,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,42,7 +javax.servlet,10,29,3,,,,,,,,,,,,,,1,,,,,,,,,,,2,,,,,,,,,,3,,,2,,2,,,,,,,,,,,29,3, +javax.smartcardio,,,34,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,24,10 +javax.sound.midi,,,60,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,51,9 +javax.sound.sampled,,,90,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,53,37 +javax.sql,7,,126,,,,4,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,68,58 +javax.tools,,,66,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,62,4 +javax.transaction.xa,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, +javax.validation,1,1,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,, +javax.ws.rs.client,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,, +javax.ws.rs.container,,9,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9,, +javax.ws.rs.core,3,,149,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,2,,,,,,,,,,,94,55 +javax.xml.bind.attachment,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,, +javax.xml.catalog,,,12,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,11,1 +javax.xml.crypto,,,269,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,172,97 +javax.xml.datatype,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,5,1 +javax.xml.namespace,,,15,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,5,10 +javax.xml.parsers,,,37,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,35,2 +javax.xml.stream,,,221,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,201,20 +javax.xml.transform,2,,134,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,1,,,,,,,,,72,62 +javax.xml.validation,,,29,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,29, +javax.xml.xpath,3,,26,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,,,,,,,,,,26, +jenkins,,,523,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,500,23 +jodd.json,,,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10 +kotlin,16,,1849,,,,,,,,,,,,,,,,,,,,,,,,11,3,,,,,,,,,2,,,,,,,,,,,,,,,,,,1836,13 +liquibase.database.jvm,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,, +liquibase.statement.core,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,, +net.lingala.zip4j,2,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +net.schmizz.sshj,4,,,,,,2,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +net.sf.json,2,,338,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,321,17 +net.sf.saxon.s9api,5,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,5,,,,,,,,,, +ognl,6,,,,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +okhttp3,4,,50,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,,,,,,,,,,,,,,,,,,23,27 +org.acegisecurity,,,49,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,49, +org.antlr.runtime,1,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.avro,18,19,,,,,,,,,,,,,,,,,,,,,,,,,17,,,,,,,,,,1,,,,,,,,,,,,1,,,,17,1,, +org.apache.commons.codec,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6, +org.apache.commons.collections,,,800,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,17,783 +org.apache.commons.collections4,,,806,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,17,789 +org.apache.commons.compress.archivers.tar,,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4, +org.apache.commons.exec,10,,,,6,,,,,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.commons.fileupload,,11,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,11,4, +org.apache.commons.httpclient.util,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, +org.apache.commons.io,124,,570,,,,,,,,,4,,,,,,,,,,,,,,,102,3,,,,,,,,,15,,,,,,,,,,,,,,,,,,556,14 +org.apache.commons.jelly,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,,, +org.apache.commons.jexl2,15,,,,,,,,,,,,,,,,,,15,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.commons.jexl3,15,,,,,,,,,,,,,,,,,,15,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.commons.lang,1,,767,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,596,171 +org.apache.commons.lang3,7,,425,,,,,,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,1,,,,,,,,,,,,,294,131 +org.apache.commons.logging,6,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.commons.net,13,12,,,,,2,2,,,,,,,,,,,,,,,,,,,,3,,,,,,,,,6,,,,,,,,,,,,,,,,,12,, +org.apache.commons.ognl,6,,,,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.commons.text,,,272,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,220,52 +org.apache.cxf.catalog,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,, +org.apache.cxf.common.classloader,3,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,2,,,,,,,,,,,,,,,,,,, +org.apache.cxf.common.jaxb,1,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.cxf.common.logging,6,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.cxf.configuration.jsse,2,,,,,,,,,,,,,,1,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.cxf.helpers,10,,,,,,,,,,,,,,,,,,,,,,,,,,5,,,,,,,,,,,,,,,,,,5,,,,,,,,,,, +org.apache.cxf.resource,9,,,,,,,,,,,,,,,,,,,,,,,,,,4,,,,,,,,,,5,,,,,,,,,,,,,,,,,,, +org.apache.cxf.staxutils,1,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.cxf.tools.corba.utils,4,,,,,,,,,,,,,,,,,,,,,,,,,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.cxf.tools.util,10,,,,,,,,,,,,,,,,,,,,,,,,,,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.cxf.transform,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,,,,,,,,,, +org.apache.directory.ldap.client.api,1,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.hadoop.fs,3,,11,,,,,,,,,,,,,,,,,,,,,,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,11, +org.apache.hadoop.hive.metastore,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,,,,,,,,,,,,,,,,, +org.apache.hadoop.hive.ql.exec,1,,1,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, +org.apache.hadoop.hive.ql.metadata,1,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.hc.client5.http.async.methods,84,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,84,,,,,,,,,,,,,,,,,,, +org.apache.hc.client5.http.classic.methods,37,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,37,,,,,,,,,,,,,,,,,,, +org.apache.hc.client5.http.fluent,19,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,19,,,,,,,,,,,,,,,,,,, +org.apache.hc.client5.http.protocol,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 +org.apache.hc.client5.http.utils,,,7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,7, +org.apache.hc.core5.benchmark,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,, +org.apache.hc.core5.function,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, +org.apache.hc.core5.http,73,2,45,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,72,,,,,,,,,,,,,,,,,2,45, +org.apache.hc.core5.net,,,18,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,18, +org.apache.hc.core5.util,,,24,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,18,6 +org.apache.hive.hcatalog.templeton,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,, +org.apache.http,53,3,117,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,,,,51,,,,,,,,,,,,,,,,,3,108,9 +org.apache.ibatis.jdbc,6,,57,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,57, +org.apache.ibatis.mapping,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, +org.apache.log4j,11,,,,,,,,,,,,,,,,,,,,,,11,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.logging.log4j,359,,8,,,,,,,,,,,,,,,,,,,,359,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,4 +org.apache.poi,43,24,2216,,,,,,,,,,,,,,,,,,,,,,,,40,,,,,,,,,,,,,,,,,,,,3,,,,,1,23,,1160,1056 +org.apache.shiro.authc,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,, +org.apache.shiro.codec,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, +org.apache.shiro.jndi,1,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.shiro.mgt,1,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.sshd.client.session,3,,,,,,1,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.struts.beanvalidation.validation.interceptor,,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4, +org.apache.struts2,14,,3873,,,,,,,,,,,,,,,,,,,,,,,11,,,,,,,,,,,,,,,3,,,,,,,,,,,,,,3839,34 +org.apache.tools.ant,14,,,,1,,,,,,,,,,,,,,,,,,,,,,5,8,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.apache.tools.zip,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, +org.apache.velocity.app,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,,,,,,,,,,,,,,,, +org.apache.velocity.runtime,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,,,,,,,,,,,,,,,, +org.codehaus.cargo.container.installer,3,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,1,,,,,,,,,,,,,,,,,,, +org.codehaus.groovy.control,1,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.dom4j,20,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,20,,,,,,,,,,, +org.eclipse.jetty.client,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,, +org.exolab.castor.xml,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,, +org.fusesource.leveldbjni,1,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.geogebra.web.full.main,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,, +org.gradle.api.file,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3, +org.hibernate,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10,,,,,,,,,,,,,,,,, +org.ho.yaml,8,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,8,,,,,,,,,,,,,, +org.influxdb,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,, +org.jabsorb,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,, +org.jboss.logging,324,,,,,,,,,,,,,,,,,,,,,,324,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.jboss.vfs,1,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.jdbi.v3.core,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,,,,,,,,,,,,,,,,,,, +org.jenkins.ui.icon,,,49,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,48,1 +org.jenkins.ui.symbol,,,33,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,25,8 +org.jooq,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,, +org.json,,,236,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,198,38 +org.keycloak.models.map.storage,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,, +org.kohsuke.stapler,20,24,363,,,,,,,,,,,,,2,,,,,,,,,,,8,1,,,,,,,,,3,,,,,,1,5,,,,,,,,,,24,352,11 +org.lastaflute.web,,1,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,4, +org.mvel2,16,,,,,,,,,,,,,,,,,,,,,,,16,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.openjdk.jmh.runner.options,1,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.owasp.esapi,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, +org.pac4j.jwt.config.encryption,4,,,,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.pac4j.jwt.config.signature,4,,,,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.scijava.log,13,,,,,,,,,,,,,,,,,,,,,,13,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.slf4j,55,,6,,,,,,,,,,,,,,,,,,,,55,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,4 +org.springframework.beans,,,30,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,30 +org.springframework.boot.jdbc,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,, +org.springframework.cache,,,13,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,13 +org.springframework.context,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3, +org.springframework.core.io,17,,6,,,,,,,,,,,,,,,,,,,,,,,,16,,,,,,,,,,1,,,,,,,,,,,,,,,,,,6, +org.springframework.data.repository,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 +org.springframework.http,14,,77,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,14,,,,,,,,,,,,,,,,,,67,10 +org.springframework.jdbc.core,19,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,19,,,,,,,,,,,,,,,,, +org.springframework.jdbc.datasource,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,,,,,,,,,,,,,,,,,,, +org.springframework.jdbc.object,9,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9,,,,,,,,,,,,,,,,, +org.springframework.jndi,1,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.springframework.ldap,47,,,,,,,,,,,,,,,,,,,33,,14,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.springframework.security.core.userdetails,2,,,,,,1,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +org.springframework.security.web.savedrequest,,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,, +org.springframework.ui,,,32,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,32 +org.springframework.util,10,,142,,,,,,,,,,,,,,,,,,,,,,,,9,1,,,,,,,,,,,,,,,,,,,,,,,,,,,90,52 +org.springframework.validation,,,13,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,13, +org.springframework.web.client,13,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,13,,,,,,,,,,,,,,,,,3,, +org.springframework.web.context.request,,8,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,8,, +org.springframework.web.multipart,,12,12,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,12,12, +org.springframework.web.portlet,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,, +org.springframework.web.reactive.function.client,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,,,,,,,,,,,,,,,,,,, +org.springframework.web.servlet,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,, +org.springframework.web.socket,,8,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,8,6, +org.springframework.web.util,,9,159,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,9,134,25 +org.thymeleaf,2,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,2, +org.xml.sax,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, +org.xmlpull.v1,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,, +org.yaml.snakeyaml,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1, +play.libs.ws,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,,,,,,,,,,,,,,,,,,, +play.mvc,1,13,24,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,13,24, +ratpack.core.form,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3, +ratpack.core.handling,,6,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,4, +ratpack.core.http,,10,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10,10, +ratpack.exec,,,48,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,48 +ratpack.form,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3, +ratpack.func,,,35,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,35 +ratpack.handling,,6,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,6,4, +ratpack.http,,10,10,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10,10, +ratpack.util,,,35,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,35 +retrofit2,1,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,,,,1, +software.amazon.awssdk.transfer.s3.model,8,,,,,,,,,,,,,,,,,,,,,,,,,,8,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +sun.jvmstat.perfdata.monitor.protocol.local,3,,,,,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +sun.jvmstat.perfdata.monitor.protocol.rmi,1,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +sun.misc,3,,,,,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +sun.net.ftp,5,,,,,,2,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +sun.net.www.protocol.http,3,,,,,,2,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +sun.security.acl,1,,,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +sun.security.jgss.krb5,2,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +sun.security.krb5,9,,,,,3,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +sun.security.pkcs,4,,,,,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +sun.security.pkcs11,3,,,,,1,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +sun.security.provider,2,,,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +sun.security.ssl,3,,,,,,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +sun.security.x509,1,,,,,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +sun.tools.jconsole,28,,,,,,13,15,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/java/documentation/library-coverage/coverage.rst b/java/documentation/library-coverage/coverage.rst index 28f862d8743..cc3da4594a6 100644 --- a/java/documentation/library-coverage/coverage.rst +++ b/java/documentation/library-coverage/coverage.rst @@ -40,6 +40,6 @@ Java framework & library support `Spring `_,``org.springframework.*``,46,494,144,26,,28,14,,36 `Thymeleaf `_,``org.thymeleaf``,,2,2,,,,,, `jOOQ `_,``org.jooq``,,,1,,,1,,, - Others,"``actions.osgi``, ``antlr``, ``ch.ethz.ssh2``, ``cn.hutool.core.codec``, ``com.alibaba.com.caucho.hessian.io``, ``com.alibaba.druid.sql``, ``com.alibaba.fastjson2``, ``com.amazonaws.auth``, ``com.auth0.jwt.algorithms``, ``com.azure.identity``, ``com.caucho.burlap.io``, ``com.caucho.hessian.io``, ``com.cedarsoftware.util.io``, ``com.esotericsoftware.kryo.io``, ``com.esotericsoftware.kryo5.io``, ``com.esotericsoftware.yamlbeans``, ``com.hubspot.jinjava``, ``com.jcraft.jsch``, ``com.microsoft.sqlserver.jdbc``, ``com.mitchellbosecke.pebble``, ``com.opensymphony.xwork2``, ``com.sshtools.j2ssh.authentication``, ``com.sun.crypto.provider``, ``com.sun.jndi.ldap``, ``com.sun.net.httpserver``, ``com.sun.net.ssl``, ``com.sun.rowset``, ``com.sun.security.auth.module``, ``com.sun.security.ntlm``, ``com.sun.security.sasl.digest``, ``com.thoughtworks.xstream``, ``com.trilead.ssh2``, ``com.unboundid.ldap.sdk``, ``com.zaxxer.hikari``, ``flexjson``, ``hudson``, ``io.jsonwebtoken``, ``io.undertow.server.handlers.resource``, ``javafx.scene.web``, ``jenkins``, ``jodd.json``, ``liquibase.database.jvm``, ``liquibase.statement.core``, ``net.lingala.zip4j``, ``net.schmizz.sshj``, ``net.sf.json``, ``net.sf.saxon.s9api``, ``ognl``, ``org.acegisecurity``, ``org.antlr.runtime``, ``org.apache.avro``, ``org.apache.commons.codec``, ``org.apache.commons.compress.archivers.tar``, ``org.apache.commons.exec``, ``org.apache.commons.fileupload``, ``org.apache.commons.httpclient.util``, ``org.apache.commons.jelly``, ``org.apache.commons.jexl2``, ``org.apache.commons.jexl3``, ``org.apache.commons.lang``, ``org.apache.commons.logging``, ``org.apache.commons.net``, ``org.apache.commons.ognl``, ``org.apache.cxf.catalog``, ``org.apache.cxf.common.classloader``, ``org.apache.cxf.common.jaxb``, ``org.apache.cxf.common.logging``, ``org.apache.cxf.configuration.jsse``, ``org.apache.cxf.helpers``, ``org.apache.cxf.resource``, ``org.apache.cxf.staxutils``, ``org.apache.cxf.tools.corba.utils``, ``org.apache.cxf.tools.util``, ``org.apache.cxf.transform``, ``org.apache.directory.ldap.client.api``, ``org.apache.hadoop.fs``, ``org.apache.hadoop.hive.metastore``, ``org.apache.hadoop.hive.ql.exec``, ``org.apache.hadoop.hive.ql.metadata``, ``org.apache.hc.client5.http.async.methods``, ``org.apache.hc.client5.http.classic.methods``, ``org.apache.hc.client5.http.fluent``, ``org.apache.hc.client5.http.protocol``, ``org.apache.hc.client5.http.utils``, ``org.apache.hive.hcatalog.templeton``, ``org.apache.ibatis.jdbc``, ``org.apache.ibatis.mapping``, ``org.apache.log4j``, ``org.apache.shiro.authc``, ``org.apache.shiro.codec``, ``org.apache.shiro.jndi``, ``org.apache.shiro.mgt``, ``org.apache.sshd.client.session``, ``org.apache.tools.ant``, ``org.apache.tools.zip``, ``org.codehaus.cargo.container.installer``, ``org.dom4j``, ``org.exolab.castor.xml``, ``org.fusesource.leveldbjni``, ``org.geogebra.web.full.main``, ``org.gradle.api.file``, ``org.ho.yaml``, ``org.influxdb``, ``org.jabsorb``, ``org.jboss.vfs``, ``org.jdbi.v3.core``, ``org.jenkins.ui.icon``, ``org.jenkins.ui.symbol``, ``org.keycloak.models.map.storage``, ``org.kohsuke.stapler``, ``org.lastaflute.web``, ``org.mvel2``, ``org.openjdk.jmh.runner.options``, ``org.owasp.esapi``, ``org.pac4j.jwt.config.encryption``, ``org.pac4j.jwt.config.signature``, ``org.scijava.log``, ``org.xml.sax``, ``org.xmlpull.v1``, ``play.libs.ws``, ``play.mvc``, ``ratpack.core.form``, ``ratpack.core.handling``, ``ratpack.core.http``, ``ratpack.exec``, ``ratpack.form``, ``ratpack.func``, ``ratpack.handling``, ``ratpack.http``, ``ratpack.util``, ``software.amazon.awssdk.transfer.s3.model``, ``sun.jvmstat.perfdata.monitor.protocol.local``, ``sun.jvmstat.perfdata.monitor.protocol.rmi``, ``sun.misc``, ``sun.net.ftp``, ``sun.net.www.protocol.http``, ``sun.security.acl``, ``sun.security.jgss.krb5``, ``sun.security.krb5``, ``sun.security.pkcs``, ``sun.security.pkcs11``, ``sun.security.provider``, ``sun.security.ssl``, ``sun.security.x509``, ``sun.tools.jconsole``",127,6042,775,148,6,14,18,,186 - Totals,,382,26411,2708,421,16,137,33,1,416 + Others,"``actions.osgi``, ``antlr``, ``ch.ethz.ssh2``, ``cn.hutool.core.codec``, ``com.alibaba.com.caucho.hessian.io``, ``com.alibaba.druid.sql``, ``com.alibaba.fastjson2``, ``com.amazonaws.auth``, ``com.auth0.jwt.algorithms``, ``com.azure.identity``, ``com.caucho.burlap.io``, ``com.caucho.hessian.io``, ``com.cedarsoftware.util.io``, ``com.esotericsoftware.kryo.io``, ``com.esotericsoftware.kryo5.io``, ``com.esotericsoftware.yamlbeans``, ``com.hubspot.jinjava``, ``com.jcraft.jsch``, ``com.microsoft.sqlserver.jdbc``, ``com.mitchellbosecke.pebble``, ``com.opensymphony.xwork2``, ``com.sshtools.j2ssh.authentication``, ``com.sun.crypto.provider``, ``com.sun.jndi.ldap``, ``com.sun.net.httpserver``, ``com.sun.net.ssl``, ``com.sun.rowset``, ``com.sun.security.auth.module``, ``com.sun.security.ntlm``, ``com.sun.security.sasl.digest``, ``com.thoughtworks.xstream``, ``com.trilead.ssh2``, ``com.unboundid.ldap.sdk``, ``com.zaxxer.hikari``, ``flexjson``, ``hudson``, ``io.jsonwebtoken``, ``io.undertow.server.handlers.resource``, ``javafx.scene.web``, ``jenkins``, ``jodd.json``, ``liquibase.database.jvm``, ``liquibase.statement.core``, ``net.lingala.zip4j``, ``net.schmizz.sshj``, ``net.sf.json``, ``net.sf.saxon.s9api``, ``ognl``, ``org.acegisecurity``, ``org.antlr.runtime``, ``org.apache.avro``, ``org.apache.commons.codec``, ``org.apache.commons.compress.archivers.tar``, ``org.apache.commons.exec``, ``org.apache.commons.fileupload``, ``org.apache.commons.httpclient.util``, ``org.apache.commons.jelly``, ``org.apache.commons.jexl2``, ``org.apache.commons.jexl3``, ``org.apache.commons.lang``, ``org.apache.commons.logging``, ``org.apache.commons.net``, ``org.apache.commons.ognl``, ``org.apache.cxf.catalog``, ``org.apache.cxf.common.classloader``, ``org.apache.cxf.common.jaxb``, ``org.apache.cxf.common.logging``, ``org.apache.cxf.configuration.jsse``, ``org.apache.cxf.helpers``, ``org.apache.cxf.resource``, ``org.apache.cxf.staxutils``, ``org.apache.cxf.tools.corba.utils``, ``org.apache.cxf.tools.util``, ``org.apache.cxf.transform``, ``org.apache.directory.ldap.client.api``, ``org.apache.hadoop.fs``, ``org.apache.hadoop.hive.metastore``, ``org.apache.hadoop.hive.ql.exec``, ``org.apache.hadoop.hive.ql.metadata``, ``org.apache.hc.client5.http.async.methods``, ``org.apache.hc.client5.http.classic.methods``, ``org.apache.hc.client5.http.fluent``, ``org.apache.hc.client5.http.protocol``, ``org.apache.hc.client5.http.utils``, ``org.apache.hive.hcatalog.templeton``, ``org.apache.ibatis.jdbc``, ``org.apache.ibatis.mapping``, ``org.apache.log4j``, ``org.apache.poi``, ``org.apache.shiro.authc``, ``org.apache.shiro.codec``, ``org.apache.shiro.jndi``, ``org.apache.shiro.mgt``, ``org.apache.sshd.client.session``, ``org.apache.tools.ant``, ``org.apache.tools.zip``, ``org.codehaus.cargo.container.installer``, ``org.dom4j``, ``org.exolab.castor.xml``, ``org.fusesource.leveldbjni``, ``org.geogebra.web.full.main``, ``org.gradle.api.file``, ``org.ho.yaml``, ``org.influxdb``, ``org.jabsorb``, ``org.jboss.vfs``, ``org.jdbi.v3.core``, ``org.jenkins.ui.icon``, ``org.jenkins.ui.symbol``, ``org.keycloak.models.map.storage``, ``org.kohsuke.stapler``, ``org.lastaflute.web``, ``org.mvel2``, ``org.openjdk.jmh.runner.options``, ``org.owasp.esapi``, ``org.pac4j.jwt.config.encryption``, ``org.pac4j.jwt.config.signature``, ``org.scijava.log``, ``org.xml.sax``, ``org.xmlpull.v1``, ``play.libs.ws``, ``play.mvc``, ``ratpack.core.form``, ``ratpack.core.handling``, ``ratpack.core.http``, ``ratpack.exec``, ``ratpack.form``, ``ratpack.func``, ``ratpack.handling``, ``ratpack.http``, ``ratpack.util``, ``software.amazon.awssdk.transfer.s3.model``, ``sun.jvmstat.perfdata.monitor.protocol.local``, ``sun.jvmstat.perfdata.monitor.protocol.rmi``, ``sun.misc``, ``sun.net.ftp``, ``sun.net.www.protocol.http``, ``sun.security.acl``, ``sun.security.jgss.krb5``, ``sun.security.krb5``, ``sun.security.pkcs``, ``sun.security.pkcs11``, ``sun.security.provider``, ``sun.security.ssl``, ``sun.security.x509``, ``sun.tools.jconsole``",151,8258,818,188,6,14,18,,186 + Totals,,406,28627,2751,461,16,137,33,1,416 From 985a79a45e4f76093a704d948763cc1c64992dae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 03:03:43 +0000 Subject: [PATCH 30/90] Bump golang.org/x/tools Bumps the extractor-dependencies group in /go/extractor with 1 update: [golang.org/x/tools](https://github.com/golang/tools). Updates `golang.org/x/tools` from 0.47.0 to 0.48.0 - [Release notes](https://github.com/golang/tools/releases) - [Commits](https://github.com/golang/tools/compare/v0.47.0...v0.48.0) --- updated-dependencies: - dependency-name: golang.org/x/tools dependency-version: 0.48.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: extractor-dependencies ... Signed-off-by: dependabot[bot] --- go/extractor/go.mod | 4 ++-- go/extractor/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go/extractor/go.mod b/go/extractor/go.mod index b95ec78da34..e121c4a98a8 100644 --- a/go/extractor/go.mod +++ b/go/extractor/go.mod @@ -10,7 +10,7 @@ toolchain go1.26.4 // bazel mod tidy require ( golang.org/x/mod v0.38.0 - golang.org/x/tools v0.47.0 + golang.org/x/tools v0.48.0 ) require github.com/stretchr/testify v1.11.1 @@ -18,6 +18,6 @@ require github.com/stretchr/testify v1.11.1 require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/sync v0.21.0 // indirect + golang.org/x/sync v0.22.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go/extractor/go.sum b/go/extractor/go.sum index de0d26d035b..76fbec137b4 100644 --- a/go/extractor/go.sum +++ b/go/extractor/go.sum @@ -8,10 +8,10 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= -golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From 059aa2da70a139c61b6eac5bfcd37645186ea767 Mon Sep 17 00:00:00 2001 From: Kristen Newbury Date: Fri, 10 Jul 2026 00:04:33 -0400 Subject: [PATCH 31/90] Add missing changenote --- .../change-notes/2026-07-09-environment-check-alteration.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 actions/ql/lib/change-notes/2026-07-09-environment-check-alteration.md diff --git a/actions/ql/lib/change-notes/2026-07-09-environment-check-alteration.md b/actions/ql/lib/change-notes/2026-07-09-environment-check-alteration.md new file mode 100644 index 00000000000..e7b49abddf7 --- /dev/null +++ b/actions/ql/lib/change-notes/2026-07-09-environment-check-alteration.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* Altered the logic of `EnvironmentCheck` to make sure it is a check that protects only for non-toctou. This change will result in more results being found by the queries: `actions/untrusted-checkout-toctou/high` and `actions/untrusted-checkout-toctou/critical`. \ No newline at end of file From 89e11e5cfcea4d935a78fd8885603c94698dd2b7 Mon Sep 17 00:00:00 2001 From: Taus Date: Mon, 6 Jul 2026 21:19:45 +0000 Subject: [PATCH 32/90] unified: Add `swift-syntax-rs` Adds an initial prototype of an interface from Rust to Swift, which enables us to use the `swift-syntax` package for parsing. At present, the parsed AST is passed between Swift and Rust as a JSON string. --- Cargo.lock | 4 + Cargo.toml | 1 + MODULE.bazel | 35 +++- unified/swift-syntax-rs/.gitignore | 2 + unified/swift-syntax-rs/.swift-version | 1 + unified/swift-syntax-rs/BUILD.bazel | 55 ++++++ unified/swift-syntax-rs/Cargo.toml | 14 ++ unified/swift-syntax-rs/README.md | 180 ++++++++++++++++++ unified/swift-syntax-rs/build.rs | 102 ++++++++++ unified/swift-syntax-rs/src/lib.rs | 125 ++++++++++++ unified/swift-syntax-rs/src/main.rs | 42 ++++ .../swift-syntax-rs/swift/Package.resolved | 15 ++ unified/swift-syntax-rs/swift/Package.swift | 30 +++ .../SwiftSyntaxFFI/SwiftSyntaxFFI.swift | 145 ++++++++++++++ 14 files changed, 750 insertions(+), 1 deletion(-) create mode 100644 unified/swift-syntax-rs/.gitignore create mode 100644 unified/swift-syntax-rs/.swift-version create mode 100644 unified/swift-syntax-rs/BUILD.bazel create mode 100644 unified/swift-syntax-rs/Cargo.toml create mode 100644 unified/swift-syntax-rs/README.md create mode 100644 unified/swift-syntax-rs/build.rs create mode 100644 unified/swift-syntax-rs/src/lib.rs create mode 100644 unified/swift-syntax-rs/src/main.rs create mode 100644 unified/swift-syntax-rs/swift/Package.resolved create mode 100644 unified/swift-syntax-rs/swift/Package.swift create mode 100644 unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift diff --git a/Cargo.lock b/Cargo.lock index 76043ec0a43..7a9f1966791 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2850,6 +2850,10 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "swift-syntax-rs" +version = "0.1.0" + [[package]] name = "syn" version = "2.0.106" diff --git a/Cargo.toml b/Cargo.toml index 9c15b486062..9f3780bb1d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "ruby/extractor", "unified/extractor", "unified/extractor/tree-sitter-swift", + "unified/swift-syntax-rs", "rust/extractor", "rust/extractor/macros", "rust/ast-generator", diff --git a/MODULE.bazel b/MODULE.bazel index e6f1ddb893d..aa1f6555e0c 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -24,13 +24,15 @@ bazel_dep(name = "rules_python", version = "1.9.0") bazel_dep(name = "rules_shell", version = "0.7.1") bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "abseil-cpp", version = "20260107.1", repo_name = "absl") -bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json") +bazel_dep(name = "nlohmann_json", version = "3.12.0.bcr.1", repo_name = "json") bazel_dep(name = "fmt", version = "12.1.0-codeql.1") bazel_dep(name = "rules_kotlin", version = "2.2.2-codeql.1") bazel_dep(name = "gazelle", version = "0.50.0") bazel_dep(name = "rules_dotnet", version = "0.21.5-codeql.1") bazel_dep(name = "googletest", version = "1.17.0.bcr.2") bazel_dep(name = "rules_rust", version = "0.69.0") +bazel_dep(name = "rules_swift", version = "4.0.0-rc4") +bazel_dep(name = "swift-syntax", version = "602.0.0.bcr.2") bazel_dep(name = "zstd", version = "1.5.7.bcr.1") bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True) @@ -219,6 +221,37 @@ use_repo( "swift-resource-dir-macos", ) +# Hermetic Swift toolchain (from swift.org) for building the `swift-syntax` +# based Rust wrapper in `unified/swift-syntax-rs`. This is the official +# `rules_swift` standalone-toolchain extension and is independent of the +# patched prebuilt toolchain wired up via `swift_deps` above. +# +# Bazel cannot auto-select between Linux distributions, so the toolchain is +# registered explicitly for the distribution our CI runs on (ubuntu24.04, +# x86_64). Add further platforms here if CI grows to cover them. +# +# We register the `exec` toolchain (normal host compilation, targeting +# linux/x86_64) rather than the `embedded` one, which targets `os:none` +# (Embedded Swift) and would not match a normal host build. +# +# The Swift version is read from `unified/swift-syntax-rs/.swift-version`, which +# is the single source of truth shared with the local `cargo` build (via +# swiftly / any Swift install) and `swift/Package.swift`. +swift = use_extension("@rules_swift//swift:extensions.bzl", "swift") +swift.toolchain( + name = "swift_toolchain", + swift_version_file = "//unified/swift-syntax-rs:.swift-version", +) +use_repo( + swift, + "swift_toolchain", + "swift_toolchain_ubuntu24.04", +) + +register_toolchains( + "@swift_toolchain//:swift_toolchain_exec_ubuntu24.04", +) + node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node") node.toolchain( name = "nodejs", diff --git a/unified/swift-syntax-rs/.gitignore b/unified/swift-syntax-rs/.gitignore new file mode 100644 index 00000000000..0b649686540 --- /dev/null +++ b/unified/swift-syntax-rs/.gitignore @@ -0,0 +1,2 @@ +/target +/swift/.build diff --git a/unified/swift-syntax-rs/.swift-version b/unified/swift-syntax-rs/.swift-version new file mode 100644 index 00000000000..42cc526d6ca --- /dev/null +++ b/unified/swift-syntax-rs/.swift-version @@ -0,0 +1 @@ +6.2.4 diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel new file mode 100644 index 00000000000..10d11a1bec1 --- /dev/null +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -0,0 +1,55 @@ +load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test") +load("@rules_swift//swift:swift.bzl", "swift_library") + +package(default_visibility = ["//visibility:public"]) + +# The pinned Swift version, shared with the local `cargo` build and +# `swift/Package.swift`. Referenced by the `swift.toolchain` extension in +# //:MODULE.bazel via `swift_version_file`. +exports_files([".swift-version"]) + +# The Swift FFI shim: wraps swift-syntax and exposes a small C ABI +# (`ssr_parse_json` / `ssr_string_free`). Built with the hermetic Swift +# toolchain registered in //:MODULE.bazel; provides a `CcInfo` that the Rust +# targets below link against. +swift_library( + name = "swift_syntax_ffi", + srcs = ["swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift"], + module_name = "SwiftSyntaxFFI", + deps = [ + "@swift-syntax//:SwiftParser", + "@swift-syntax//:SwiftSyntax", + ], +) + +# Safe Rust bindings on top of the C ABI. `build.rs` is intentionally *not* +# wired up here (no `cargo_build_script`): under Bazel the Swift side is +# provided by `:swift_syntax_ffi`, whereas `build.rs` only exists to build the +# Swift shim in the local `cargo` workflow. +rust_library( + name = "swift_syntax_rs", + srcs = ["src/lib.rs"], + edition = "2024", + deps = [":swift_syntax_ffi"], +) + +rust_binary( + name = "swift-syntax-parse", + srcs = ["src/main.rs"], + # The Swift toolchain propagates a runfiles-relative RPATH to its runtime + # `.so`s via `swift_syntax_ffi`'s `CcInfo`, but (unlike `swift_binary`) + # `rust_binary` does not copy those libraries into runfiles. Add the + # toolchain files as `data` so the runtime is found at execution time. + # (rules_swift does not yet support statically linking the runtime.) + data = ["@swift_toolchain_ubuntu24.04//:files"], + edition = "2024", + deps = [":swift_syntax_rs"], +) + +rust_test( + name = "swift_syntax_rs_test", + size = "small", + crate = ":swift_syntax_rs", + data = ["@swift_toolchain_ubuntu24.04//:files"], + edition = "2024", +) diff --git a/unified/swift-syntax-rs/Cargo.toml b/unified/swift-syntax-rs/Cargo.toml new file mode 100644 index 00000000000..6957fb0e50e --- /dev/null +++ b/unified/swift-syntax-rs/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "swift-syntax-rs" +description = "Rust wrapper around the swift-syntax package for parsing Swift source" +version = "0.1.0" +authors = ["GitHub"] +edition = "2024" + +[lib] +name = "swift_syntax_rs" +path = "src/lib.rs" + +[[bin]] +name = "swift-syntax-parse" +path = "src/main.rs" diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md new file mode 100644 index 00000000000..08796b3a6a7 --- /dev/null +++ b/unified/swift-syntax-rs/README.md @@ -0,0 +1,180 @@ +# swift-syntax-rs + +A Rust wrapper around the [swift-syntax](https://github.com/swiftlang/swift-syntax) +package, allowing Swift source code to be parsed from Rust. + +Parsing is delegated to a small Swift shim (in [`swift/`](swift/)) that links +against `SwiftSyntax`/`SwiftParser` and exposes a tiny C ABI. The Rust crate +builds that shim (via `build.rs`) and provides safe bindings on top of it. + +## Output format + +The emitted JSON tree preserves the AST's named structure. Every node has a +`kind` and a `range` with `start`/`end` positions (UTF-8 `offset` plus 1-based +`line`/`column`). Beyond that: + +- **Tokens** carry `text`, `tokenKind`, and `leadingTrivia`/`trailingTrivia` + arrays of `{ kind, text }` pieces. +- **Layout nodes** (e.g. `functionDecl`) embed their children directly as + members keyed by the child's name in the parent (`name`, `signature`, + `body`, …), alongside `kind`/`range`. Absent optional children are omitted. +- **Collection nodes** (e.g. `codeBlockItemList`) are elided: a list-valued + field is simply a JSON array of its elements (e.g. `parameters`, `statements`). + This drops the collection node's own `kind`/`range`. + +Only meaningful trivia is kept — the four comment kinds (`lineComment`, +`blockComment`, `docLineComment`, `docBlockComment`) and `unexpectedText` +(source the parser skipped). Whitespace trivia is dropped, since node ranges +already encode positions. + +### Example + +Parsing `let x = 1 // c` produces the following (each `range` object is +abbreviated here as `…`): + +```jsonc +{ + "kind": "sourceFile", + "range": …, + "statements": [ // collection node elided to an array + { + "kind": "codeBlockItem", + "range": …, + "item": { + "kind": "variableDecl", + "range": …, + "attributes": [], // empty collection → empty array + "modifiers": [], + "bindingSpecifier": { // a token + "kind": "token", + "text": "let", + "tokenKind": "keyword(SwiftSyntax.Keyword.let)", + "range": …, + "leadingTrivia": [], + "trailingTrivia": [] + }, + "bindings": [ + { + "kind": "patternBinding", + "range": …, + "pattern": { + "kind": "identifierPattern", + "range": …, + "identifier": { "kind": "token", "text": "x", "tokenKind": "identifier(\"x\")", "range": …, "leadingTrivia": [], "trailingTrivia": [] } + }, + "initializer": { + "kind": "initializerClause", + "range": …, + "equal": { "kind": "token", "text": "=", "tokenKind": "equal", "range": …, "leadingTrivia": [], "trailingTrivia": [] }, + "value": { + "kind": "integerLiteralExpr", + "range": …, + "literal": { + "kind": "token", + "text": "1", + "tokenKind": "integerLiteral(\"1\")", + "range": …, + "leadingTrivia": [], + "trailingTrivia": [ { "kind": "lineComment", "text": "// c" } ] + } + } + } + } + ] + } + } + ], + "endOfFileToken": { "kind": "token", "text": "", "tokenKind": "endOfFile", "range": …, "leadingTrivia": [], "trailingTrivia": [] } +} +``` + +Note how `statements`, `bindings`, `attributes`, and `modifiers` are plain +arrays (their collection nodes are elided), layout children such as +`bindingSpecifier` and `initializer` are embedded by name, and the `// c` +comment rides along as `trailingTrivia` on the token it follows. + +## Prerequisites + +The build does not depend on any particular version manager. You need: + +- **Rust** — pinned to `1.88` by the repo-root [`rust-toolchain.toml`](../../rust-toolchain.toml), + which `rustup` picks up automatically. +- **Swift** — pinned to the version in [`.swift-version`](.swift-version) + (currently `6.2.4`), used to build `swift-syntax` `602.0.0`. Install it any way + you like — [swift.org](https://www.swift.org/install/) or + [swiftly](https://www.swift.org/swiftly/) (which reads `.swift-version`), or a + system package. Just make sure `swift` is on your `PATH` (or point `build.rs` + at it with the `SWIFT` environment variable). + +On Debian/Ubuntu the Swift runtime also needs `libncurses6` (and related libs) +available on the system. + +## Building & testing + +With `cargo` and `swift` on `PATH`: + +```sh +cargo build +cargo test +``` + +If your `swift`/`swiftc` are not on `PATH`, point the build at them explicitly: + +```sh +SWIFT=/path/to/swift SWIFTC=/path/to/swiftc cargo build +``` + +The first build compiles `swift-syntax` and can take several minutes. + +## Building with Bazel (CI) + +CI builds this crate hermetically with Bazel. A Swift toolchain is downloaded +from swift.org by the official `rules_swift` standalone toolchain extension +(wired up in the repo-root `MODULE.bazel`), `swift-syntax` is pulled from the +Bazel Central Registry, and the FFI shim is compiled as a `swift_library` that +the Rust targets link against. `build.rs` is not used under Bazel; it only +builds the Swift shim for the local `cargo` workflow. + +```sh +bazel build //unified/swift-syntax-rs:swift-syntax-parse +bazel test //unified/swift-syntax-rs:swift_syntax_rs_test +bazel run //unified/swift-syntax-rs:swift-syntax-parse < some.swift +``` + +Requirements: + +- **`clang`** must be installed on the runner. `rules_swift` requires the Bazel + CC toolchain to use clang; the repo's `.bazelrc` already sets + `--repo_env=CC=clang`, so no extra flags are needed. +- The registered Swift toolchain currently targets **ubuntu24.04 / x86_64** + only (Bazel cannot auto-select between Linux distributions). Add more + platforms in `MODULE.bazel` (`swift.toolchain` + `register_toolchains`) if CI + grows to cover them. + +The Swift compiler version is read from [`.swift-version`](.swift-version) by +both the Bazel toolchain (`swift.toolchain(swift_version_file = …)`) and the +local build, and is kept in sync with the `swift-syntax` release pinned in +`swift/Package.swift`, so local and CI builds behave identically. + +## Usage + +Library: + +```rust +let json = swift_syntax_rs::parse_to_json("let x = 1")?; +println!("{json}"); +``` + +CLI (reads a file argument or stdin, prints the syntax tree as JSON): + +```sh +echo 'let x = 1' | cargo run --bin swift-syntax-parse +``` + +## Layout + +- `swift/` — Swift package exposing the `ssr_parse_json` / `ssr_string_free` C ABI. +- `build.rs` — builds the Swift package and emits link/rpath flags (local `cargo` only). +- `BUILD.bazel` — Bazel targets for the hermetic CI build (swift_library + rust targets). +- `src/lib.rs` — safe Rust bindings (`parse_to_json`). +- `src/main.rs` — demo CLI. diff --git a/unified/swift-syntax-rs/build.rs b/unified/swift-syntax-rs/build.rs new file mode 100644 index 00000000000..0adb140140d --- /dev/null +++ b/unified/swift-syntax-rs/build.rs @@ -0,0 +1,102 @@ +use std::env; +use std::path::PathBuf; +use std::process::Command; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let swift_dir = manifest_dir.join("swift"); + + println!( + "cargo:rerun-if-changed={}", + swift_dir.join("Package.swift").display() + ); + println!( + "cargo:rerun-if-changed={}", + swift_dir + .join("Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift") + .display() + ); + println!("cargo:rerun-if-env-changed=SWIFT"); + println!("cargo:rerun-if-env-changed=SWIFTC"); + + // Build the Swift FFI package as a release dynamic library. + let mut command = Command::new(swift_bin()); + command + .args(["build", "-c", "release"]) + .current_dir(&swift_dir); + apply_bare_repository_workaround(&mut command); + let status = command.status().unwrap_or_else(|e| { + panic!( + "failed to run `{swift} build`: {e}\n\ + Install a Swift toolchain (see https://www.swift.org/install/, e.g. via \ + swiftly) and ensure `swift` is on PATH, or set the `SWIFT` environment \ + variable to the `swift` executable. The pinned version is in `.swift-version`.", + swift = swift_bin(), + ) + }); + assert!(status.success(), "`swift build` failed"); + + // Link against the freshly built dynamic library. + let build_dir = swift_dir.join(".build/release"); + println!("cargo:rustc-link-search=native={}", build_dir.display()); + println!("cargo:rustc-link-lib=dylib=SwiftSyntaxFFI"); + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", build_dir.display()); + + // The executable also needs to find the Swift runtime libraries at run time. + if let Some(runtime) = swift_runtime_dir() { + println!("cargo:rustc-link-search=native={}", runtime.display()); + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", runtime.display()); + } +} + +/// Query the active Swift toolchain for the directory containing its runtime +/// shared libraries (e.g. `libswiftCore.so`). +fn swift_runtime_dir() -> Option { + let output = Command::new(swiftc_bin()) + .arg("-print-target-info") + .output() + .ok()?; + if !output.status.success() { + return None; + } + let info = String::from_utf8_lossy(&output.stdout); + + // Extract the value of `"runtimeResourcePath": "..."` without pulling in a + // JSON dependency. + let key = "\"runtimeResourcePath\""; + let start = info.find(key)?; + let rest = &info[start + key.len()..]; + let colon = rest.find(':')?; + let after = &rest[colon + 1..]; + let open = after.find('"')?; + let value_start = &after[open + 1..]; + let close = value_start.find('"')?; + let resource_path = &value_start[..close]; + + Some(PathBuf::from(resource_path).join("linux")) +} + +/// The `swift` driver to invoke: `$SWIFT` if set, otherwise `swift` from `PATH`. +/// This keeps the build tool-agnostic — any Swift install works; no particular +/// version manager is required. +fn swift_bin() -> String { + env::var("SWIFT").unwrap_or_else(|_| "swift".to_string()) +} + +/// The `swiftc` compiler to invoke: `$SWIFTC` if set, otherwise `swiftc` from +/// `PATH`. +fn swiftc_bin() -> String { + env::var("SWIFTC").unwrap_or_else(|_| "swiftc".to_string()) +} + +/// Some environments (notably GitHub Codespaces) inject +/// `GIT_CONFIG_KEY_0=safe.bareRepository` / `GIT_CONFIG_VALUE_0=explicit`, which +/// breaks the cached bare git repositories `swift build` uses. When exactly that +/// key is present, relax it to `all` for the `swift build` subprocess only +/// (rather than unconditionally, which could clobber an unrelated +/// `GIT_CONFIG_VALUE_0`). +fn apply_bare_repository_workaround(command: &mut Command) { + if env::var("GIT_CONFIG_KEY_0").as_deref() == Ok("safe.bareRepository") { + command.env("GIT_CONFIG_VALUE_0", "all"); + } +} diff --git a/unified/swift-syntax-rs/src/lib.rs b/unified/swift-syntax-rs/src/lib.rs new file mode 100644 index 00000000000..83dc5e2c6fb --- /dev/null +++ b/unified/swift-syntax-rs/src/lib.rs @@ -0,0 +1,125 @@ +//! Rust wrapper around the [swift-syntax](https://github.com/swiftlang/swift-syntax) +//! package, allowing Swift source code to be parsed from Rust. +//! +//! The heavy lifting is done by a small Swift shim (see `swift/`) that links +//! against `SwiftSyntax`/`SwiftParser` and exposes a tiny C ABI. This module +//! provides safe Rust bindings on top of that ABI. + +use std::ffi::{CStr, CString}; +use std::os::raw::c_char; + +// C ABI exported by the `SwiftSyntaxFFI` dynamic library. +unsafe extern "C" { + /// Parse a NUL-terminated Swift source string, returning a heap-allocated + /// NUL-terminated JSON string (or null on failure). The caller owns the + /// returned pointer and must release it with `ssr_string_free`. + fn ssr_parse_json(source: *const c_char) -> *mut c_char; + + /// Free a string previously returned by `ssr_parse_json`. + fn ssr_string_free(ptr: *mut c_char); +} + +/// Errors that can occur while parsing Swift source. +#[derive(Debug)] +pub enum ParseError { + /// The provided source contained an interior NUL byte. + NulByte, + /// The Swift side failed to produce a result. + SwiftFailure, +} + +impl std::fmt::Display for ParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ParseError::NulByte => write!(f, "source contained an interior NUL byte"), + ParseError::SwiftFailure => write!(f, "swift-syntax failed to parse the source"), + } + } +} + +impl std::error::Error for ParseError {} + +/// Parse the given Swift `source` and return a JSON representation of its +/// syntax tree. +/// +/// # Example +/// ```no_run +/// let json = swift_syntax_rs::parse_to_json("let x = 1").unwrap(); +/// println!("{json}"); +/// ``` +pub fn parse_to_json(source: &str) -> Result { + let c_source = CString::new(source).map_err(|_| ParseError::NulByte)?; + + // SAFETY: `c_source` is a valid NUL-terminated string for the duration of + // the call. The returned pointer, if non-null, is owned by us and freed via + // `ssr_string_free` before returning. + unsafe { + let ptr = ssr_parse_json(c_source.as_ptr()); + if ptr.is_null() { + return Err(ParseError::SwiftFailure); + } + let json = CStr::from_ptr(ptr).to_string_lossy().into_owned(); + ssr_string_free(ptr); + Ok(json) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_simple_declaration() { + let json = parse_to_json("let x = 1").expect("parsing should succeed"); + assert!( + json.contains("\"kind\":\"sourceFile\""), + "unexpected tree: {json}" + ); + assert!(json.contains("\"text\":\"x\""), "unexpected tree: {json}"); + // Source ranges are emitted for every node. + assert!(json.contains("\"range\""), "missing ranges: {json}"); + assert!( + json.contains("\"line\"") && json.contains("\"column\"") && json.contains("\"offset\""), + "missing location fields: {json}" + ); + } + + #[test] + fn preserves_named_structure() { + let json = parse_to_json("let x = 1").expect("parsing should succeed"); + // Layout children are embedded directly under their field name. + assert!( + json.contains("\"initializer\""), + "missing initializer field: {json}" + ); + // Collection-valued fields are elided to plain JSON arrays. + assert!( + json.contains("\"statements\":["), + "statements should be an array: {json}" + ); + assert!( + json.contains("\"bindings\":["), + "bindings should be an array: {json}" + ); + } + + #[test] + fn parses_empty_source() { + let json = parse_to_json("").expect("parsing empty source should succeed"); + assert!( + json.contains("\"kind\":\"sourceFile\""), + "unexpected tree: {json}" + ); + } + + #[test] + fn captures_trivia() { + // A leading comment is kept as trivia on the token it precedes. + let json = parse_to_json("// hi\nlet x = 1").expect("parsing should succeed"); + assert!(json.contains("\"leadingTrivia\""), "missing trivia: {json}"); + assert!( + json.contains("lineComment"), + "comment trivia not captured: {json}" + ); + } +} diff --git a/unified/swift-syntax-rs/src/main.rs b/unified/swift-syntax-rs/src/main.rs new file mode 100644 index 00000000000..d79cd2ffef1 --- /dev/null +++ b/unified/swift-syntax-rs/src/main.rs @@ -0,0 +1,42 @@ +//! Small demo CLI: parse Swift source and print the syntax tree as JSON. +//! +//! Usage: +//! swift-syntax-parse [FILE] +//! +//! Reads from FILE if given, otherwise from standard input. + +use std::io::Read; +use std::process::ExitCode; + +fn main() -> ExitCode { + let source = match read_source() { + Ok(s) => s, + Err(e) => { + eprintln!("error reading source: {e}"); + return ExitCode::FAILURE; + } + }; + + match swift_syntax_rs::parse_to_json(&source) { + Ok(json) => { + println!("{json}"); + ExitCode::SUCCESS + } + Err(e) => { + eprintln!("error: {e}"); + ExitCode::FAILURE + } + } +} + +fn read_source() -> std::io::Result { + let mut args = std::env::args().skip(1); + match args.next() { + Some(path) => std::fs::read_to_string(path), + None => { + let mut buf = String::new(); + std::io::stdin().read_to_string(&mut buf)?; + Ok(buf) + } + } +} diff --git a/unified/swift-syntax-rs/swift/Package.resolved b/unified/swift-syntax-rs/swift/Package.resolved new file mode 100644 index 00000000000..9cebf34ec54 --- /dev/null +++ b/unified/swift-syntax-rs/swift/Package.resolved @@ -0,0 +1,15 @@ +{ + "originHash" : "ad84ad340ee01aa6e38b877dd29e38fa9cb40603dfdefa4e6a69e27f2db208d1", + "pins" : [ + { + "identity" : "swift-syntax", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-syntax.git", + "state" : { + "revision" : "4799286537280063c85a32f09884cfbca301b1a1", + "version" : "602.0.0" + } + } + ], + "version" : 3 +} diff --git a/unified/swift-syntax-rs/swift/Package.swift b/unified/swift-syntax-rs/swift/Package.swift new file mode 100644 index 00000000000..1b0111364ce --- /dev/null +++ b/unified/swift-syntax-rs/swift/Package.swift @@ -0,0 +1,30 @@ +// swift-tools-version:6.0 +import PackageDescription + +let package = Package( + name: "SwiftSyntaxFFI", + products: [ + // Dynamic library so the produced .so records its dependency on the + // Swift runtime libraries, keeping the Rust link step simple. + .library( + name: "SwiftSyntaxFFI", + type: .dynamic, + targets: ["SwiftSyntaxFFI"] + ) + ], + dependencies: [ + .package( + url: "https://github.com/swiftlang/swift-syntax.git", + exact: "602.0.0" + ) + ], + targets: [ + .target( + name: "SwiftSyntaxFFI", + dependencies: [ + .product(name: "SwiftSyntax", package: "swift-syntax"), + .product(name: "SwiftParser", package: "swift-syntax"), + ] + ) + ] +) diff --git a/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift b/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift new file mode 100644 index 00000000000..177e8ff319d --- /dev/null +++ b/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift @@ -0,0 +1,145 @@ +import Foundation +import SwiftParser + +// `@_spi(RawSyntax)` exposes the `childName(_:)` helper that maps a child's +// key path to its field name in the parent layout; used to emit named fields. +@_spi(RawSyntax) import SwiftSyntax + +#if canImport(Glibc) + import Glibc +#elseif canImport(Darwin) + import Darwin +#endif + +/// Convert an absolute position into an `{ offset, line, column }` dictionary. +/// +/// `offset` is a UTF-8 byte offset; `line`/`column` are 1-based. +private func location( + _ position: AbsolutePosition, + _ converter: SourceLocationConverter +) -> [String: Any] { + let loc = converter.location(for: position) + return [ + "offset": position.utf8Offset, + "line": loc.line, + "column": loc.column, + ] +} + +/// Trivia kinds worth preserving in the serialized tree. Comments carry +/// developer intent (including doc comments), and `unexpectedText` flags source +/// the parser had to skip. Whitespace and multi-line-string escape markers are +/// dropped: node ranges already encode positions, so they would only bloat the +/// output. +private let keptTriviaKinds: Set = [ + "lineComment", + "blockComment", + "docLineComment", + "docBlockComment", + "unexpectedText", +] + +/// Serialize a trivia collection into an array of `{ kind, text }` pieces, +/// keeping only the kinds in `keptTriviaKinds`. +private func serializeTrivia(_ trivia: Trivia) -> [Any] { + trivia.pieces.compactMap { piece -> [String: Any]? in + // The label of an enum case mirror is the case name (e.g. "spaces", + // "lineComment"), which gives us a stable kind without an exhaustive + // switch over every `TriviaPiece` case. + let kind = Mirror(reflecting: piece).children.first?.label ?? "\(piece)" + guard keptTriviaKinds.contains(kind) else { return nil } + return [ + "kind": kind, + "text": Trivia(pieces: [piece]).description, + ] + } +} + +/// Recursively convert a SwiftSyntax node into a JSON-serializable value. +/// +/// * Tokens and layout nodes (e.g. `functionDecl`) become objects carrying +/// `kind` and source `range`. Layout nodes additionally embed their children +/// directly as members keyed by the child's name in the parent (e.g. `name`, +/// `signature`, `body`); absent optional children are omitted. Field names +/// never collide with `kind`/`range`. +/// * Collection nodes (e.g. `codeBlockItemList`) are *elided*: they become a +/// plain array of their serialized elements, taking the place of the +/// collection node itself. A list-valued layout field (e.g. `parameters`) is +/// therefore simply a JSON array. This drops the collection node's own +/// `kind`/`range`, which are unnamed and largely recoverable from the +/// elements. +private func serialize( + _ node: Syntax, + _ converter: SourceLocationConverter +) -> Any { + if node.kind.isSyntaxCollection { + return node.children(viewMode: .sourceAccurate).map { + serialize($0, converter) + } + } + + // Source range covering the node's content, excluding surrounding trivia. + let range: [String: Any] = [ + "start": location(node.positionAfterSkippingLeadingTrivia, converter), + "end": location(node.endPositionBeforeTrailingTrivia, converter), + ] + + if let token = node.as(TokenSyntax.self) { + return [ + "kind": "token", + "tokenKind": "\(token.tokenKind)", + "text": token.text, + "range": range, + "leadingTrivia": serializeTrivia(token.leadingTrivia), + "trailingTrivia": serializeTrivia(token.trailingTrivia), + ] + } + + var result: [String: Any] = [ + "kind": "\(node.kind)", + "range": range, + ] + var unnamed = 0 + for child in node.children(viewMode: .sourceAccurate) { + // Layout children are named; embed each under its field name in the + // parent (the same mechanism SwiftSyntax uses for its debug dump). A + // child that is a collection serializes to an array (see above). + if let keyPath = child.keyPathInParent, let name = childName(keyPath) { + result[name] = serialize(child, converter) + } else { + // Defensive fallback for any unnamed layout child. + result["child\(unnamed)"] = serialize(child, converter) + unnamed += 1 + } + } + return result +} + +/// Parse the given NUL-terminated Swift source string and return a +/// heap-allocated, NUL-terminated JSON representation of the syntax tree. +/// +/// The returned pointer is owned by the caller and MUST be released with +/// `ssr_string_free`. Returns `nil` on failure. +@_cdecl("ssr_parse_json") +public func ssr_parse_json(_ source: UnsafePointer?) -> UnsafeMutablePointer? { + guard let source = source else { return nil } + let code = String(cString: source) + let tree = Parser.parse(source: code) + let converter = SourceLocationConverter(fileName: "", tree: tree) + let json = serialize(Syntax(tree), converter) + + guard + let data = try? JSONSerialization.data( + withJSONObject: json, options: [.sortedKeys]), + let string = String(data: data, encoding: .utf8) + else { + return nil + } + return strdup(string) +} + +/// Free a string previously returned by this library. +@_cdecl("ssr_string_free") +public func ssr_string_free(_ ptr: UnsafeMutablePointer?) { + free(ptr) +} From 87c81731252678b5fe3cb7d14a0a19c65e4243f9 Mon Sep 17 00:00:00 2001 From: Taus Date: Tue, 7 Jul 2026 13:35:07 +0000 Subject: [PATCH 33/90] unified: Don't emit empty trivia These were taking up roughly 20% of the JSON payload. --- .../SwiftSyntaxFFI/SwiftSyntaxFFI.swift | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift b/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift index 177e8ff319d..3dcfcb2c389 100644 --- a/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift +++ b/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift @@ -57,11 +57,14 @@ private func serializeTrivia(_ trivia: Trivia) -> [Any] { /// Recursively convert a SwiftSyntax node into a JSON-serializable value. /// -/// * Tokens and layout nodes (e.g. `functionDecl`) become objects carrying -/// `kind` and source `range`. Layout nodes additionally embed their children -/// directly as members keyed by the child's name in the parent (e.g. `name`, -/// `signature`, `body`); absent optional children are omitted. Field names -/// never collide with `kind`/`range`. +/// * Tokens carry `kind`, `tokenKind`, `text`, and `range`, plus +/// `leadingTrivia`/`trailingTrivia` — but only when non-empty (after +/// filtering, most tokens have no trivia, so the keys are simply absent). +/// * Layout nodes (e.g. `functionDecl`) carry `kind` and source `range`, and +/// additionally embed their children directly as members keyed by the +/// child's name in the parent (e.g. `name`, `signature`, `body`); absent +/// optional children are omitted. Field names never collide with +/// `kind`/`range`. /// * Collection nodes (e.g. `codeBlockItemList`) are *elided*: they become a /// plain array of their serialized elements, taking the place of the /// collection node itself. A list-valued layout field (e.g. `parameters`) is @@ -85,14 +88,22 @@ private func serialize( ] if let token = node.as(TokenSyntax.self) { - return [ + var result: [String: Any] = [ "kind": "token", "tokenKind": "\(token.tokenKind)", "text": token.text, "range": range, - "leadingTrivia": serializeTrivia(token.leadingTrivia), - "trailingTrivia": serializeTrivia(token.trailingTrivia), ] + // Only emit trivia when present; after filtering, most tokens have none. + let leading = serializeTrivia(token.leadingTrivia) + if !leading.isEmpty { + result["leadingTrivia"] = leading + } + let trailing = serializeTrivia(token.trailingTrivia) + if !trailing.isEmpty { + result["trailingTrivia"] = trailing + } + return result } var result: [String: Any] = [ From 446ff7c463bf3d8496b13458fc1a6749b4c2eb43 Mon Sep 17 00:00:00 2001 From: Taus Date: Tue, 7 Jul 2026 13:36:10 +0000 Subject: [PATCH 34/90] unified: Add mapping from swift-syntax JSON to yeast AST Adds a preliminary mapping that decodes the JSON into the yeast AST. The JSON AST itself is still very verbose, but it would be useful to see if it actually contains all of the things we want it to contain. --- Cargo.lock | 4 + shared/yeast/src/lib.rs | 42 ++- unified/swift-syntax-rs/BUILD.bazel | 11 +- unified/swift-syntax-rs/Cargo.toml | 4 + unified/swift-syntax-rs/README.md | 37 ++- unified/swift-syntax-rs/src/lib.rs | 2 + unified/swift-syntax-rs/src/yeast_adapter.rs | 282 +++++++++++++++++++ 7 files changed, 369 insertions(+), 13 deletions(-) create mode 100644 unified/swift-syntax-rs/src/yeast_adapter.rs diff --git a/Cargo.lock b/Cargo.lock index 7a9f1966791..24885fea856 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2853,6 +2853,10 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "swift-syntax-rs" version = "0.1.0" +dependencies = [ + "serde_json", + "yeast", +] [[package]] name = "syn" diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index e6759f1ddb9..fa9074805b8 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -333,6 +333,28 @@ impl Ast { ast } + /// Construct an empty AST backed by `schema`, for building a tree + /// programmatically from a source other than a tree-sitter parse (e.g. an + /// external parser's output). Populate it with [`Ast::create_node`] / + /// [`Ast::create_node_with_range`] and then designate the root with + /// [`Ast::set_root`]. The `schema` must already have every node kind and + /// field name registered (see [`schema::Schema`]). + pub fn with_schema(schema: schema::Schema) -> Self { + Self { + root: Id(0), + nodes: Vec::new(), + schema, + source: Vec::new(), + } + } + + /// Set the original source bytes, used to resolve `NodeContent::Range` + /// nodes to text. Nodes built with inline `NodeContent::DynamicString` + /// content do not require this. + pub fn set_source(&mut self, source: Vec) { + self.source = source; + } + /// Returns the source text for `id`, resolving `NodeContent::Range` /// against the stored source bytes when available. pub fn source_text(&self, id: Id) -> String { @@ -463,6 +485,24 @@ impl Ast { Id(id) } + /// Register a named node kind, returning its id (idempotent). Lets callers + /// build an AST in a single pass, registering kinds as nodes are created + /// rather than pre-populating the schema. + pub fn register_kind(&mut self, name: &str) -> KindId { + self.schema.register_kind(name) + } + + /// Register an anonymous (unnamed) token kind, returning its id + /// (idempotent). Anonymous tokens are keyed by their text (e.g. `"func"`). + pub fn register_unnamed_kind(&mut self, name: &str) -> KindId { + self.schema.register_unnamed_kind(name) + } + + /// Register a field name, returning its id (idempotent). + pub fn register_field(&mut self, name: &str) -> FieldId { + self.schema.register_field(name) + } + fn union_source_range_of_children( &self, fields: &BTreeMap>, @@ -604,7 +644,7 @@ impl Ast { } } - fn id_for_unnamed_node_kind(&self, kind: &str) -> Option { + pub fn id_for_unnamed_node_kind(&self, kind: &str) -> Option { let id = self.schema.id_for_unnamed_node_kind(kind).unwrap_or(0); if id == 0 { None diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index 10d11a1bec1..3a2e5ed0796 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -28,9 +28,16 @@ swift_library( # Swift shim in the local `cargo` workflow. rust_library( name = "swift_syntax_rs", - srcs = ["src/lib.rs"], + srcs = glob( + ["src/**/*.rs"], + exclude = ["src/main.rs"], + ), edition = "2024", - deps = [":swift_syntax_ffi"], + deps = [ + ":swift_syntax_ffi", + "//shared/yeast", + "@vendor_ts__serde_json-1.0.145//:serde_json", + ], ) rust_binary( diff --git a/unified/swift-syntax-rs/Cargo.toml b/unified/swift-syntax-rs/Cargo.toml index 6957fb0e50e..57624646848 100644 --- a/unified/swift-syntax-rs/Cargo.toml +++ b/unified/swift-syntax-rs/Cargo.toml @@ -12,3 +12,7 @@ path = "src/lib.rs" [[bin]] name = "swift-syntax-parse" path = "src/main.rs" + +[dependencies] +serde_json = "1.0" +yeast = { path = "../../shared/yeast" } diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index 08796b3a6a7..bb4f7f667a5 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -13,8 +13,8 @@ The emitted JSON tree preserves the AST's named structure. Every node has a `kind` and a `range` with `start`/`end` positions (UTF-8 `offset` plus 1-based `line`/`column`). Beyond that: -- **Tokens** carry `text`, `tokenKind`, and `leadingTrivia`/`trailingTrivia` - arrays of `{ kind, text }` pieces. +- **Tokens** carry `text`, `tokenKind`, and — only when non-empty — + `leadingTrivia`/`trailingTrivia` arrays of `{ kind, text }` pieces. - **Layout nodes** (e.g. `functionDecl`) embed their children directly as members keyed by the child's name in the parent (`name`, `signature`, `body`, …), alongside `kind`/`range`. Absent optional children are omitted. @@ -49,9 +49,7 @@ abbreviated here as `…`): "kind": "token", "text": "let", "tokenKind": "keyword(SwiftSyntax.Keyword.let)", - "range": …, - "leadingTrivia": [], - "trailingTrivia": [] + "range": … }, "bindings": [ { @@ -60,12 +58,12 @@ abbreviated here as `…`): "pattern": { "kind": "identifierPattern", "range": …, - "identifier": { "kind": "token", "text": "x", "tokenKind": "identifier(\"x\")", "range": …, "leadingTrivia": [], "trailingTrivia": [] } + "identifier": { "kind": "token", "text": "x", "tokenKind": "identifier(\"x\")", "range": … } }, "initializer": { "kind": "initializerClause", "range": …, - "equal": { "kind": "token", "text": "=", "tokenKind": "equal", "range": …, "leadingTrivia": [], "trailingTrivia": [] }, + "equal": { "kind": "token", "text": "=", "tokenKind": "equal", "range": … }, "value": { "kind": "integerLiteralExpr", "range": …, @@ -74,7 +72,6 @@ abbreviated here as `…`): "text": "1", "tokenKind": "integerLiteral(\"1\")", "range": …, - "leadingTrivia": [], "trailingTrivia": [ { "kind": "lineComment", "text": "// c" } ] } } @@ -84,14 +81,15 @@ abbreviated here as `…`): } } ], - "endOfFileToken": { "kind": "token", "text": "", "tokenKind": "endOfFile", "range": …, "leadingTrivia": [], "trailingTrivia": [] } + "endOfFileToken": { "kind": "token", "text": "", "tokenKind": "endOfFile", "range": … } } ``` Note how `statements`, `bindings`, `attributes`, and `modifiers` are plain arrays (their collection nodes are elided), layout children such as `bindingSpecifier` and `initializer` are embedded by name, and the `// c` -comment rides along as `trailingTrivia` on the token it follows. +comment rides along as `trailingTrivia` on the token it follows. Tokens without +trivia (most of them) simply omit the `leadingTrivia`/`trailingTrivia` keys. ## Prerequisites @@ -171,10 +169,29 @@ CLI (reads a file argument or stdin, prints the syntax tree as JSON): echo 'let x = 1' | cargo run --bin swift-syntax-parse ``` +## Converting to a yeast AST + +For use in the CodeQL extractor, the JSON tree can be converted into a +[`yeast::Ast`](../../shared/yeast) — the in-memory format the extractor's +rewrite rules operate on — via [`yeast_adapter::json_to_ast`](src/yeast_adapter.rs): + +```rust +let json = swift_syntax_rs::parse_to_json("let x = 1")?; +let ast = swift_syntax_rs::yeast_adapter::json_to_ast(&json)?; +``` + +The adapter mirrors tree-sitter's node model, which is what yeast expects: +layout nodes and varying tokens (identifiers, literals, operators) become +**named** nodes; fixed tokens (keywords, punctuation) become **anonymous** +nodes keyed by their text. It preserves swift-syntax's own kind/field names — +aligning them with the tree-sitter-swift schema so the existing rewrite rules +fire is a separate, later step. + ## Layout - `swift/` — Swift package exposing the `ssr_parse_json` / `ssr_string_free` C ABI. - `build.rs` — builds the Swift package and emits link/rpath flags (local `cargo` only). - `BUILD.bazel` — Bazel targets for the hermetic CI build (swift_library + rust targets). - `src/lib.rs` — safe Rust bindings (`parse_to_json`). +- `src/yeast_adapter.rs` — converts the JSON tree into a `yeast::Ast`. - `src/main.rs` — demo CLI. diff --git a/unified/swift-syntax-rs/src/lib.rs b/unified/swift-syntax-rs/src/lib.rs index 83dc5e2c6fb..aa711e322a4 100644 --- a/unified/swift-syntax-rs/src/lib.rs +++ b/unified/swift-syntax-rs/src/lib.rs @@ -5,6 +5,8 @@ //! against `SwiftSyntax`/`SwiftParser` and exposes a tiny C ABI. This module //! provides safe Rust bindings on top of that ABI. +pub mod yeast_adapter; + use std::ffi::{CStr, CString}; use std::os::raw::c_char; diff --git a/unified/swift-syntax-rs/src/yeast_adapter.rs b/unified/swift-syntax-rs/src/yeast_adapter.rs new file mode 100644 index 00000000000..5f84e1304bb --- /dev/null +++ b/unified/swift-syntax-rs/src/yeast_adapter.rs @@ -0,0 +1,282 @@ +//! Adapter that converts the swift-syntax JSON tree (see [`crate::parse_to_json`]) +//! into a [`yeast::Ast`], the in-memory format the CodeQL desugaring rules +//! operate on. +//! +//! The mapping mirrors tree-sitter's node model, which is what yeast (and the +//! extractor's rewrite rules) expect: +//! +//! * **Layout nodes** (e.g. `functionDecl`) and **varying tokens** (identifiers, +//! literals, operators — the ones whose text is not determined by their kind) +//! become **named** nodes, keyed by their kind name. +//! * **Fixed tokens** (keywords and punctuation, whose text is fully determined +//! by their kind) become **anonymous** nodes, keyed by their text — exactly +//! how tree-sitter models anonymous tokens (e.g. `"func"`, `"->"`). +//! * Collection nodes are already elided to JSON arrays upstream, so a +//! list-valued field maps directly to that field holding several children. +//! +//! Note: this preserves swift-syntax's own kind/field names. Aligning those +//! names with the tree-sitter-swift schema (so the existing rewrite rules fire) +//! is a separate, later step; this module is only concerned with getting the +//! tree into yeast's format. + +use std::collections::BTreeMap; + +use serde_json::Value; +use yeast::schema::Schema; +use yeast::{Ast, Id, NodeContent}; + +/// swift-syntax `TokenKind` cases whose text is *not* determined by the kind +/// (i.e. `TokenKind.defaultText == nil`). These carry varying information and +/// are modelled as named leaf nodes; every other token is a fixed +/// keyword/punctuation token modelled as an anonymous token keyed by its text. +const VARYING_TOKEN_KINDS: &[&str] = &[ + "identifier", + "integerLiteral", + "floatLiteral", + "stringSegment", + "binaryOperator", + "prefixOperator", + "postfixOperator", + "dollarIdentifier", + "regexLiteralPattern", + "rawStringPoundDelimiter", + "regexPoundDelimiter", + "shebang", + "unknown", +]; + +/// Keys of a node object that carry metadata rather than a structural child. +fn is_metadata_key(key: &str) -> bool { + matches!( + key, + "kind" | "range" | "tokenKind" | "text" | "leadingTrivia" | "trailingTrivia" + ) +} + +/// The classification of a JSON node into a yeast kind name and named-ness. +struct KindInfo { + /// The name under which the kind is registered in the schema. + name: String, + /// `true` for named nodes (layout nodes + varying tokens), `false` for + /// anonymous tokens (fixed keywords/punctuation). + is_named: bool, + /// The leaf text for tokens (empty for layout nodes). + text: String, +} + +/// Determine the kind name / named-ness / text for a JSON node object. +fn classify(node: &Value) -> Result { + let kind = node + .get("kind") + .and_then(Value::as_str) + .ok_or("node object is missing a string `kind`")?; + + if kind != "token" { + // Layout node: named, keyed by its kind, no leaf text. + return Ok(KindInfo { + name: kind.to_string(), + is_named: true, + text: String::new(), + }); + } + + let token_kind = node + .get("tokenKind") + .and_then(Value::as_str) + .ok_or("token is missing a string `tokenKind`")?; + let text = node + .get("text") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + + // The case name is the part before any `(payload)` in the debug rendering. + let case_name = token_kind.split('(').next().unwrap_or(token_kind); + + if VARYING_TOKEN_KINDS.contains(&case_name) { + // Varying token: named leaf, keyed by its case name. + Ok(KindInfo { + name: case_name.to_string(), + is_named: true, + text, + }) + } else { + // Fixed token: anonymous, keyed by its text (as tree-sitter does). + // `endOfFile` has empty text, so fall back to the case name there. + let name = if text.is_empty() { + case_name.to_string() + } else { + text.clone() + }; + Ok(KindInfo { + name, + is_named: false, + text, + }) + } +} + +/// Iterate over a node object's structural (field, value) pairs in a stable +/// order, skipping metadata keys. +fn field_entries(node: &Value) -> Vec<(&str, &Value)> { + node.as_object() + .map(|map| { + map.iter() + .filter(|(k, _)| !is_metadata_key(k)) + .map(|(k, v)| (k.as_str(), v)) + .collect() + }) + .unwrap_or_default() +} + +/// The child node objects held by a field value, which is either a single node +/// object or an array of them (an elided collection). +fn children_of(value: &Value) -> Vec<&Value> { + match value { + Value::Array(items) => items.iter().collect(), + other => vec![other], + } +} + +/// Recursively build `node` (and its descendants) into `ast`, returning its id. +/// +/// This is a single traversal: each node's kind and field names are registered +/// in the schema on the fly, immediately before the node is created. Children +/// are built first so a parent's field lists reference existing ids. +fn build(node: &Value, ast: &mut Ast) -> Result { + let info = classify(node)?; + + let mut fields: BTreeMap> = BTreeMap::new(); + for (field, value) in field_entries(node) { + let field_id = ast.register_field(field); + let mut ids = Vec::new(); + for child in children_of(value) { + ids.push(build(child, ast)?); + } + fields.insert(field_id, ids); + } + + let kind_id = if info.is_named { + ast.register_kind(&info.name) + } else { + ast.register_unnamed_kind(&info.name) + }; + + Ok(ast.create_node_with_range( + kind_id, + NodeContent::DynamicString(info.text), + fields, + info.is_named, + None, + )) +} + +/// Convert a swift-syntax JSON tree (as produced by [`crate::parse_to_json`]) +/// into a [`yeast::Ast`]. +pub fn json_to_ast(json: &str) -> Result { + let root: Value = serde_json::from_str(json).map_err(|e| format!("invalid JSON: {e}"))?; + + let mut ast = Ast::with_schema(Schema::new()); + let root_id = build(&root, &mut ast)?; + ast.set_root(root_id); + Ok(ast) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A hand-written JSON tree exercising layout nodes, a named (varying) + /// token, a fixed keyword token, and an elided collection field — so the + /// adapter is tested without needing the Swift toolchain. + fn sample_json() -> &'static str { + r#"{ + "kind": "sourceFile", + "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":9,"line":1,"column":10}}, + "statements": [ + { + "kind": "variableDecl", + "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":9,"line":1,"column":10}}, + "bindingSpecifier": { + "kind": "token", + "tokenKind": "keyword(SwiftSyntax.Keyword.let)", + "text": "let", + "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":3,"line":1,"column":4}} + }, + "name": { + "kind": "token", + "tokenKind": "identifier(\"x\")", + "text": "x", + "range": {"start":{"offset":4,"line":1,"column":5},"end":{"offset":5,"line":1,"column":6}} + } + } + ] + }"# + } + + #[test] + fn builds_ast_from_json() { + let ast = json_to_ast(sample_json()).expect("adapter should succeed"); + let root = ast.get_root(); + let root_node = ast.get_node(root).expect("root exists"); + assert_eq!(root_node.kind_name(), "sourceFile"); + assert!(root_node.is_named()); + } + + #[test] + fn classifies_named_and_anonymous_tokens() { + let ast = json_to_ast(sample_json()).expect("adapter should succeed"); + // Walk all nodes and collect (kind_name, is_named) for the two tokens. + let mut let_named = None; + let mut ident_named = None; + for node in ast.nodes() { + match node.kind_name() { + "let" => let_named = Some(node.is_named()), + "identifier" => ident_named = Some(node.is_named()), + _ => {} + } + } + // Fixed keyword `let` is anonymous (keyed by its text "let"). + assert_eq!(let_named, Some(false)); + // Varying `identifier` is a named leaf. + assert_eq!(ident_named, Some(true)); + } + + #[test] + fn preserves_leaf_text() { + let ast = json_to_ast(sample_json()).expect("adapter should succeed"); + let ident = ast + .nodes() + .iter() + .enumerate() + .find(|(_, n)| n.kind_name() == "identifier") + .map(|(i, _)| Id(i)) + .expect("identifier node exists"); + assert_eq!(ast.source_text(ident), "x"); + } + + /// End-to-end: real Swift source parsed by the shim, then adapted into a + /// `yeast::Ast`. Requires the Swift toolchain (like the crate's FFI tests). + #[test] + fn end_to_end_from_swift_source() { + let json = crate::parse_to_json("func f(n: Int) -> Int { return n }") + .expect("parsing should succeed"); + let ast = json_to_ast(&json).expect("adapter should succeed"); + + let root = ast.get_node(ast.get_root()).expect("root exists"); + assert_eq!(root.kind_name(), "sourceFile"); + + // The tree contains a `functionDecl` layout node and an anonymous + // `func` keyword token keyed by its text. + let mut kinds: Vec<&str> = ast.nodes().iter().map(|n| n.kind_name()).collect(); + kinds.sort_unstable(); + assert!( + kinds.contains(&"functionDecl"), + "expected a functionDecl node, got kinds: {kinds:?}" + ); + assert!( + kinds.contains(&"func"), + "expected an anonymous `func` token, got kinds: {kinds:?}" + ); + } +} From 8909fae81e7f8b93df2822b0a4c1e3ddce7a4dae Mon Sep 17 00:00:00 2001 From: Taus Date: Tue, 7 Jul 2026 14:28:19 +0000 Subject: [PATCH 35/90] unified: Extract locations from `swift-syntax` AST Introduces new yeast types for `Point`s and `Range`s (corresponding to the tree-sitter equivalents), since it seemed silly to have `swift-syntax-rs` pull in `tree-sitter` just to have those types available. This _does_ mean there is a slight overhead in the shared extractor when converting between these types, but I think this is negligible. --- .../src/extractor/mod.rs | 12 +++- shared/yeast-macros/src/parse.rs | 2 +- shared/yeast/src/build.rs | 10 +-- shared/yeast/src/lib.rs | 61 +++++++++++++------ shared/yeast/src/range.rs | 29 ++++++--- unified/swift-syntax-rs/src/yeast_adapter.rs | 49 ++++++++++++++- 6 files changed, 124 insertions(+), 39 deletions(-) diff --git a/shared/tree-sitter-extractor/src/extractor/mod.rs b/shared/tree-sitter-extractor/src/extractor/mod.rs index 54b01ba5146..474dc096a2a 100644 --- a/shared/tree-sitter-extractor/src/extractor/mod.rs +++ b/shared/tree-sitter-extractor/src/extractor/mod.rs @@ -81,10 +81,18 @@ impl AstNode for yeast::Node { yeast::Node::is_extra(self) } fn start_position(&self) -> tree_sitter::Point { - yeast::Node::start_position(self) + let p = yeast::Node::start_position(self); + tree_sitter::Point { + row: p.row, + column: p.column, + } } fn end_position(&self) -> tree_sitter::Point { - yeast::Node::end_position(self) + let p = yeast::Node::end_position(self); + tree_sitter::Point { + row: p.row, + column: p.column, + } } fn byte_range(&self) -> std::ops::Range { yeast::Node::byte_range(self) diff --git a/shared/yeast-macros/src/parse.rs b/shared/yeast-macros/src/parse.rs index 01c0b574b1c..55ada358849 100644 --- a/shared/yeast-macros/src/parse.rs +++ b/shared/yeast-macros/src/parse.rs @@ -903,7 +903,7 @@ pub fn parse_rule_top(input: TokenStream) -> Result { Ok(quote! { { let __query = #query_code; - yeast::Rule::new(__query, Box::new(|__ast: &mut yeast::Ast, mut __captures: yeast::captures::Captures, __fresh: &yeast::tree_builder::FreshScope, __source_range: Option, __user_ctx: &mut _, __translator: yeast::TranslatorHandle<'_, _>| { + yeast::Rule::new(__query, Box::new(|__ast: &mut yeast::Ast, mut __captures: yeast::captures::Captures, __fresh: &yeast::tree_builder::FreshScope, __source_range: Option, __user_ctx: &mut _, __translator: yeast::TranslatorHandle<'_, _>| { // Auto-translation prefix: recursively translate every // captured node before invoking the user's transform body, // except for `@@name` captures listed in `__skip` which the diff --git a/shared/yeast/src/build.rs b/shared/yeast/src/build.rs index 68a4c883242..508a1e73134 100644 --- a/shared/yeast/src/build.rs +++ b/shared/yeast/src/build.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use crate::captures::Captures; use crate::tree_builder::FreshScope; -use crate::{Ast, FieldId, Id, NodeContent, TranslatorHandle}; +use crate::{Ast, FieldId, Id, NodeContent, Range, TranslatorHandle}; /// Context for building new AST nodes during a transformation. /// @@ -34,7 +34,7 @@ pub struct BuildCtx<'a, C: 'a = ()> { pub captures: &'a Captures, pub fresh: &'a FreshScope, /// Source range of the matched node, inherited by synthetic nodes. - pub source_range: Option, + pub source_range: Option, /// User-supplied context, accessible directly via `ctx.field` (via Deref). pub user_ctx: &'a mut C, /// Optional translator handle, populated when the context is built by @@ -63,7 +63,7 @@ impl<'a, C> BuildCtx<'a, C> { ast: &'a mut Ast, captures: &'a Captures, fresh: &'a FreshScope, - source_range: Option, + source_range: Option, user_ctx: &'a mut C, ) -> Self { Self { @@ -82,7 +82,7 @@ impl<'a, C> BuildCtx<'a, C> { ast: &'a mut Ast, captures: &'a Captures, fresh: &'a FreshScope, - source_range: Option, + source_range: Option, user_ctx: &'a mut C, translator: TranslatorHandle<'a, C>, ) -> Self { @@ -143,7 +143,7 @@ impl<'a, C> BuildCtx<'a, C> { &mut self, kind: &'static str, value: &str, - source_range: Option, + source_range: Option, ) -> Id { self.ast.create_named_token_with_range( kind, diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index fa9074805b8..b2709ab406e 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -15,11 +15,32 @@ pub mod schema; pub mod tree_builder; mod visitor; +pub use range::{Point, Range}; pub use yeast_macros::{query, rule, rules, tree, trees}; use captures::Captures; use query::QueryNode; +impl From for Point { + fn from(p: tree_sitter::Point) -> Self { + Point { + row: p.row, + column: p.column, + } + } +} + +impl From for Range { + fn from(r: tree_sitter::Range) -> Self { + Range { + start_byte: r.start_byte, + end_byte: r.end_byte, + start_point: r.start_point.into(), + end_point: r.end_point.into(), + } + } +} + /// Node id: an index into the [`Ast`] arena. A newtype around `usize` /// rather than a bare alias so that it can carry its own /// [`YeastDisplay`] / [`YeastSourceRange`] / [`IntoFieldIds`] impls @@ -106,7 +127,7 @@ pub trait YeastDisplay { /// rule's source range. `Id` returns the referenced node's range, letting /// `(kind #{capture})` carry the captured node's location. pub trait YeastSourceRange { - fn yeast_source_range(&self, ast: &Ast) -> Option; + fn yeast_source_range(&self, ast: &Ast) -> Option; } impl YeastDisplay for Id { @@ -116,9 +137,9 @@ impl YeastDisplay for Id { } impl YeastSourceRange for Id { - fn yeast_source_range(&self, ast: &Ast) -> Option { + fn yeast_source_range(&self, ast: &Ast) -> Option { ast.get_node(*self).and_then(|n| match &n.content { - NodeContent::Range(r) => Some(r.clone()), + NodeContent::Range(r) => Some(*r), _ => n.source_range, }) } @@ -134,7 +155,7 @@ macro_rules! impl_yeast_display_via_display { } impl YeastSourceRange for $t { - fn yeast_source_range(&self, _ast: &Ast) -> Option { + fn yeast_source_range(&self, _ast: &Ast) -> Option { None } } @@ -157,7 +178,7 @@ impl YeastDisplay for &T { } impl YeastSourceRange for &T { - fn yeast_source_range(&self, ast: &Ast) -> Option { + fn yeast_source_range(&self, ast: &Ast) -> Option { (**self).yeast_source_range(ast) } } @@ -361,7 +382,7 @@ impl Ast { let Some(node) = self.get_node(id) else { return String::new(); }; - let read_range = |range: &tree_sitter::Range| { + let read_range = |range: &Range| { let start = range.start_byte; let end = range.end_byte; if end <= self.source.len() && start <= end { @@ -459,7 +480,7 @@ impl Ast { content: NodeContent, fields: BTreeMap>, is_named: bool, - source_range: Option, + source_range: Option, ) -> Id { let source_range = match &content { // Parsed nodes already carry an exact source range in their content. @@ -506,11 +527,11 @@ impl Ast { fn union_source_range_of_children( &self, fields: &BTreeMap>, - ) -> Option { + ) -> Option { let mut start_byte: Option = None; let mut end_byte: Option = None; - let mut start_point = tree_sitter::Point { row: 0, column: 0 }; - let mut end_point = tree_sitter::Point { row: 0, column: 0 }; + let mut start_point = Point { row: 0, column: 0 }; + let mut end_point = Point { row: 0, column: 0 }; for child_ids in fields.values() { for &child_id in child_ids { @@ -553,7 +574,7 @@ impl Ast { } match (start_byte, end_byte) { - (Some(start_byte), Some(end_byte)) => Some(tree_sitter::Range { + (Some(start_byte), Some(end_byte)) => Some(Range { start_byte, end_byte, start_point, @@ -571,7 +592,7 @@ impl Ast { &mut self, kind: &'static str, content: String, - source_range: Option, + source_range: Option, ) -> Id { let kind_id = self.schema.id_for_node_kind(kind).unwrap_or_else(|| { panic!("create_named_token: node kind '{kind}' not found in schema") @@ -664,7 +685,7 @@ pub struct Node { /// For synthetic nodes, the source range of the original node they /// were desugared from. Used for location information in TRAP output. #[serde(skip)] - source_range: Option, + source_range: Option, is_named: bool, is_missing: bool, is_extra: bool, @@ -692,11 +713,11 @@ impl Node { self.is_error } - fn fake_point(&self) -> tree_sitter::Point { - tree_sitter::Point { row: 0, column: 0 } + fn fake_point(&self) -> Point { + Point { row: 0, column: 0 } } - pub fn start_position(&self) -> tree_sitter::Point { + pub fn start_position(&self) -> Point { match self.content { NodeContent::Range(range) => range.start_point, _ => self @@ -705,7 +726,7 @@ impl Node { } } - pub fn end_position(&self) -> tree_sitter::Point { + pub fn end_position(&self) -> Point { match self.content { NodeContent::Range(range) => range.end_point, _ => self @@ -754,7 +775,7 @@ impl Node { /// or a new string if the node is synthesized. #[derive(PartialEq, Eq, Debug, Clone, Serialize)] pub enum NodeContent { - Range(#[serde(with = "range::Range")] tree_sitter::Range), + Range(Range), String(&'static str), DynamicString(String), } @@ -767,7 +788,7 @@ impl From<&'static str> for NodeContent { impl From for NodeContent { fn from(value: tree_sitter::Range) -> Self { - NodeContent::Range(value) + NodeContent::Range(value.into()) } } @@ -894,7 +915,7 @@ pub type Transform = Box< &mut Ast, Captures, &tree_builder::FreshScope, - Option, + Option, &mut C, TranslatorHandle<'_, C>, ) -> Result, String> diff --git a/shared/yeast/src/range.rs b/shared/yeast/src/range.rs index ec670b438d5..e56bbcc6fb8 100644 --- a/shared/yeast/src/range.rs +++ b/shared/yeast/src/range.rs @@ -1,21 +1,32 @@ -//! (de)-serialize helpers for tree_sitter::Range +//! Location types for yeast ASTs. +//! +//! These are plain owned structs (mirroring tree-sitter's `Point`/`Range`) so +//! that code building an AST programmatically can populate node locations +//! without depending on tree-sitter. Conversions from the tree-sitter types +//! live in the crate root, next to the `from_tree` construction path. use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize)] -#[serde(remote = "tree_sitter::Point")] +/// A position in a source file: a 0-based `row` (line) and 0-based `column` +/// (UTF-8 byte offset within the line), matching tree-sitter's convention. +#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub struct Point { pub row: usize, pub column: usize, } -#[derive(Serialize, Deserialize)] -#[serde(remote = "tree_sitter::Range")] +impl Point { + pub fn new(row: usize, column: usize) -> Self { + Self { row, column } + } +} + +/// A half-open source range: byte offsets plus start/end [`Point`]s. The end is +/// exclusive, matching tree-sitter's convention. +#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub struct Range { pub start_byte: usize, pub end_byte: usize, - #[serde(with = "Point")] - pub start_point: tree_sitter::Point, - #[serde(with = "Point")] - pub end_point: tree_sitter::Point, + pub start_point: Point, + pub end_point: Point, } diff --git a/unified/swift-syntax-rs/src/yeast_adapter.rs b/unified/swift-syntax-rs/src/yeast_adapter.rs index 5f84e1304bb..8a0baf7a4cd 100644 --- a/unified/swift-syntax-rs/src/yeast_adapter.rs +++ b/unified/swift-syntax-rs/src/yeast_adapter.rs @@ -23,7 +23,7 @@ use std::collections::BTreeMap; use serde_json::Value; use yeast::schema::Schema; -use yeast::{Ast, Id, NodeContent}; +use yeast::{Ast, Id, NodeContent, Point, Range}; /// swift-syntax `TokenKind` cases whose text is *not* determined by the kind /// (i.e. `TokenKind.defaultText == nil`). These carry varying information and @@ -167,10 +167,39 @@ fn build(node: &Value, ast: &mut Ast) -> Result { NodeContent::DynamicString(info.text), fields, info.is_named, - None, + parse_range(node), )) } +/// Parse a node's `range` into a [`yeast::Range`]. +/// +/// The JSON carries, for `start` and `end`, a 0-based UTF-8 file byte `offset`, +/// a 1-based `line`, and a 1-based UTF-8 byte `column`. yeast (like tree-sitter) +/// uses byte offsets with 0-based rows/columns and an exclusive end, so the +/// line/column are shifted down by one. swift-syntax's end position is already +/// exclusive, so the byte offsets map across directly. +fn parse_range(node: &Value) -> Option { + let range = node.get("range")?; + let point = |key: &str| -> Option<(usize, Point)> { + let p = range.get(key)?; + let offset = p.get("offset")?.as_u64()? as usize; + let line = p.get("line")?.as_u64()? as usize; + let column = p.get("column")?.as_u64()? as usize; + Some(( + offset, + Point::new(line.saturating_sub(1), column.saturating_sub(1)), + )) + }; + let (start_byte, start_point) = point("start")?; + let (end_byte, end_point) = point("end")?; + Some(Range { + start_byte, + end_byte, + start_point, + end_point, + }) +} + /// Convert a swift-syntax JSON tree (as produced by [`crate::parse_to_json`]) /// into a [`yeast::Ast`]. pub fn json_to_ast(json: &str) -> Result { @@ -255,6 +284,22 @@ mod tests { assert_eq!(ast.source_text(ident), "x"); } + #[test] + fn maps_source_locations() { + let ast = json_to_ast(sample_json()).expect("adapter should succeed"); + let ident = ast + .nodes() + .iter() + .find(|n| n.kind_name() == "identifier") + .expect("identifier node exists"); + // `x` is at file offset 4..5, line 1, column 5 (1-based) in the JSON, + // which maps to 0-based row 0, column 4 and byte range 4..5. + assert_eq!(ident.start_byte(), 4); + assert_eq!(ident.end_byte(), 5); + assert_eq!(ident.start_position(), Point::new(0, 4)); + assert_eq!(ident.end_position(), Point::new(0, 5)); + } + /// End-to-end: real Swift source parsed by the shim, then adapted into a /// `yeast::Ast`. Requires the Swift toolchain (like the crate's FFI tests). #[test] From ddab6ccb35691f368d09ef0dfa63a5e1bb47c104 Mon Sep 17 00:00:00 2001 From: Taus Date: Tue, 7 Jul 2026 14:43:59 +0000 Subject: [PATCH 36/90] unified: Emit locations for comments Also gathers these into a separate side-channel during the initial AST construction. That way, we don't encounter these as weird extra nodes while running yeast. --- unified/swift-syntax-rs/README.md | 13 +- unified/swift-syntax-rs/src/yeast_adapter.rs | 147 ++++++++++++++++-- .../SwiftSyntaxFFI/SwiftSyntaxFFI.swift | 47 ++++-- 3 files changed, 179 insertions(+), 28 deletions(-) diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index bb4f7f667a5..08cabeb736b 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -177,15 +177,20 @@ rewrite rules operate on — via [`yeast_adapter::json_to_ast`](src/yeast_adapte ```rust let json = swift_syntax_rs::parse_to_json("let x = 1")?; -let ast = swift_syntax_rs::yeast_adapter::json_to_ast(&json)?; +let adapted = swift_syntax_rs::yeast_adapter::json_to_ast(&json)?; +let ast = adapted.ast; // the yeast::Ast +let comments = adapted.trivia; // side-channel comment/unexpectedText tokens ``` The adapter mirrors tree-sitter's node model, which is what yeast expects: layout nodes and varying tokens (identifiers, literals, operators) become **named** nodes; fixed tokens (keywords, punctuation) become **anonymous** -nodes keyed by their text. It preserves swift-syntax's own kind/field names — -aligning them with the tree-sitter-swift schema so the existing rewrite rules -fire is a separate, later step. +nodes keyed by their text. Comments (and `unexpectedText`) are harvested into a +side channel (`adapted.trivia`) during the same traversal rather than embedded +in the tree, matching how the extractor treats tree-sitter `extra` nodes. It +preserves swift-syntax's own kind/field names — aligning them with the +tree-sitter-swift schema so the existing rewrite rules fire is a separate, +later step. ## Layout diff --git a/unified/swift-syntax-rs/src/yeast_adapter.rs b/unified/swift-syntax-rs/src/yeast_adapter.rs index 8a0baf7a4cd..e2adf1df5a6 100644 --- a/unified/swift-syntax-rs/src/yeast_adapter.rs +++ b/unified/swift-syntax-rs/src/yeast_adapter.rs @@ -25,6 +25,29 @@ use serde_json::Value; use yeast::schema::Schema; use yeast::{Ast, Id, NodeContent, Point, Range}; +/// A comment (or `unexpectedText`) recovered from the syntax tree's trivia. +/// +/// These are collected into a side channel rather than embedded in the +/// [`yeast::Ast`], mirroring how the extractor treats tree-sitter `extra` +/// nodes: they carry a location and text but are not attached to a parent. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TriviaToken { + /// The trivia kind (e.g. `lineComment`, `blockComment`, `docLineComment`, + /// `docBlockComment`, `unexpectedText`). + pub kind: String, + /// The verbatim source text of the piece (e.g. `// comment`). + pub text: String, + /// The source range the piece occupies. + pub range: Range, +} + +/// The result of adapting a swift-syntax JSON tree: the [`yeast::Ast`] plus the +/// comment/`unexpectedText` trivia harvested from it (in source order). +pub struct AdaptedTree { + pub ast: Ast, + pub trivia: Vec, +} + /// swift-syntax `TokenKind` cases whose text is *not* determined by the kind /// (i.e. `TokenKind.defaultText == nil`). These carry varying information and /// are modelled as named leaf nodes; every other token is a fixed @@ -142,16 +165,19 @@ fn children_of(value: &Value) -> Vec<&Value> { /// /// This is a single traversal: each node's kind and field names are registered /// in the schema on the fly, immediately before the node is created. Children -/// are built first so a parent's field lists reference existing ids. -fn build(node: &Value, ast: &mut Ast) -> Result { +/// are built first so a parent's field lists reference existing ids. Any +/// comment/`unexpectedText` trivia carried by a token is harvested into +/// `trivia` during the same pass rather than embedded in the tree. +fn build(node: &Value, ast: &mut Ast, trivia: &mut Vec) -> Result { let info = classify(node)?; + collect_trivia(node, trivia); let mut fields: BTreeMap> = BTreeMap::new(); for (field, value) in field_entries(node) { let field_id = ast.register_field(field); let mut ids = Vec::new(); for child in children_of(value) { - ids.push(build(child, ast)?); + ids.push(build(child, ast, trivia)?); } fields.insert(field_id, ids); } @@ -171,6 +197,35 @@ fn build(node: &Value, ast: &mut Ast) -> Result { )) } +/// Harvest a token's `leadingTrivia`/`trailingTrivia` pieces (each already +/// filtered to comments/`unexpectedText` upstream) into `out`. Non-token nodes +/// have no trivia keys, so this is a no-op for them. +fn collect_trivia(node: &Value, out: &mut Vec) { + for key in ["leadingTrivia", "trailingTrivia"] { + let Some(Value::Array(pieces)) = node.get(key) else { + continue; + }; + for piece in pieces { + let (Some(kind), Some(range)) = ( + piece.get("kind").and_then(Value::as_str), + parse_range(piece), + ) else { + continue; + }; + let text = piece + .get("text") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + out.push(TriviaToken { + kind: kind.to_string(), + text, + range, + }); + } + } +} + /// Parse a node's `range` into a [`yeast::Range`]. /// /// The JSON carries, for `start` and `end`, a 0-based UTF-8 file byte `offset`, @@ -201,14 +256,20 @@ fn parse_range(node: &Value) -> Option { } /// Convert a swift-syntax JSON tree (as produced by [`crate::parse_to_json`]) -/// into a [`yeast::Ast`]. -pub fn json_to_ast(json: &str) -> Result { +/// into a [`yeast::Ast`] plus the comment/`unexpectedText` trivia harvested +/// from it. Both are produced in a single traversal. +pub fn json_to_ast(json: &str) -> Result { let root: Value = serde_json::from_str(json).map_err(|e| format!("invalid JSON: {e}"))?; let mut ast = Ast::with_schema(Schema::new()); - let root_id = build(&root, &mut ast)?; + let mut trivia = Vec::new(); + let root_id = build(&root, &mut ast, &mut trivia)?; ast.set_root(root_id); - Ok(ast) + + // Emit trivia in source order (the traversal visits nodes bottom-up). + trivia.sort_by_key(|t| t.range.start_byte); + + Ok(AdaptedTree { ast, trivia }) } #[cfg(test)] @@ -245,7 +306,9 @@ mod tests { #[test] fn builds_ast_from_json() { - let ast = json_to_ast(sample_json()).expect("adapter should succeed"); + let ast = json_to_ast(sample_json()) + .expect("adapter should succeed") + .ast; let root = ast.get_root(); let root_node = ast.get_node(root).expect("root exists"); assert_eq!(root_node.kind_name(), "sourceFile"); @@ -254,7 +317,9 @@ mod tests { #[test] fn classifies_named_and_anonymous_tokens() { - let ast = json_to_ast(sample_json()).expect("adapter should succeed"); + let ast = json_to_ast(sample_json()) + .expect("adapter should succeed") + .ast; // Walk all nodes and collect (kind_name, is_named) for the two tokens. let mut let_named = None; let mut ident_named = None; @@ -273,7 +338,9 @@ mod tests { #[test] fn preserves_leaf_text() { - let ast = json_to_ast(sample_json()).expect("adapter should succeed"); + let ast = json_to_ast(sample_json()) + .expect("adapter should succeed") + .ast; let ident = ast .nodes() .iter() @@ -286,7 +353,9 @@ mod tests { #[test] fn maps_source_locations() { - let ast = json_to_ast(sample_json()).expect("adapter should succeed"); + let ast = json_to_ast(sample_json()) + .expect("adapter should succeed") + .ast; let ident = ast .nodes() .iter() @@ -300,13 +369,55 @@ mod tests { assert_eq!(ident.end_position(), Point::new(0, 5)); } + #[test] + fn collects_trivia_into_side_channel() { + // A token carrying a trailing line comment in its trivia. + let json = r#"{ + "kind": "sourceFile", + "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":14,"line":1,"column":15}}, + "value": { + "kind": "token", + "tokenKind": "integerLiteral(\"1\")", + "text": "1", + "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":1,"line":1,"column":2}}, + "trailingTrivia": [ + { + "kind": "lineComment", + "text": "// c", + "range": {"start":{"offset":2,"line":1,"column":3},"end":{"offset":6,"line":1,"column":7}} + } + ] + } + }"#; + let adapted = json_to_ast(json).expect("adapter should succeed"); + + // The comment is in the side channel, with its text and location. + assert_eq!(adapted.trivia.len(), 1); + let comment = &adapted.trivia[0]; + assert_eq!(comment.kind, "lineComment"); + assert_eq!(comment.text, "// c"); + assert_eq!(comment.range.start_byte, 2); + assert_eq!(comment.range.end_byte, 6); + + // It is not embedded in the AST as a node. + assert!( + adapted + .ast + .nodes() + .iter() + .all(|n| n.kind_name() != "lineComment"), + "comment should not appear as an AST node" + ); + } + /// End-to-end: real Swift source parsed by the shim, then adapted into a /// `yeast::Ast`. Requires the Swift toolchain (like the crate's FFI tests). #[test] fn end_to_end_from_swift_source() { - let json = crate::parse_to_json("func f(n: Int) -> Int { return n }") + let json = crate::parse_to_json("func f(n: Int) -> Int { return n } // trailing") .expect("parsing should succeed"); - let ast = json_to_ast(&json).expect("adapter should succeed"); + let adapted = json_to_ast(&json).expect("adapter should succeed"); + let ast = &adapted.ast; let root = ast.get_node(ast.get_root()).expect("root exists"); assert_eq!(root.kind_name(), "sourceFile"); @@ -323,5 +434,15 @@ mod tests { kinds.contains(&"func"), "expected an anonymous `func` token, got kinds: {kinds:?}" ); + + // The trailing comment is recovered into the side channel. + assert!( + adapted + .trivia + .iter() + .any(|t| t.kind == "lineComment" && t.text == "// trailing"), + "expected the trailing comment in the trivia side channel, got: {:?}", + adapted.trivia + ); } } diff --git a/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift b/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift index 3dcfcb2c389..6305b1610d0 100644 --- a/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift +++ b/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift @@ -39,20 +39,39 @@ private let keptTriviaKinds: Set = [ "unexpectedText", ] -/// Serialize a trivia collection into an array of `{ kind, text }` pieces, -/// keeping only the kinds in `keptTriviaKinds`. -private func serializeTrivia(_ trivia: Trivia) -> [Any] { - trivia.pieces.compactMap { piece -> [String: Any]? in +/// Serialize a trivia collection into an array of `{ kind, text, range }` +/// pieces, keeping only the kinds in `keptTriviaKinds`. +/// +/// `start` is the absolute position of the first piece (a token's leading +/// trivia starts at `token.position`; its trailing trivia at +/// `token.endPositionBeforeTrailingTrivia`). Each piece's range is derived by +/// accumulating piece lengths, so kept pieces carry an exact source location. +private func serializeTrivia( + _ trivia: Trivia, + startingAt start: AbsolutePosition, + _ converter: SourceLocationConverter +) -> [Any] { + var result: [Any] = [] + var offset = start.utf8Offset + for piece in trivia.pieces { + let length = piece.sourceLength.utf8Length // The label of an enum case mirror is the case name (e.g. "spaces", // "lineComment"), which gives us a stable kind without an exhaustive // switch over every `TriviaPiece` case. let kind = Mirror(reflecting: piece).children.first?.label ?? "\(piece)" - guard keptTriviaKinds.contains(kind) else { return nil } - return [ - "kind": kind, - "text": Trivia(pieces: [piece]).description, - ] + if keptTriviaKinds.contains(kind) { + result.append([ + "kind": kind, + "text": Trivia(pieces: [piece]).description, + "range": [ + "start": location(AbsolutePosition(utf8Offset: offset), converter), + "end": location(AbsolutePosition(utf8Offset: offset + length), converter), + ], + ]) + } + offset += length } + return result } /// Recursively convert a SwiftSyntax node into a JSON-serializable value. @@ -95,11 +114,17 @@ private func serialize( "range": range, ] // Only emit trivia when present; after filtering, most tokens have none. - let leading = serializeTrivia(token.leadingTrivia) + // Leading trivia starts at the token's own position; trailing trivia + // starts just after the token's content. + let leading = serializeTrivia( + token.leadingTrivia, startingAt: token.position, converter) if !leading.isEmpty { result["leadingTrivia"] = leading } - let trailing = serializeTrivia(token.trailingTrivia) + let trailing = serializeTrivia( + token.trailingTrivia, + startingAt: token.endPositionBeforeTrailingTrivia, + converter) if !trailing.isEmpty { result["trailingTrivia"] = trailing } From 58ddeacde08e66e31f5dd8a05b39bcceecfedd4f Mon Sep 17 00:00:00 2001 From: Taus Date: Tue, 7 Jul 2026 15:56:56 +0000 Subject: [PATCH 37/90] unified: Hook up swift-syntax AST At this point it's only a proof-of-concept translation -- it translates `sourceFile` nodes, but everything else gets mapped to 'unsupported_node`. --- Cargo.lock | 4 - shared/yeast-schema/src/schema.rs | 22 ++ shared/yeast/src/lib.rs | 39 +++- shared/yeast/tests/test.rs | 53 +++++ unified/extractor/src/languages/mod.rs | 9 + .../src/languages/swift/adapter.rs} | 51 +---- .../extractor/src/languages/swift/swift.rs | 19 ++ .../tests/fixtures/let_x.swiftsyntax.json | 196 ++++++++++++++++++ .../extractor/tests/swift_syntax_pipeline.rs | 49 +++++ unified/swift-syntax-rs/BUILD.bazel | 2 - unified/swift-syntax-rs/Cargo.toml | 4 - unified/swift-syntax-rs/README.md | 27 +-- unified/swift-syntax-rs/src/lib.rs | 9 +- 13 files changed, 404 insertions(+), 80 deletions(-) rename unified/{swift-syntax-rs/src/yeast_adapter.rs => extractor/src/languages/swift/adapter.rs} (88%) create mode 100644 unified/extractor/tests/fixtures/let_x.swiftsyntax.json create mode 100644 unified/extractor/tests/swift_syntax_pipeline.rs diff --git a/Cargo.lock b/Cargo.lock index 24885fea856..7a9f1966791 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2853,10 +2853,6 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "swift-syntax-rs" version = "0.1.0" -dependencies = [ - "serde_json", - "yeast", -] [[package]] name = "syn" diff --git a/shared/yeast-schema/src/schema.rs b/shared/yeast-schema/src/schema.rs index 4acd14377a4..9c438d7c22e 100644 --- a/shared/yeast-schema/src/schema.rs +++ b/shared/yeast-schema/src/schema.rs @@ -167,6 +167,28 @@ impl Schema { id } + /// Register every kind (named and unnamed) and field *name* from `other` + /// into this schema (idempotent). Ids are assigned in this schema's own id + /// space; existing ids are unchanged. + /// + /// This is used when running desugaring rules over an AST that was built + /// against a different schema (e.g. from an external parser): the rules + /// build output nodes whose kind/field names come from `other`, and those + /// names must resolve in the AST's own schema. Only names are needed — the + /// rule engine resolves kinds/fields by name and does not consult + /// `other`'s field-type or supertype information. + pub fn register_names_from(&mut self, other: &Schema) { + for name in other.kind_ids.keys() { + self.register_kind(name); + } + for name in other.unnamed_kind_ids.keys() { + self.register_unnamed_kind(name); + } + for name in other.field_ids.keys() { + self.register_field(name); + } + } + /// Track a name for a kind ID without registering it as named or /// unnamed. Useful when importing tree-sitter ID tables that may /// contain duplicate IDs across the named/unnamed split. diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index b2709ab406e..89facfaea99 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -524,10 +524,15 @@ impl Ast { self.schema.register_field(name) } - fn union_source_range_of_children( - &self, - fields: &BTreeMap>, - ) -> Option { + /// Register every kind and field name from `schema` into this AST's schema + /// (idempotent). Used before desugaring an externally-built AST so that + /// rules can build output nodes whose kind/field names come from the + /// desugarer's output schema. + pub fn register_names_from_schema(&mut self, schema: &schema::Schema) { + self.schema.register_names_from(schema); + } + + fn union_source_range_of_children(&self, fields: &BTreeMap>) -> Option { let mut start_byte: Option = None; let mut end_byte: Option = None; let mut start_point = Point { row: 0, column: 0 }; @@ -1448,6 +1453,18 @@ impl<'a, C: Clone + Default> Runner<'a, C> { let mut user_ctx = C::default(); self.run_with_ctx(input, &mut user_ctx) } + + /// Run all phases over an already-built `ast`, using the default context + /// (`C::default()`). Unlike [`run_from_tree`](Self::run_from_tree), the AST + /// is supplied by the caller (e.g. built from an external parser's output) + /// rather than constructed from a tree-sitter tree. The caller is + /// responsible for ensuring the AST's schema knows any output kind/field + /// names the rules will build (see [`Ast::register_names_from_schema`]). + pub fn run_from_ast(&self, mut ast: Ast) -> Result { + let mut user_ctx = C::default(); + self.run_phases(&mut ast, &mut user_ctx)?; + Ok(ast) + } } // --------------------------------------------------------------------------- @@ -1470,6 +1487,12 @@ pub trait Desugarer: Send + Sync { /// Parse `tree` against `source` and run the desugaring pipeline. /// Each call constructs a fresh default user context internally. fn run_from_tree(&self, tree: &tree_sitter::Tree, source: &[u8]) -> Result; + + /// Run the desugaring pipeline over an already-built `ast` (e.g. produced + /// by an external parser rather than tree-sitter). The desugarer ensures + /// the AST's schema knows its output kind/field names before running the + /// rules. Each call constructs a fresh default user context internally. + fn run_from_ast(&self, ast: Ast) -> Result; } /// A concrete [`Desugarer`] backed by a [`DesugaringConfig`] for a @@ -1507,4 +1530,12 @@ impl Desugarer for ConcreteDesugarer let runner = Runner::with_schema(self.language.clone(), &self.schema, &self.config.phases); runner.run_from_tree(tree, source) } + + fn run_from_ast(&self, mut ast: Ast) -> Result { + // The AST was built against its own (external) schema; make sure the + // output kind/field names the rules build are resolvable in it. + ast.register_names_from_schema(&self.schema); + let runner = Runner::with_schema(self.language.clone(), &self.schema, &self.config.phases); + runner.run_from_ast(ast) + } } diff --git a/shared/yeast/tests/test.rs b/shared/yeast/tests/test.rs index 3a24709dd9f..756e219bd89 100644 --- a/shared/yeast/tests/test.rs +++ b/shared/yeast/tests/test.rs @@ -266,6 +266,59 @@ fn test_query_match() { assert!(captures.get_var("right").is_ok()); } +#[test] +fn test_run_from_ast_desugars_hand_built_tree() { + use std::collections::BTreeMap; + + // Output schema for the desugared tree. Its kind/field names must become + // resolvable in the hand-built AST's schema for the rule to build them. + let schema_yaml = r#" +named: + assignment: + left: leaf + leaf: +"#; + + // A rule over an *input* kind (`wrapper`) that is not in the output schema, + // rewriting to an output `assignment` node. + let rules: Vec = vec![yeast::rule!( + (wrapper) + => + (assignment left: (leaf "lit")) + )]; + + let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); + let config = DesugaringConfig::<()>::new() + .add_phase("test", PhaseKind::OneShot, rules) + .with_output_node_types_yaml(schema_yaml); + let desugarer = ConcreteDesugarer::new(lang, config).unwrap(); + + // Build the input AST by hand, as an external parser adapter would. The + // schema starts empty and gains the `wrapper` input kind on the fly. + let mut ast = Ast::with_schema(yeast::schema::Schema::new()); + let wrapper_kind = ast.register_kind("wrapper"); + let root = ast.create_node_with_range( + wrapper_kind, + NodeContent::DynamicString(String::new()), + BTreeMap::new(), + true, + None, + ); + ast.set_root(root); + + // Desugaring the hand-built AST applies the rule, producing `assignment` + // even though the AST was built against a schema with no output kinds. + let out = desugarer + .run_from_ast(ast) + .expect("run_from_ast should succeed"); + let out_root = out.get_node(out.get_root()).expect("root exists"); + assert_eq!(out_root.kind_name(), "assignment"); + let dump = dump_ast(&out, out.get_root(), ""); + assert!(dump.contains("assignment"), "unexpected dump: {dump}"); + assert!(dump.contains("left"), "unexpected dump: {dump}"); + assert!(dump.contains("leaf"), "unexpected dump: {dump}"); +} + #[test] fn test_query_no_match() { let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); diff --git a/unified/extractor/src/languages/mod.rs b/unified/extractor/src/languages/mod.rs index 20ad599edfb..52a1bd40ffc 100644 --- a/unified/extractor/src/languages/mod.rs +++ b/unified/extractor/src/languages/mod.rs @@ -3,6 +3,15 @@ use codeql_extractor::extractor::simple; #[path = "swift/swift.rs"] mod swift; +/// swift-syntax JSON -> `yeast::Ast` adapter for the Swift front-end. +/// +/// Currently exercised by tests and the forthcoming runtime extraction path; +/// `allow(dead_code)` because this is a binary crate, so its public API isn't +/// counted as used until the binary itself calls it. +#[path = "swift/adapter.rs"] +#[allow(dead_code)] +pub mod swift_adapter; + /// Shared YEAST output AST schema for all languages. pub(crate) const OUTPUT_AST_SCHEMA: &str = include_str!("../../ast_types.yml"); diff --git a/unified/swift-syntax-rs/src/yeast_adapter.rs b/unified/extractor/src/languages/swift/adapter.rs similarity index 88% rename from unified/swift-syntax-rs/src/yeast_adapter.rs rename to unified/extractor/src/languages/swift/adapter.rs index e2adf1df5a6..37d5aac00d2 100644 --- a/unified/swift-syntax-rs/src/yeast_adapter.rs +++ b/unified/extractor/src/languages/swift/adapter.rs @@ -1,6 +1,10 @@ -//! Adapter that converts the swift-syntax JSON tree (see [`crate::parse_to_json`]) -//! into a [`yeast::Ast`], the in-memory format the CodeQL desugaring rules -//! operate on. +//! Converts the swift-syntax JSON syntax tree into a [`yeast::Ast`], the +//! in-memory format the CodeQL desugaring rules operate on. +//! +//! The JSON tree is produced by the `swift-syntax-rs` crate's Swift FFI shim +//! (`parse_to_json`). This module is pure Rust (only `yeast` + `serde_json`), +//! so the extractor consumes swift-syntax output without pulling in the Swift +//! toolchain (the JSON is produced out-of-process). //! //! The mapping mirrors tree-sitter's node model, which is what yeast (and the //! extractor's rewrite rules) expect: @@ -15,9 +19,8 @@ //! list-valued field maps directly to that field holding several children. //! //! Note: this preserves swift-syntax's own kind/field names. Aligning those -//! names with the tree-sitter-swift schema (so the existing rewrite rules fire) -//! is a separate, later step; this module is only concerned with getting the -//! tree into yeast's format. +//! names with the tree-sitter-swift schema (so the rewrite rules in +//! [`super::swift`] fire) is done incrementally in the rules. use std::collections::BTreeMap; @@ -409,40 +412,4 @@ mod tests { "comment should not appear as an AST node" ); } - - /// End-to-end: real Swift source parsed by the shim, then adapted into a - /// `yeast::Ast`. Requires the Swift toolchain (like the crate's FFI tests). - #[test] - fn end_to_end_from_swift_source() { - let json = crate::parse_to_json("func f(n: Int) -> Int { return n } // trailing") - .expect("parsing should succeed"); - let adapted = json_to_ast(&json).expect("adapter should succeed"); - let ast = &adapted.ast; - - let root = ast.get_node(ast.get_root()).expect("root exists"); - assert_eq!(root.kind_name(), "sourceFile"); - - // The tree contains a `functionDecl` layout node and an anonymous - // `func` keyword token keyed by its text. - let mut kinds: Vec<&str> = ast.nodes().iter().map(|n| n.kind_name()).collect(); - kinds.sort_unstable(); - assert!( - kinds.contains(&"functionDecl"), - "expected a functionDecl node, got kinds: {kinds:?}" - ); - assert!( - kinds.contains(&"func"), - "expected an anonymous `func` token, got kinds: {kinds:?}" - ); - - // The trailing comment is recovered into the side channel. - assert!( - adapted - .trivia - .iter() - .any(|t| t.kind == "lineComment" && t.text == "// trailing"), - "expected the trailing comment in the trivia side channel, got: {:?}", - adapted.trivia - ); - } } diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index 74809ea18e4..13f1a6beadf 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -135,6 +135,25 @@ fn translation_rules() -> Vec> { // Declarations may be wrapped in local/global wrapper nodes. rule!((global_declaration _ @inner) => stmt { inner }), rule!((local_declaration _ @inner) => stmt { inner }), + // ---- swift-syntax front-end (minimal hook-up) ---- + // These rules target the swift-syntax AST (camelCase kind names), + // produced by the sibling `adapter` module. They coexist with the + // tree-sitter rules (snake_case names): rules are dispatched by exact + // kind name, and the two name spaces never collide, so these are inert + // on the tree-sitter path. Only the minimal top-level mapping lives here + // to demonstrate the pipeline end-to-end; the full translation is added + // separately. Unmatched swift-syntax nodes fall through to the + // `unsupported_node` fallback at the end. + // + // `sourceFile` holds its top-level statements in an (elided) + // `statements` collection; each element is a `codeBlockItem` wrapping + // the real node. + rule!( + (sourceFile statements: _* @items) + => + (top_level body: (block stmt: {items})) + ), + rule!((codeBlockItem item: @item) => stmt { item }), // ---- Literals ---- rule!((integer_literal) => (int_literal)), rule!((hex_literal) => (int_literal)), diff --git a/unified/extractor/tests/fixtures/let_x.swiftsyntax.json b/unified/extractor/tests/fixtures/let_x.swiftsyntax.json new file mode 100644 index 00000000000..6c3d18e2fef --- /dev/null +++ b/unified/extractor/tests/fixtures/let_x.swiftsyntax.json @@ -0,0 +1,196 @@ +{ + "endOfFileToken": { + "kind": "token", + "range": { + "end": { + "column": 1, + "line": 2, + "offset": 10 + }, + "start": { + "column": 1, + "line": 2, + "offset": 10 + } + }, + "text": "", + "tokenKind": "endOfFile" + }, + "kind": "sourceFile", + "range": { + "end": { + "column": 1, + "line": 2, + "offset": 10 + }, + "start": { + "column": 1, + "line": 1, + "offset": 0 + } + }, + "statements": [ + { + "item": { + "attributes": [], + "bindings": [ + { + "initializer": { + "equal": { + "kind": "token", + "range": { + "end": { + "column": 8, + "line": 1, + "offset": 7 + }, + "start": { + "column": 7, + "line": 1, + "offset": 6 + } + }, + "text": "=", + "tokenKind": "equal" + }, + "kind": "initializerClause", + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 7, + "line": 1, + "offset": 6 + } + }, + "value": { + "kind": "integerLiteralExpr", + "literal": { + "kind": "token", + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 9, + "line": 1, + "offset": 8 + } + }, + "text": "1", + "tokenKind": "integerLiteral(\"1\")" + }, + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 9, + "line": 1, + "offset": 8 + } + } + } + }, + "kind": "patternBinding", + "pattern": { + "identifier": { + "kind": "token", + "range": { + "end": { + "column": 6, + "line": 1, + "offset": 5 + }, + "start": { + "column": 5, + "line": 1, + "offset": 4 + } + }, + "text": "x", + "tokenKind": "identifier(\"x\")" + }, + "kind": "identifierPattern", + "range": { + "end": { + "column": 6, + "line": 1, + "offset": 5 + }, + "start": { + "column": 5, + "line": 1, + "offset": 4 + } + } + }, + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 5, + "line": 1, + "offset": 4 + } + } + } + ], + "bindingSpecifier": { + "kind": "token", + "range": { + "end": { + "column": 4, + "line": 1, + "offset": 3 + }, + "start": { + "column": 1, + "line": 1, + "offset": 0 + } + }, + "text": "let", + "tokenKind": "keyword(SwiftSyntax.Keyword.let)" + }, + "kind": "variableDecl", + "modifiers": [], + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 1, + "line": 1, + "offset": 0 + } + } + }, + "kind": "codeBlockItem", + "range": { + "end": { + "column": 10, + "line": 1, + "offset": 9 + }, + "start": { + "column": 1, + "line": 1, + "offset": 0 + } + } + } + ] +} diff --git a/unified/extractor/tests/swift_syntax_pipeline.rs b/unified/extractor/tests/swift_syntax_pipeline.rs new file mode 100644 index 00000000000..cdaae1f47c6 --- /dev/null +++ b/unified/extractor/tests/swift_syntax_pipeline.rs @@ -0,0 +1,49 @@ +//! Integration test for the swift-syntax front-end pipeline: +//! +//! swift-syntax JSON -> `swift_adapter::json_to_ast` -> yeast `Ast` +//! -> `Desugarer::run_from_ast` (the real Swift translation rules) -> dump. +//! +//! This exercises the whole chain *without* the Swift toolchain: the JSON +//! fixture is a real `parse_to_json` dump (see the file header) fed through the +//! pure-Rust adapter module. It verifies the desugarer runs end-to-end over an +//! externally-built AST. + +use yeast::dump::dump_ast; + +#[path = "../src/languages/mod.rs"] +mod languages; + +/// A real `swift-syntax-rs` JSON dump of the Swift source `let x = 1`. +const LET_X_JSON: &str = include_str!("fixtures/let_x.swiftsyntax.json"); + +#[test] +fn swift_syntax_json_runs_through_the_desugarer() { + let lang = languages::all_language_specs() + .into_iter() + .find(|l| l.file_globs.iter().any(|g| g.contains("swift"))) + .expect("swift language spec"); + let desugarer = lang.desugar.as_deref().expect("swift desugarer"); + + // Adapt the swift-syntax JSON into a yeast AST (pure Rust, no Swift FFI). + let adapted = + languages::swift_adapter::json_to_ast(LET_X_JSON).expect("adapter should succeed"); + assert_eq!( + adapted + .ast + .get_node(adapted.ast.get_root()) + .unwrap() + .kind_name(), + "sourceFile" + ); + + // Run the real Swift desugaring rules over the externally-built AST. The + // top-level swift-syntax rules map `sourceFile` to `top_level`/`block`; + // kinds without swift-syntax rules yet fall back to `unsupported_node`. + let desugared = desugarer + .run_from_ast(adapted.ast) + .expect("desugaring an externally-built AST should not error"); + + let dump = dump_ast(&desugared, desugared.get_root(), ""); + assert!(dump.contains("top_level"), "unexpected dump: {dump}"); + assert!(dump.contains("block"), "unexpected dump: {dump}"); +} diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index 3a2e5ed0796..9db40d4db30 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -35,8 +35,6 @@ rust_library( edition = "2024", deps = [ ":swift_syntax_ffi", - "//shared/yeast", - "@vendor_ts__serde_json-1.0.145//:serde_json", ], ) diff --git a/unified/swift-syntax-rs/Cargo.toml b/unified/swift-syntax-rs/Cargo.toml index 57624646848..6957fb0e50e 100644 --- a/unified/swift-syntax-rs/Cargo.toml +++ b/unified/swift-syntax-rs/Cargo.toml @@ -12,7 +12,3 @@ path = "src/lib.rs" [[bin]] name = "swift-syntax-parse" path = "src/main.rs" - -[dependencies] -serde_json = "1.0" -yeast = { path = "../../shared/yeast" } diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index 08cabeb736b..a2d66495a54 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -171,26 +171,12 @@ echo 'let x = 1' | cargo run --bin swift-syntax-parse ## Converting to a yeast AST -For use in the CodeQL extractor, the JSON tree can be converted into a -[`yeast::Ast`](../../shared/yeast) — the in-memory format the extractor's -rewrite rules operate on — via [`yeast_adapter::json_to_ast`](src/yeast_adapter.rs): - -```rust -let json = swift_syntax_rs::parse_to_json("let x = 1")?; -let adapted = swift_syntax_rs::yeast_adapter::json_to_ast(&json)?; -let ast = adapted.ast; // the yeast::Ast -let comments = adapted.trivia; // side-channel comment/unexpectedText tokens -``` - -The adapter mirrors tree-sitter's node model, which is what yeast expects: -layout nodes and varying tokens (identifiers, literals, operators) become -**named** nodes; fixed tokens (keywords, punctuation) become **anonymous** -nodes keyed by their text. Comments (and `unexpectedText`) are harvested into a -side channel (`adapted.trivia`) during the same traversal rather than embedded -in the tree, matching how the extractor treats tree-sitter `extra` nodes. It -preserves swift-syntax's own kind/field names — aligning them with the -tree-sitter-swift schema so the existing rewrite rules fire is a separate, -later step. +The JSON tree is consumed by the CodeQL extractor, which converts it into a +[`yeast::Ast`](../../shared/yeast) — the in-memory format its rewrite rules +operate on. That adapter is a pure-Rust module living in the extractor +(`unified/extractor/src/languages/swift/adapter.rs`), so the extractor never +needs the Swift toolchain: it consumes the JSON produced out-of-process by this +crate's `parse_to_json` / the `swift-syntax-parse` binary. ## Layout @@ -198,5 +184,4 @@ later step. - `build.rs` — builds the Swift package and emits link/rpath flags (local `cargo` only). - `BUILD.bazel` — Bazel targets for the hermetic CI build (swift_library + rust targets). - `src/lib.rs` — safe Rust bindings (`parse_to_json`). -- `src/yeast_adapter.rs` — converts the JSON tree into a `yeast::Ast`. - `src/main.rs` — demo CLI. diff --git a/unified/swift-syntax-rs/src/lib.rs b/unified/swift-syntax-rs/src/lib.rs index aa711e322a4..2544c0bc408 100644 --- a/unified/swift-syntax-rs/src/lib.rs +++ b/unified/swift-syntax-rs/src/lib.rs @@ -3,9 +3,12 @@ //! //! The heavy lifting is done by a small Swift shim (see `swift/`) that links //! against `SwiftSyntax`/`SwiftParser` and exposes a tiny C ABI. This module -//! provides safe Rust bindings on top of that ABI. - -pub mod yeast_adapter; +//! provides safe Rust bindings on top of that ABI, exposing [`parse_to_json`] +//! which turns Swift source into a JSON syntax tree. +//! +//! Converting that JSON into a `yeast::Ast` (for the CodeQL extractor) is done +//! by the extractor's own pure-Rust adapter module, keeping the Swift toolchain +//! out of the extractor's build. use std::ffi::{CStr, CString}; use std::os::raw::c_char; From 33261f441e1ed81de8841cde84c072b32adcd132 Mon Sep 17 00:00:00 2001 From: Anders Schack-Mulligen Date: Fri, 10 Jul 2026 15:32:44 +0200 Subject: [PATCH 38/90] Java: Improve join order. Before: ``` [2026-07-10 15:15:58] Evaluated non-recursive predicate ControlFlowGraph::NonReturningCalls::EffectivelyNonVirtualMethod.getAnAccess/0#dispred#efac59ed@090459qt in 4303ms (size: 186002). Evaluated relational algebra for predicate ControlFlowGraph::NonReturningCalls::EffectivelyNonVirtualMethod.getAnAccess/0#dispred#efac59ed@090459qt with tuple counts: 790592 ~0% {2} r1 = JOIN `Expr::MethodCall.getMethod/0#dispred#41989dc9_10#join_rhs` WITH `Member::Method.getSourceDeclaration/0#dispred#93e6cdf8` ON FIRST 1 OUTPUT Rhs.1, Lhs.1 539755681 ~2% {2} | JOIN WITH `Member::SrcMethod.getAPossibleImplementationOfSrcMethod/0#dispred#4f4317e6#bf` ON FIRST 1 OUTPUT Rhs.1, Lhs.1 186002 ~2% {2} | JOIN WITH ControlFlowGraph::NonReturningCalls::EffectivelyNonVirtualMethod#86c19e07 ON FIRST 1 OUTPUT Lhs.0, Lhs.1 return r1 ``` After: ``` [2026-07-10 15:29:39] Evaluated non-recursive predicate ControlFlowGraph::NonReturningCalls::EffectivelyNonVirtualMethod.getAnAccess/0#dispred#efac59ed@32fb2e61 in 10ms (size: 186002). Evaluated relational algebra for predicate ControlFlowGraph::NonReturningCalls::EffectivelyNonVirtualMethod.getAnAccess/0#dispred#efac59ed@32fb2e61 with tuple counts: 122765 ~0% {2} r1 = SCAN ControlFlowGraph::NonReturningCalls::EffectivelyNonVirtualMethod#86c19e07 OUTPUT In.0, In.0 122766 ~0% {2} | JOIN WITH `Member::SrcMethod.getAPossibleImplementationOfSrcMethod/0#dispred#4f4317e6_10#join_rhs` ON FIRST 1 OUTPUT Rhs.1, Lhs.1 126911 ~0% {2} | JOIN WITH `Member::Method.getSourceDeclaration/0#dispred#93e6cdf8_10#join_rhs` ON FIRST 1 OUTPUT Rhs.1, Lhs.1 186002 ~2% {2} | JOIN WITH `Expr::MethodCall.getMethod/0#dispred#41989dc9_10#join_rhs` ON FIRST 1 OUTPUT Lhs.1, Rhs.1 return r1 ``` --- java/ql/lib/semmle/code/java/ControlFlowGraph.qll | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/java/ql/lib/semmle/code/java/ControlFlowGraph.qll b/java/ql/lib/semmle/code/java/ControlFlowGraph.qll index 7390be9cb3c..c4f0c44b381 100644 --- a/java/ql/lib/semmle/code/java/ControlFlowGraph.qll +++ b/java/ql/lib/semmle/code/java/ControlFlowGraph.qll @@ -406,7 +406,9 @@ private module NonReturningCalls { } /** Gets a `MethodCall` that calls this method. */ - MethodCall getAnAccess() { result.getMethod().getAPossibleImplementation() = this } + MethodCall getAnAccess() { + result.getMethod().getAPossibleImplementation() = pragma[only_bind_out](this) + } } /** Holds if a call to `m` indicates that `m` is expected to return. */ From 2cd9220b741975a11f3db63639f456ab7933b14d Mon Sep 17 00:00:00 2001 From: Kristen Newbury Date: Fri, 10 Jul 2026 09:36:53 -0400 Subject: [PATCH 39/90] Format --- actions/ql/lib/codeql/actions/security/ControlChecks.qll | 3 ++- .../ql/src/Security/CWE-367/UntrustedCheckoutTOCTOUCritical.ql | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/actions/ql/lib/codeql/actions/security/ControlChecks.qll b/actions/ql/lib/codeql/actions/security/ControlChecks.qll index 900f62556b0..ed8d6c2d050 100644 --- a/actions/ql/lib/codeql/actions/security/ControlChecks.qll +++ b/actions/ql/lib/codeql/actions/security/ControlChecks.qll @@ -144,7 +144,8 @@ class EnvironmentCheck extends ControlCheck instanceof Environment { // Environment checks are not effective against any mutable attacks // they do actually protect against untrusted code execution (sha) override predicate protectsCategoryAndEvent(string category, string event) { - event = [actor_is_attacker_event(), actor_not_attacker_event()] and category = non_toctou_category() + event = [actor_is_attacker_event(), actor_not_attacker_event()] and + category = non_toctou_category() } } diff --git a/actions/ql/src/Security/CWE-367/UntrustedCheckoutTOCTOUCritical.ql b/actions/ql/src/Security/CWE-367/UntrustedCheckoutTOCTOUCritical.ql index 14a6c9a4c56..2aacf20b35f 100644 --- a/actions/ql/src/Security/CWE-367/UntrustedCheckoutTOCTOUCritical.ql +++ b/actions/ql/src/Security/CWE-367/UntrustedCheckoutTOCTOUCritical.ql @@ -23,8 +23,7 @@ where // the checked-out code may lead to arbitrary code execution checkout.getAFollowingStep() = step and // the checkout occurs in a privileged context - inPrivilegedContext(checkout, event) - and + inPrivilegedContext(checkout, event) and // the mutable checkout step is protected by an Insufficient access check exists(ControlCheck check1 | check1.protects(checkout, event, "untrusted-checkout")) and not exists(ControlCheck check2 | check2.protects(checkout, event, "untrusted-checkout-toctou")) From 6057b4b6e9c187732b2f985366d2dc081e8dadbb Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Sat, 11 Jul 2026 07:28:14 +0100 Subject: [PATCH 40/90] Fix path sanitizer --- .../2026-07-11-file-getname-path-sanitizer.md | 4 +++ java/ql/lib/ext/java.io.model.yml | 5 ---- .../code/java/security/PathSanitizer.qll | 20 ++++++++++--- .../CWE-022/semmle/tests/TaintedPath.expected | 26 +++++++++++++++-- .../CWE-022/semmle/tests/TaintedPath.java | 28 ++++++++++++++++--- 5 files changed, 68 insertions(+), 15 deletions(-) create mode 100644 java/ql/lib/change-notes/2026-07-11-file-getname-path-sanitizer.md diff --git a/java/ql/lib/change-notes/2026-07-11-file-getname-path-sanitizer.md b/java/ql/lib/change-notes/2026-07-11-file-getname-path-sanitizer.md new file mode 100644 index 00000000000..dddcae521ff --- /dev/null +++ b/java/ql/lib/change-notes/2026-07-11-file-getname-path-sanitizer.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* `java.io.File.getName()` is no longer treated as a complete sanitizer for `java/path-injection`, since it does not remove a `..` path component (for example `new File("..").getName()` returns `".."`). It is now only recognized as a sanitizer when combined with a subsequent check for `..` components, which may result in new alerts. diff --git a/java/ql/lib/ext/java.io.model.yml b/java/ql/lib/ext/java.io.model.yml index dd47342d590..f7c236de4ee 100644 --- a/java/ql/lib/ext/java.io.model.yml +++ b/java/ql/lib/ext/java.io.model.yml @@ -162,8 +162,3 @@ extensions: extensible: sourceModel data: - ["java.io", "FileInputStream", True, "FileInputStream", "", "", "Argument[this]", "file", "manual"] - - addsTo: - pack: codeql/java-all - extensible: barrierModel - data: - - ["java.io", "File", True, "getName", "()", "", "ReturnValue", "path-injection", "manual"] diff --git a/java/ql/lib/semmle/code/java/security/PathSanitizer.qll b/java/ql/lib/semmle/code/java/security/PathSanitizer.qll index 42304bb72dc..7c428a6691c 100644 --- a/java/ql/lib/semmle/code/java/security/PathSanitizer.qll +++ b/java/ql/lib/semmle/code/java/security/PathSanitizer.qll @@ -97,16 +97,28 @@ private class AllowedPrefixSanitizer extends PathInjectionSanitizer { } } +/** A call to `java.io.File.getName`, which returns the final component of a path. */ +private class FileGetNameCall extends MethodCall { + FileGetNameCall() { this.getMethod().hasQualifiedName("java.io", "File", "getName") } +} + /** * Holds if `g` is a guard that considers a path safe because it is checked for `..` components, having previously - * been checked for a trusted prefix. + * been checked for a trusted prefix or been reduced to its final path component by `File.getName`. */ private predicate dotDotCheckGuard(Guard g, Expr e, boolean branch) { pathTraversalGuard(g, e, branch) and - exists(Guard previousGuard | - previousGuard.(AllowedPrefixGuard).controls(g.getBasicBlock(), true) + ( + exists(Guard previousGuard | + previousGuard.(AllowedPrefixGuard).controls(g.getBasicBlock(), true) + or + previousGuard.(BlockListGuard).controls(g.getBasicBlock(), false) + ) or - previousGuard.(BlockListGuard).controls(g.getBasicBlock(), false) + // `File.getName` strips any directory prefix, returning only the final path + // component. The only remaining path traversal risk is when that component is + // itself `..`, so a check for `..` components completes the sanitization. + exists(FileGetNameCall getName | localTaintFlowToPathGuard(getName, g)) ) } diff --git a/java/ql/test/query-tests/security/CWE-022/semmle/tests/TaintedPath.expected b/java/ql/test/query-tests/security/CWE-022/semmle/tests/TaintedPath.expected index bfeb4d730c2..b44b294edc4 100644 --- a/java/ql/test/query-tests/security/CWE-022/semmle/tests/TaintedPath.expected +++ b/java/ql/test/query-tests/security/CWE-022/semmle/tests/TaintedPath.expected @@ -1,6 +1,7 @@ #select | SanitizationTests2.java:16:59:16:62 | path | SanitizationTests2.java:14:59:14:91 | path : String | SanitizationTests2.java:16:59:16:62 | path | This path depends on a $@. | SanitizationTests2.java:14:59:14:91 | path | user-provided value | | TaintedPath.java:16:71:16:78 | filename | TaintedPath.java:13:58:13:78 | getInputStream(...) : InputStream | TaintedPath.java:16:71:16:78 | filename | This path depends on a $@. | TaintedPath.java:13:58:13:78 | getInputStream(...) | user-provided value | +| TaintedPath.java:105:71:105:78 | baseName | TaintedPath.java:98:58:98:78 | getInputStream(...) : InputStream | TaintedPath.java:105:71:105:78 | baseName | This path depends on a $@. | TaintedPath.java:98:58:98:78 | getInputStream(...) | user-provided value | | Test.java:37:52:37:68 | (...)... | Test.java:32:16:32:45 | getParameter(...) : String | Test.java:37:52:37:68 | (...)... | This path depends on a $@. | Test.java:32:16:32:45 | getParameter(...) | user-provided value | | Test.java:39:32:39:48 | (...)... | Test.java:32:16:32:45 | getParameter(...) : String | Test.java:39:32:39:48 | (...)... | This path depends on a $@. | Test.java:32:16:32:45 | getParameter(...) | user-provided value | | Test.java:41:47:41:63 | (...)... | Test.java:32:16:32:45 | getParameter(...) : String | Test.java:41:47:41:63 | (...)... | This path depends on a $@. | Test.java:32:16:32:45 | getParameter(...) | user-provided value | @@ -81,9 +82,18 @@ edges | SanitizationTests2.java:14:59:14:91 | path : String | SanitizationTests2.java:16:59:16:62 | path | provenance | Sink:MaD:23 | | TaintedPath.java:13:17:13:89 | new BufferedReader(...) : BufferedReader | TaintedPath.java:14:27:14:40 | filenameReader : BufferedReader | provenance | | | TaintedPath.java:13:36:13:88 | new InputStreamReader(...) : InputStreamReader | TaintedPath.java:13:17:13:89 | new BufferedReader(...) : BufferedReader | provenance | MaD:74 | -| TaintedPath.java:13:58:13:78 | getInputStream(...) : InputStream | TaintedPath.java:13:36:13:88 | new InputStreamReader(...) : InputStreamReader | provenance | Src:MaD:72 MaD:76 | +| TaintedPath.java:13:58:13:78 | getInputStream(...) : InputStream | TaintedPath.java:13:36:13:88 | new InputStreamReader(...) : InputStreamReader | provenance | Src:MaD:72 MaD:78 | | TaintedPath.java:14:27:14:40 | filenameReader : BufferedReader | TaintedPath.java:14:27:14:51 | readLine(...) : String | provenance | MaD:75 | | TaintedPath.java:14:27:14:51 | readLine(...) : String | TaintedPath.java:16:71:16:78 | filename | provenance | Sink:MaD:27 | +| TaintedPath.java:98:17:98:89 | new BufferedReader(...) : BufferedReader | TaintedPath.java:99:27:99:40 | filenameReader : BufferedReader | provenance | | +| TaintedPath.java:98:36:98:88 | new InputStreamReader(...) : InputStreamReader | TaintedPath.java:98:17:98:89 | new BufferedReader(...) : BufferedReader | provenance | MaD:74 | +| TaintedPath.java:98:58:98:78 | getInputStream(...) : InputStream | TaintedPath.java:98:36:98:88 | new InputStreamReader(...) : InputStreamReader | provenance | Src:MaD:72 MaD:78 | +| TaintedPath.java:99:27:99:40 | filenameReader : BufferedReader | TaintedPath.java:99:27:99:51 | readLine(...) : String | provenance | MaD:75 | +| TaintedPath.java:99:27:99:51 | readLine(...) : String | TaintedPath.java:100:30:100:37 | filename : String | provenance | | +| TaintedPath.java:100:21:100:38 | new File(...) : File | TaintedPath.java:101:27:101:30 | file : File | provenance | | +| TaintedPath.java:100:30:100:37 | filename : String | TaintedPath.java:100:21:100:38 | new File(...) : File | provenance | MaD:76 | +| TaintedPath.java:101:27:101:30 | file : File | TaintedPath.java:101:27:101:40 | getName(...) : String | provenance | MaD:77 | +| TaintedPath.java:101:27:101:40 | getName(...) : String | TaintedPath.java:105:71:105:78 | baseName | provenance | Sink:MaD:27 | | Test.java:32:16:32:45 | getParameter(...) : String | Test.java:37:61:37:68 | source(...) : String | provenance | Src:MaD:73 | | Test.java:32:16:32:45 | getParameter(...) : String | Test.java:39:41:39:48 | source(...) : String | provenance | Src:MaD:73 | | Test.java:32:16:32:45 | getParameter(...) : String | Test.java:41:56:41:63 | source(...) : String | provenance | Src:MaD:73 | @@ -312,7 +322,9 @@ models | 73 | Source: javax.servlet; ServletRequest; false; getParameter; (String); ; ReturnValue; remote; manual | | 74 | Summary: java.io; BufferedReader; false; BufferedReader; ; ; Argument[0]; Argument[this]; taint; manual | | 75 | Summary: java.io; BufferedReader; true; readLine; ; ; Argument[this]; ReturnValue; taint; manual | -| 76 | Summary: java.io; InputStreamReader; false; InputStreamReader; ; ; Argument[0]; Argument[this]; taint; manual | +| 76 | Summary: java.io; File; false; File; ; ; Argument[0]; Argument[this]; taint; manual | +| 77 | Summary: java.io; File; true; getName; (); ; Argument[this]; ReturnValue; taint; manual | +| 78 | Summary: java.io; InputStreamReader; false; InputStreamReader; ; ; Argument[0]; Argument[this]; taint; manual | nodes | SanitizationTests2.java:14:59:14:91 | path : String | semmle.label | path : String | | SanitizationTests2.java:16:59:16:62 | path | semmle.label | path | @@ -322,6 +334,16 @@ nodes | TaintedPath.java:14:27:14:40 | filenameReader : BufferedReader | semmle.label | filenameReader : BufferedReader | | TaintedPath.java:14:27:14:51 | readLine(...) : String | semmle.label | readLine(...) : String | | TaintedPath.java:16:71:16:78 | filename | semmle.label | filename | +| TaintedPath.java:98:17:98:89 | new BufferedReader(...) : BufferedReader | semmle.label | new BufferedReader(...) : BufferedReader | +| TaintedPath.java:98:36:98:88 | new InputStreamReader(...) : InputStreamReader | semmle.label | new InputStreamReader(...) : InputStreamReader | +| TaintedPath.java:98:58:98:78 | getInputStream(...) : InputStream | semmle.label | getInputStream(...) : InputStream | +| TaintedPath.java:99:27:99:40 | filenameReader : BufferedReader | semmle.label | filenameReader : BufferedReader | +| TaintedPath.java:99:27:99:51 | readLine(...) : String | semmle.label | readLine(...) : String | +| TaintedPath.java:100:21:100:38 | new File(...) : File | semmle.label | new File(...) : File | +| TaintedPath.java:100:30:100:37 | filename : String | semmle.label | filename : String | +| TaintedPath.java:101:27:101:30 | file : File | semmle.label | file : File | +| TaintedPath.java:101:27:101:40 | getName(...) : String | semmle.label | getName(...) : String | +| TaintedPath.java:105:71:105:78 | baseName | semmle.label | baseName | | Test.java:32:16:32:45 | getParameter(...) : String | semmle.label | getParameter(...) : String | | Test.java:37:52:37:68 | (...)... | semmle.label | (...)... | | Test.java:37:61:37:68 | source(...) : String | semmle.label | source(...) : String | diff --git a/java/ql/test/query-tests/security/CWE-022/semmle/tests/TaintedPath.java b/java/ql/test/query-tests/security/CWE-022/semmle/tests/TaintedPath.java index fffb93c6291..fc01e1d2356 100644 --- a/java/ql/test/query-tests/security/CWE-022/semmle/tests/TaintedPath.java +++ b/java/ql/test/query-tests/security/CWE-022/semmle/tests/TaintedPath.java @@ -93,18 +93,38 @@ public class TaintedPath { } } - public void sendUserFileGood4(Socket sock, String user) throws IOException { + public void sendUserFileBad2(Socket sock, String user) throws IOException { BufferedReader filenameReader = - new BufferedReader(new InputStreamReader(sock.getInputStream(), "UTF-8")); + new BufferedReader(new InputStreamReader(sock.getInputStream(), "UTF-8")); // $ Source[java/path-injection] String filename = filenameReader.readLine(); File file = new File(filename); String baseName = file.getName(); - // GOOD: only use the final component of the user provided path - BufferedReader fileReader = new BufferedReader(new FileReader(baseName)); + // BAD: `getName()` strips directory separators but does not remove a `..` + // component (`new File("..").getName()` returns ".."), so it is not a + // complete path injection sanitizer. + BufferedReader fileReader = new BufferedReader(new FileReader(baseName)); // $ Alert[java/path-injection] String fileLine = fileReader.readLine(); while (fileLine != null) { sock.getOutputStream().write(fileLine.getBytes()); fileLine = fileReader.readLine(); } } + + public void sendUserFileGood4(Socket sock, String user) throws IOException { + BufferedReader filenameReader = + new BufferedReader(new InputStreamReader(sock.getInputStream(), "UTF-8")); + String filename = filenameReader.readLine(); + File file = new File(filename); + String baseName = file.getName(); + // GOOD: `getName()` strips directory separators and the `..` check removes + // the only remaining traversal possibility. + if (!baseName.contains("..")) { + BufferedReader fileReader = new BufferedReader(new FileReader(baseName)); + String fileLine = fileReader.readLine(); + while (fileLine != null) { + sock.getOutputStream().write(fileLine.getBytes()); + fileLine = fileReader.readLine(); + } + } + } } From 63ff22abd6cad01854c0c087456489c82acb0dd5 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Mon, 13 Jul 2026 00:08:51 +0200 Subject: [PATCH 41/90] Go: Remove dead `ObjectsOverride` --- go/extractor/extractor.go | 31 ------------------------------- go/extractor/trap/trapwriter.go | 20 +++++++++----------- 2 files changed, 9 insertions(+), 42 deletions(-) diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index 158f0029704..f4188cfb681 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -1644,7 +1644,6 @@ func extractType(tw *trap.Writer, tp types.Type) trap.Label { dbscheme.TypeNameTable.Emit(tw, lbl, origintp.Obj().Name()) underlying := origintp.Underlying() extractUnderlyingType(tw, lbl, underlying) - trackInstantiatedStructFields(tw, tp, origintp) entitylbl, exists := tw.Labeler.LookupObjectID(origintp.Obj(), lbl) if entitylbl == trap.InvalidLabel { @@ -2035,36 +2034,6 @@ func getObjectBeingUsed(tw *trap.Writer, ident *ast.Ident) types.Object { } } -// trackInstantiatedStructFields tries to give the fields of an instantiated -// struct type underlying `tp` the same labels as the corresponding fields of -// the generic struct type. This is so that when we come across the -// instantiated field in `tw.Package.TypesInfo.Uses` we will get the label for -// the generic field instead. -func trackInstantiatedStructFields(tw *trap.Writer, tp, origintp *types.Named) { - if tp == origintp { - return - } - - if instantiatedStruct, ok := tp.Underlying().(*types.Struct); ok { - genericStruct, ok2 := origintp.Underlying().(*types.Struct) - if !ok2 { - log.Fatalf( - "Error: underlying type of instantiated type is a struct but underlying type of generic type is %s", - origintp.Underlying()) - } - - if instantiatedStruct.NumFields() != genericStruct.NumFields() { - log.Fatalf( - "Error: instantiated struct %s has different number of fields than the generic version %s (%d != %d)", - instantiatedStruct, genericStruct, instantiatedStruct.NumFields(), genericStruct.NumFields()) - } - - for i := 0; i < instantiatedStruct.NumFields(); i++ { - tw.ObjectsOverride[instantiatedStruct.Field(i)] = genericStruct.Field(i) - } - } -} - func getTypeParamParentLabel(tw *trap.Writer, tp *types.TypeParam) trap.Label { parent, exists := typeParamParent[tp] if !exists { diff --git a/go/extractor/trap/trapwriter.go b/go/extractor/trap/trapwriter.go index 50763719f50..c565f343665 100644 --- a/go/extractor/trap/trapwriter.go +++ b/go/extractor/trap/trapwriter.go @@ -17,16 +17,15 @@ import ( // A Writer provides methods for writing data to a TRAP file type Writer struct { - zip *gzip.Writer - wzip *bufio.Writer - wfile *bufio.Writer - file *os.File - Labeler *Labeler - path string - trapFilePath string - Package *packages.Package - TypesOverride map[ast.Expr]types.Type - ObjectsOverride map[types.Object]types.Object + zip *gzip.Writer + wzip *bufio.Writer + wfile *bufio.Writer + file *os.File + Labeler *Labeler + path string + trapFilePath string + Package *packages.Package + TypesOverride map[ast.Expr]types.Type } func FileFor(path string) (string, error) { @@ -67,7 +66,6 @@ func NewWriter(path string, pkg *packages.Package) (*Writer, error) { trapFilePath, pkg, make(map[ast.Expr]types.Type), - make(map[types.Object]types.Object), } tw.Labeler = newLabeler(tw) return tw, nil From 559f9f38500dbaff9ba091e1af725e2b14592450 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Mon, 13 Jul 2026 00:38:58 +0200 Subject: [PATCH 42/90] Go: Remove dead `isAlias` --- go/extractor/extractor.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index f4188cfb681..0e7fd199e7c 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -1529,12 +1529,6 @@ func extractSpec(tw *trap.Writer, spec ast.Spec, parent trap.Label, idx int) { extractNodeLocation(tw, spec, lbl) } -// Determines whether the given type is an alias. -func isAlias(tp types.Type) bool { - _, ok := tp.(*types.Alias) - return ok -} - // extractType extracts type information for `tp` and returns its associated label; // types are only extracted once, so the second time `extractType` is invoked it simply returns the label func extractType(tw *trap.Writer, tp types.Type) trap.Label { From 0584ca09dca8dfde7f7fd309be033f95b51a5429 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 04:12:46 +0000 Subject: [PATCH 43/90] Go: Update to 1.26.5 --- MODULE.bazel | 2 +- go/actions/test/action.yml | 2 +- go/extractor/go.mod | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index e6f1ddb893d..9f9a2125da9 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -276,7 +276,7 @@ use_repo( ) go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") -go_sdk.download(version = "1.26.4") +go_sdk.download(version = "1.26.5") go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps") go_deps.from_file(go_mod = "//go/extractor:go.mod") diff --git a/go/actions/test/action.yml b/go/actions/test/action.yml index 3cc3334d39e..d4c391cd010 100644 --- a/go/actions/test/action.yml +++ b/go/actions/test/action.yml @@ -4,7 +4,7 @@ inputs: go-test-version: description: Which Go version to use for running the tests required: false - default: "~1.26.4" + default: "~1.26.5" run-code-checks: description: Whether to run formatting, code and qhelp generation checks required: false diff --git a/go/extractor/go.mod b/go/extractor/go.mod index e121c4a98a8..b45815abbc9 100644 --- a/go/extractor/go.mod +++ b/go/extractor/go.mod @@ -2,7 +2,7 @@ module github.com/github/codeql-go/extractor go 1.26 -toolchain go1.26.4 +toolchain go1.26.5 // when updating this, run // bazel run @rules_go//go -- mod tidy From 032636cf347366b163fc3a8b4c60ffb5ba65fea3 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Sun, 12 Jul 2026 21:22:53 +0200 Subject: [PATCH 44/90] Go: Track whether a type parameter was defined as part of a receiver --- go/extractor/dbscheme/dbscheme.go | 5 +++ go/extractor/dbscheme/tables.go | 3 +- go/extractor/extractor.go | 59 +++++++++++++++++++------------ go/extractor/trap/trapwriter.go | 2 ++ go/ql/lib/go.dbscheme | 6 ++-- go/ql/lib/go.dbscheme.stats | 4 +++ go/ql/lib/semmle/go/Types.qll | 8 ++--- 7 files changed, 56 insertions(+), 31 deletions(-) diff --git a/go/extractor/dbscheme/dbscheme.go b/go/extractor/dbscheme/dbscheme.go index 550c7920c71..45aea39b4e5 100644 --- a/go/extractor/dbscheme/dbscheme.go +++ b/go/extractor/dbscheme/dbscheme.go @@ -277,6 +277,11 @@ func FloatColumn(columnName string) Column { return Column{columnName, FLOAT, false, true} } +// BooleanColumn constructs a column with name `columnName` holding boolean values +func BooleanColumn(columnName string) Column { + return Column{columnName, BOOLEAN, false, true} +} + // A Table represents a database table type Table struct { name string diff --git a/go/extractor/dbscheme/tables.go b/go/extractor/dbscheme/tables.go index 9c537fbaf89..b72c3379518 100644 --- a/go/extractor/dbscheme/tables.go +++ b/go/extractor/dbscheme/tables.go @@ -1241,4 +1241,5 @@ var TypeParamTable = NewTable("typeparam", EntityColumn(CompositeType, "bound"), EntityColumn(TypeParamParentObjectType, "parent"), IntColumn("idx"), -).KeySet("parent", "idx") + BooleanColumn("is_from_recv"), +).KeySet("parent", "idx", "is_from_recv") diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index 0e7fd199e7c..f421688e52a 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -32,7 +32,13 @@ import ( ) var MaxGoRoutines int -var typeParamParent map[*types.TypeParam]types.Object = make(map[*types.TypeParam]types.Object) + +type typeParamParentEntry struct { + parent types.Object + isFromReceiver bool +} + +var typeParamParent map[*types.TypeParam]typeParamParentEntry = make(map[*types.TypeParam]typeParamParentEntry) func init() { // this sets the number of threads that the Go runtime will spawn; this is separate @@ -530,8 +536,7 @@ func extractObjects(tw *trap.Writer, scope *types.Scope, scopeLabel trap.Label) // do not appear as objects in any scope, so they have to be dealt // with separately in extractMethods. if funcObj, ok := obj.(*types.Func); ok { - populateTypeParamParents(funcObj.Type().(*types.Signature).TypeParams(), obj) - populateTypeParamParents(funcObj.Type().(*types.Signature).RecvTypeParams(), obj) + populateTypeParamParentsFromFunction(funcObj) } // Populate type parameter parents for defined types and alias types. if typeNameObj, ok := obj.(*types.TypeName); ok { @@ -542,9 +547,9 @@ func extractObjects(tw *trap.Writer, scope *types.Scope, scopeLabel trap.Label) // careful with alias types because before Go 1.24 they would // return the underlying type. if tp, ok := typeNameObj.Type().(*types.Named); ok && !typeNameObj.IsAlias() { - populateTypeParamParents(tp.TypeParams(), obj) + populateTypeParamParents(tp.TypeParams(), obj, false) } else if tp, ok := typeNameObj.Type().(*types.Alias); ok { - populateTypeParamParents(tp.TypeParams(), obj) + populateTypeParamParents(tp.TypeParams(), obj, false) } } extractObject(tw, obj, lbl) @@ -570,8 +575,7 @@ func extractMethod(tw *trap.Writer, meth *types.Func) trap.Label { if !exists { // Populate type parameter parents for methods. They do not appear as // objects in any scope, so they have to be dealt with separately here. - populateTypeParamParents(meth.Type().(*types.Signature).TypeParams(), meth) - populateTypeParamParents(meth.Type().(*types.Signature).RecvTypeParams(), meth) + populateTypeParamParentsFromFunction(meth) extractObject(tw, meth, methlbl) } @@ -1677,9 +1681,9 @@ func extractType(tw *trap.Writer, tp types.Type) trap.Label { } case *types.TypeParam: kind = dbscheme.TypeParamType.Index() - parentlbl := getTypeParamParentLabel(tw, tp) + parentlbl, isReceiverChild := getTypeParamParentLabel(tw, tp) constraintLabel := extractType(tw, tp.Constraint()) - dbscheme.TypeParamTable.Emit(tw, lbl, tp.Obj().Name(), constraintLabel, parentlbl, tp.Index()) + dbscheme.TypeParamTable.Emit(tw, lbl, tp.Obj().Name(), constraintLabel, parentlbl, tp.Index(), isReceiverChild) case *types.Union: kind = dbscheme.TypeSetLiteral.Index() for i := 0; i < tp.Len(); i++ { @@ -1819,9 +1823,9 @@ func getTypeLabel(tw *trap.Writer, tp types.Type) (trap.Label, bool) { } lbl = tw.Labeler.GlobalID(fmt.Sprintf("{%s};definedtype", entitylbl)) case *types.TypeParam: - parentlbl := getTypeParamParentLabel(tw, tp) + parentlbl, isReceiverChild := getTypeParamParentLabel(tw, tp) idx := tp.Index() - lbl = tw.Labeler.GlobalID(fmt.Sprintf("{%v},%d,%s;typeparamtype", parentlbl, idx, tp.Obj().Name())) + lbl = tw.Labeler.GlobalID(fmt.Sprintf("{%v},%d,%t,%s;typeparamtype", parentlbl, idx, isReceiverChild, tp.Obj().Name())) case *types.Union: var b strings.Builder for i := 0; i < tp.Len(); i++ { @@ -2005,11 +2009,20 @@ func extractTypeParamDecls(tw *trap.Writer, fields *ast.FieldList, parent trap.L } } +func populateTypeParamParentsFromFunction(funcObj *types.Func) { + recvTypeParams := funcObj.Type().(*types.Signature).RecvTypeParams() + populateTypeParamParents(recvTypeParams, funcObj, true) + typeParams := funcObj.Type().(*types.Signature).TypeParams() + populateTypeParamParents(typeParams, funcObj, false) + +} + // populateTypeParamParents sets `parent` as the parent of the elements of `typeparams` -func populateTypeParamParents(typeparams *types.TypeParamList, parent types.Object) { +// and records whether elements are defined in a receiver. +func populateTypeParamParents(typeparams *types.TypeParamList, parent types.Object, isFromReceiver bool) { if typeparams != nil { - for idx := 0; idx < typeparams.Len(); idx++ { - setTypeParamParent(typeparams.At(idx), parent) + for tparam := range typeparams.TypeParams() { + setTypeParamParent(tparam, parent, isFromReceiver) } } } @@ -2028,24 +2041,24 @@ func getObjectBeingUsed(tw *trap.Writer, ident *ast.Ident) types.Object { } } -func getTypeParamParentLabel(tw *trap.Writer, tp *types.TypeParam) trap.Label { - parent, exists := typeParamParent[tp] +func getTypeParamParentLabel(tw *trap.Writer, tp *types.TypeParam) (trap.Label, bool) { + entry, exists := typeParamParent[tp] if !exists { log.Fatalf("Parent of type parameter does not exist: %s %s", tp.String(), tp.Constraint().String()) } - parentlbl, _ := tw.Labeler.ScopedObjectID(parent, func() trap.Label { + parentlbl, _ := tw.Labeler.ScopedObjectID(entry.parent, func() trap.Label { log.Fatalf("getTypeLabel() called for parent of type parameter %s", tp.String()) return trap.InvalidLabel }) - return parentlbl + return parentlbl, entry.isFromReceiver } -func setTypeParamParent(tp *types.TypeParam, newobj types.Object) { - obj, exists := typeParamParent[tp] +func setTypeParamParent(tp *types.TypeParam, newobj types.Object, isFromReceiver bool) { + entry, exists := typeParamParent[tp] if !exists { - typeParamParent[tp] = newobj - } else if newobj != obj { - log.Fatalf("Parent of type parameter '%s %s' being set to a different value: '%s' vs '%s'", tp.String(), tp.Constraint().String(), obj, newobj) + typeParamParent[tp] = typeParamParentEntry{newobj, isFromReceiver} + } else if entry.parent != newobj { + log.Fatalf("Parent of type parameter '%s %s' being set to a different value: '%s' vs '%s'", tp.String(), tp.Constraint().String(), entry.parent, newobj) } } diff --git a/go/extractor/trap/trapwriter.go b/go/extractor/trap/trapwriter.go index c565f343665..fa7436be48b 100644 --- a/go/extractor/trap/trapwriter.go +++ b/go/extractor/trap/trapwriter.go @@ -165,6 +165,8 @@ func (tw *Writer) Emit(table string, values []interface{}) error { fmt.Fprintf(tw.wzip, "%d", value) case float64: fmt.Fprintf(tw.wzip, "%e", value) + case bool: + fmt.Fprintf(tw.wzip, "%t", value) default: return errors.New("Cannot emit value") } diff --git a/go/ql/lib/go.dbscheme b/go/ql/lib/go.dbscheme index b1341734d68..5ff5325d274 100644 --- a/go/ql/lib/go.dbscheme +++ b/go/ql/lib/go.dbscheme @@ -244,9 +244,9 @@ has_ellipsis(int id: @callorconversionexpr ref); variadic(int id: @signaturetype ref); -#keyset[parent, idx] -typeparam(unique int tp: @typeparamtype ref, string name: string ref, int bound: @compositetype ref, - int parent: @typeparamparentobject ref, int idx: int ref); +#keyset[parent, idx, is_from_recv] +typeparam(unique int tp: @typeparamtype ref, string name: string ref, + int bound: @compositetype ref, int parent: @typeparamparentobject ref, int idx: int ref, boolean is_from_recv: boolean ref); @container = @file | @folder; diff --git a/go/ql/lib/go.dbscheme.stats b/go/ql/lib/go.dbscheme.stats index 126bbff00f7..dd9440b0998 100644 --- a/go/ql/lib/go.dbscheme.stats +++ b/go/ql/lib/go.dbscheme.stats @@ -17791,6 +17791,10 @@ idx 3126 + + is_from_recv + 0 + diff --git a/go/ql/lib/semmle/go/Types.qll b/go/ql/lib/semmle/go/Types.qll index 574d7568543..217ac5d211c 100644 --- a/go/ql/lib/semmle/go/Types.qll +++ b/go/ql/lib/semmle/go/Types.qll @@ -382,18 +382,18 @@ class CompositeType extends @compositetype, Type { } /** A type that comes from a type parameter. */ class TypeParamType extends @typeparamtype, CompositeType { /** Gets the name of this type parameter type. */ - string getParamName() { typeparam(this, result, _, _, _) } + string getParamName() { typeparam(this, result, _, _, _, _) } /** Gets the constraint of this type parameter type. */ - Type getConstraint() { typeparam(this, _, result, _, _) } + Type getConstraint() { typeparam(this, _, result, _, _, _) } override InterfaceType getUnderlyingType() { result = this.getConstraint().getUnderlyingType() } /** Gets the parent object of this type parameter type. */ - TypeParamParentEntity getParent() { typeparam(this, _, _, result, _) } + TypeParamParentEntity getParent() { typeparam(this, _, _, result, _, _) } /** Gets the index of this type parameter type. */ - int getIndex() { typeparam(this, _, _, _, result) } + int getIndex() { typeparam(this, _, _, _, result, _) } override string pp() { result = this.getParamName() } From 37454ab5f647692fee2c6dde530d64c464b7a8df Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Mon, 13 Jul 2026 11:53:26 +0200 Subject: [PATCH 45/90] Go: Add upgrade and downgrade scripts --- .../go.dbscheme | 563 ++++++++++++++++++ .../old.dbscheme | 563 ++++++++++++++++++ .../typeparam.ql | 15 + .../upgrade.properties | 3 + .../go.dbscheme | 563 ++++++++++++++++++ .../old.dbscheme | 563 ++++++++++++++++++ .../typeparam.ql | 15 + .../upgrade.properties | 3 + 8 files changed, 2288 insertions(+) create mode 100644 go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/go.dbscheme create mode 100644 go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/old.dbscheme create mode 100644 go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/typeparam.ql create mode 100644 go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/upgrade.properties create mode 100644 go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/go.dbscheme create mode 100644 go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/old.dbscheme create mode 100644 go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/typeparam.ql create mode 100644 go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/upgrade.properties diff --git a/go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/go.dbscheme b/go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/go.dbscheme new file mode 100644 index 00000000000..b1341734d68 --- /dev/null +++ b/go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/go.dbscheme @@ -0,0 +1,563 @@ +/** Auto-generated dbscheme; do not edit. Run `make gen` in directory `go/` to regenerate. */ + + +/** Duplicate code **/ + +duplicateCode( + unique int id : @duplication, + varchar(900) relativePath : string ref, + int equivClass : int ref); + +similarCode( + unique int id : @similarity, + varchar(900) relativePath : string ref, + int equivClass : int ref); + +@duplication_or_similarity = @duplication | @similarity; + +tokens( + int id : @duplication_or_similarity ref, + int offset : int ref, + int beginLine : int ref, + int beginColumn : int ref, + int endLine : int ref, + int endColumn : int ref); + +/** External data **/ + +externalData( + int id : @externalDataElement, + varchar(900) path : string ref, + int column: int ref, + varchar(900) value : string ref +); + +snapshotDate(unique date snapshotDate : date ref); + +sourceLocationPrefix(varchar(900) prefix : string ref); + +/** Overlay support **/ + +databaseMetadata( + string metadataKey: string ref, + string value: string ref +); + +overlayChangedFiles( + string path: string ref +); + + +/* + * XML Files + */ + +xmlEncoding( + unique int id: @file ref, + string encoding: string ref +); + +xmlDTDs( + unique int id: @xmldtd, + string root: string ref, + string publicId: string ref, + string systemId: string ref, + int fileid: @file ref +); + +xmlElements( + unique int id: @xmlelement, + string name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref +); + +xmlAttrs( + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + string name: string ref, + string value: string ref, + int idx: int ref, + int fileid: @file ref +); + +xmlNs( + int id: @xmlnamespace, + string prefixName: string ref, + string URI: string ref, + int fileid: @file ref +); + +xmlHasNs( + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref +); + +xmlComments( + unique int id: @xmlcomment, + string text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref +); + +xmlChars( + unique int id: @xmlcharacters, + string text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref +); + +@xmlparent = @file | @xmlelement; +@xmlnamespaceable = @xmlelement | @xmlattribute; + +xmllocations( + int xmlElement: @xmllocatable ref, + int location: @location_default ref +); + +@xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace; + +compilations(unique int id: @compilation, string cwd: string ref); + +#keyset[id, num] +compilation_args(int id: @compilation ref, int num: int ref, string arg: string ref); + +#keyset[id, num, kind] +compilation_time(int id: @compilation ref, int num: int ref, int kind: int ref, float secs: float ref); + +diagnostic_for(unique int diagnostic: @diagnostic ref, int compilation: @compilation ref, int file_number: int ref, int file_number_diagnostic_number: int ref); + +compilation_finished(unique int id: @compilation ref, float cpu_seconds: float ref, float elapsed_seconds: float ref); + +#keyset[id, num] +compilation_compiling_files(int id: @compilation ref, int num: int ref, int file: @file ref); + +diagnostics(unique int id: @diagnostic, int severity: int ref, string error_tag: string ref, string error_message: string ref, + string full_error_message: string ref, int location: @location ref); + +locations_default(unique int id: @location_default, int file: @file ref, int beginLine: int ref, int beginColumn: int ref, + int endLine: int ref, int endColumn: int ref); + +numlines(int element_id: @sourceline ref, int num_lines: int ref, int num_code: int ref, int num_comment: int ref); + +files(unique int id: @file, string name: string ref); + +folders(unique int id: @folder, string name: string ref); + +containerparent(int parent: @container ref, unique int child: @container ref); + +has_location(unique int locatable: @locatable ref, int location: @location ref); + +#keyset[parent, idx] +comment_groups(unique int id: @comment_group, int parent: @file ref, int idx: int ref); + +comments(unique int id: @comment, int kind: int ref, int parent: @comment_group ref, int idx: int ref, string text: string ref); + +doc_comments(unique int node: @documentable ref, int comment: @comment_group ref); + +#keyset[parent, idx] +exprs(unique int id: @expr, int kind: int ref, int parent: @exprparent ref, int idx: int ref); + +literals(unique int expr: @expr ref, string value: string ref, string raw: string ref); + +constvalues(unique int expr: @expr ref, string value: string ref, string exact: string ref); + +fields(unique int id: @field, int parent: @fieldparent ref, int idx: int ref); + +typeparamdecls(unique int id: @typeparamdecl, int parent: @typeparamdeclparent ref, int idx: int ref); + +#keyset[parent, idx] +stmts(unique int id: @stmt, int kind: int ref, int parent: @stmtparent ref, int idx: int ref); + +#keyset[parent, idx] +decls(unique int id: @decl, int kind: int ref, int parent: @declparent ref, int idx: int ref); + +#keyset[parent, idx] +specs(unique int id: @spec, int kind: int ref, int parent: @gendecl ref, int idx: int ref); + +scopes(unique int id: @scope, int kind: int ref); + +scopenesting(unique int inner: @scope ref, int outer: @scope ref); + +scopenodes(unique int node: @scopenode ref, int scope: @localscope ref); + +objects(unique int id: @object, int kind: int ref, string name: string ref); + +objectscopes(unique int object: @object ref, int scope: @scope ref); + +objecttypes(unique int object: @object ref, int tp: @type ref); + +methodreceivers(unique int method: @object ref, int receiver: @object ref); + +fieldstructs(unique int field: @object ref, int struct: @structtype ref); + +methodhosts(int method: @object ref, int host: @definedtype ref); + +defs(int ident: @ident ref, int object: @object ref); + +uses(int ident: @ident ref, int object: @object ref); + +types(unique int id: @type, int kind: int ref); + +type_of(unique int expr: @expr ref, int tp: @type ref); + +typename(unique int tp: @type ref, string name: string ref); + +key_type(unique int map: @maptype ref, int tp: @type ref); + +element_type(unique int container: @containertype ref, int tp: @type ref); + +base_type(unique int ptr: @pointertype ref, int tp: @type ref); + +underlying_type(unique int defined: @definedtype ref, int tp: @type ref); + +#keyset[parent, index] +component_types(int parent: @compositetype ref, int index: int ref, string name: string ref, int tp: @type ref); + +#keyset[parent, index] +struct_tags(int parent: @structtype ref, int index: int ref, string tag: string ref); + +#keyset[interface, index] +interface_private_method_ids(int interface: @interfacetype ref, int index: int ref, string id: string ref); + +array_length(unique int tp: @arraytype ref, string len: string ref); + +type_objects(unique int tp: @type ref, int object: @object ref); + +packages(unique int id: @package, string name: string ref, string path: string ref, int scope: @packagescope ref); + +#keyset[parent, idx] +modexprs(unique int id: @modexpr, int kind: int ref, int parent: @modexprparent ref, int idx: int ref); + +#keyset[parent, idx] +modtokens(string token: string ref, int parent: @modexpr ref, int idx: int ref); + +#keyset[package, idx] +errors(unique int id: @error, int kind: int ref, string msg: string ref, string rawpos: string ref, + string file: string ref, int line: int ref, int col: int ref, int package: @package ref, int idx: int ref); + +has_ellipsis(int id: @callorconversionexpr ref); + +variadic(int id: @signaturetype ref); + +#keyset[parent, idx] +typeparam(unique int tp: @typeparamtype ref, string name: string ref, int bound: @compositetype ref, + int parent: @typeparamparentobject ref, int idx: int ref); + +@container = @file | @folder; + +@locatable = @xmllocatable | @node | @localscope; + +@node = @documentable | @exprparent | @modexprparent | @fieldparent | @stmtparent | @declparent | @typeparamdeclparent + | @scopenode | @comment_group | @comment; + +@documentable = @file | @field | @typeparamdecl | @spec | @gendecl | @funcdecl | @modexpr; + +@exprparent = @funcdef | @file | @expr | @field | @stmt | @decl | @typeparamdecl | @spec; + +@modexprparent = @file | @modexpr; + +@fieldparent = @decl | @structtypeexpr | @functypeexpr | @interfacetypeexpr; + +@stmtparent = @funcdef | @stmt | @decl; + +@declparent = @file | @declstmt; + +@typeparamdeclparent = @funcdecl | @typespec; + +@funcdef = @funclit | @funcdecl; + +@scopenode = @file | @functypeexpr | @blockstmt | @ifstmt | @caseclause | @switchstmt | @commclause | @loopstmt; + +@location = @location_default; + +@sourceline = @locatable; + +case @comment.kind of + 0 = @slashslashcomment +| 1 = @slashstarcomment; + +case @expr.kind of + 0 = @badexpr +| 1 = @ident +| 2 = @ellipsis +| 3 = @intlit +| 4 = @floatlit +| 5 = @imaglit +| 6 = @charlit +| 7 = @stringlit +| 8 = @funclit +| 9 = @compositelit +| 10 = @parenexpr +| 11 = @selectorexpr +| 12 = @indexexpr +| 13 = @genericfunctioninstantiationexpr +| 14 = @generictypeinstantiationexpr +| 15 = @sliceexpr +| 16 = @typeassertexpr +| 17 = @callorconversionexpr +| 18 = @starexpr +| 19 = @keyvalueexpr +| 20 = @arraytypeexpr +| 21 = @structtypeexpr +| 22 = @functypeexpr +| 23 = @interfacetypeexpr +| 24 = @maptypeexpr +| 25 = @typesetliteralexpr +| 26 = @plusexpr +| 27 = @minusexpr +| 28 = @notexpr +| 29 = @complementexpr +| 30 = @derefexpr +| 31 = @addressexpr +| 32 = @arrowexpr +| 33 = @lorexpr +| 34 = @landexpr +| 35 = @eqlexpr +| 36 = @neqexpr +| 37 = @lssexpr +| 38 = @leqexpr +| 39 = @gtrexpr +| 40 = @geqexpr +| 41 = @addexpr +| 42 = @subexpr +| 43 = @orexpr +| 44 = @xorexpr +| 45 = @mulexpr +| 46 = @quoexpr +| 47 = @remexpr +| 48 = @shlexpr +| 49 = @shrexpr +| 50 = @andexpr +| 51 = @andnotexpr +| 52 = @sendchantypeexpr +| 53 = @recvchantypeexpr +| 54 = @sendrcvchantypeexpr; + +@basiclit = @intlit | @floatlit | @imaglit | @charlit | @stringlit; + +@operatorexpr = @logicalexpr | @arithmeticexpr | @bitwiseexpr | @unaryexpr | @binaryexpr; + +@logicalexpr = @logicalunaryexpr | @logicalbinaryexpr; + +@arithmeticexpr = @arithmeticunaryexpr | @arithmeticbinaryexpr; + +@bitwiseexpr = @bitwiseunaryexpr | @bitwisebinaryexpr; + +@unaryexpr = @logicalunaryexpr | @bitwiseunaryexpr | @arithmeticunaryexpr | @derefexpr | @addressexpr | @arrowexpr; + +@logicalunaryexpr = @notexpr; + +@bitwiseunaryexpr = @complementexpr; + +@arithmeticunaryexpr = @plusexpr | @minusexpr; + +@binaryexpr = @logicalbinaryexpr | @bitwisebinaryexpr | @arithmeticbinaryexpr | @comparison; + +@logicalbinaryexpr = @lorexpr | @landexpr; + +@bitwisebinaryexpr = @shiftexpr | @orexpr | @xorexpr | @andexpr | @andnotexpr; + +@arithmeticbinaryexpr = @addexpr | @subexpr | @mulexpr | @quoexpr | @remexpr; + +@shiftexpr = @shlexpr | @shrexpr; + +@comparison = @equalitytest | @relationalcomparison; + +@equalitytest = @eqlexpr | @neqexpr; + +@relationalcomparison = @lssexpr | @leqexpr | @gtrexpr | @geqexpr; + +@chantypeexpr = @sendchantypeexpr | @recvchantypeexpr | @sendrcvchantypeexpr; + +case @stmt.kind of + 0 = @badstmt +| 1 = @declstmt +| 2 = @emptystmt +| 3 = @labeledstmt +| 4 = @exprstmt +| 5 = @sendstmt +| 6 = @incstmt +| 7 = @decstmt +| 8 = @gostmt +| 9 = @deferstmt +| 10 = @returnstmt +| 11 = @breakstmt +| 12 = @continuestmt +| 13 = @gotostmt +| 14 = @fallthroughstmt +| 15 = @blockstmt +| 16 = @ifstmt +| 17 = @caseclause +| 18 = @exprswitchstmt +| 19 = @typeswitchstmt +| 20 = @commclause +| 21 = @selectstmt +| 22 = @forstmt +| 23 = @rangestmt +| 24 = @assignstmt +| 25 = @definestmt +| 26 = @addassignstmt +| 27 = @subassignstmt +| 28 = @mulassignstmt +| 29 = @quoassignstmt +| 30 = @remassignstmt +| 31 = @andassignstmt +| 32 = @orassignstmt +| 33 = @xorassignstmt +| 34 = @shlassignstmt +| 35 = @shrassignstmt +| 36 = @andnotassignstmt; + +@incdecstmt = @incstmt | @decstmt; + +@assignment = @simpleassignstmt | @compoundassignstmt; + +@simpleassignstmt = @assignstmt | @definestmt; + +@compoundassignstmt = @addassignstmt | @subassignstmt | @mulassignstmt | @quoassignstmt | @remassignstmt + | @andassignstmt | @orassignstmt | @xorassignstmt | @shlassignstmt | @shrassignstmt | @andnotassignstmt; + +@branchstmt = @breakstmt | @continuestmt | @gotostmt | @fallthroughstmt; + +@switchstmt = @exprswitchstmt | @typeswitchstmt; + +@loopstmt = @forstmt | @rangestmt; + +case @decl.kind of + 0 = @baddecl +| 1 = @importdecl +| 2 = @constdecl +| 3 = @typedecl +| 4 = @vardecl +| 5 = @funcdecl; + +@gendecl = @importdecl | @constdecl | @typedecl | @vardecl; + +case @spec.kind of + 0 = @importspec +| 1 = @valuespec +| 2 = @typedefspec +| 3 = @aliasspec; + +@typespec = @typedefspec | @aliasspec; + +case @object.kind of + 0 = @pkgobject +| 1 = @decltypeobject +| 2 = @builtintypeobject +| 3 = @declconstobject +| 4 = @builtinconstobject +| 5 = @declvarobject +| 6 = @declfunctionobject +| 7 = @builtinfunctionobject +| 8 = @labelobject; + +@typeparamparentobject = @decltypeobject | @declfunctionobject; + +@declobject = @decltypeobject | @declconstobject | @declvarobject | @declfunctionobject; + +@builtinobject = @builtintypeobject | @builtinconstobject | @builtinfunctionobject; + +@typeobject = @decltypeobject | @builtintypeobject; + +@valueobject = @constobject | @varobject | @functionobject; + +@constobject = @declconstobject | @builtinconstobject; + +@varobject = @declvarobject; + +@functionobject = @declfunctionobject | @builtinfunctionobject; + +case @scope.kind of + 0 = @universescope +| 1 = @packagescope +| 2 = @localscope; + +case @type.kind of + 0 = @invalidtype +| 1 = @boolexprtype +| 2 = @inttype +| 3 = @int8type +| 4 = @int16type +| 5 = @int32type +| 6 = @int64type +| 7 = @uinttype +| 8 = @uint8type +| 9 = @uint16type +| 10 = @uint32type +| 11 = @uint64type +| 12 = @uintptrtype +| 13 = @float32type +| 14 = @float64type +| 15 = @complex64type +| 16 = @complex128type +| 17 = @stringexprtype +| 18 = @unsafepointertype +| 19 = @boolliteraltype +| 20 = @intliteraltype +| 21 = @runeliteraltype +| 22 = @floatliteraltype +| 23 = @complexliteraltype +| 24 = @stringliteraltype +| 25 = @nilliteraltype +| 26 = @typeparamtype +| 27 = @arraytype +| 28 = @slicetype +| 29 = @structtype +| 30 = @pointertype +| 31 = @interfacetype +| 32 = @tupletype +| 33 = @signaturetype +| 34 = @maptype +| 35 = @sendchantype +| 36 = @recvchantype +| 37 = @sendrcvchantype +| 38 = @definedtype +| 39 = @typesetliteraltype; + +@basictype = @booltype | @numerictype | @stringtype | @literaltype | @invalidtype | @unsafepointertype; + +@booltype = @boolexprtype | @boolliteraltype; + +@numerictype = @integertype | @floattype | @complextype; + +@integertype = @signedintegertype | @unsignedintegertype; + +@signedintegertype = @inttype | @int8type | @int16type | @int32type | @int64type | @intliteraltype | @runeliteraltype; + +@unsignedintegertype = @uinttype | @uint8type | @uint16type | @uint32type | @uint64type | @uintptrtype; + +@floattype = @float32type | @float64type | @floatliteraltype; + +@complextype = @complex64type | @complex128type | @complexliteraltype; + +@stringtype = @stringexprtype | @stringliteraltype; + +@literaltype = @boolliteraltype | @intliteraltype | @runeliteraltype | @floatliteraltype | @complexliteraltype + | @stringliteraltype | @nilliteraltype; + +@compositetype = @typeparamtype | @containertype | @structtype | @pointertype | @interfacetype | @tupletype + | @signaturetype | @definedtype | @typesetliteraltype; + +@containertype = @arraytype | @slicetype | @maptype | @chantype; + +@chantype = @sendchantype | @recvchantype | @sendrcvchantype; + +case @modexpr.kind of + 0 = @modcommentblock +| 1 = @modline +| 2 = @modlineblock +| 3 = @modlparen +| 4 = @modrparen; + +case @error.kind of + 0 = @unknownerror +| 1 = @listerror +| 2 = @parseerror +| 3 = @typeerror; + diff --git a/go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/old.dbscheme b/go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/old.dbscheme new file mode 100644 index 00000000000..5ff5325d274 --- /dev/null +++ b/go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/old.dbscheme @@ -0,0 +1,563 @@ +/** Auto-generated dbscheme; do not edit. Run `make gen` in directory `go/` to regenerate. */ + + +/** Duplicate code **/ + +duplicateCode( + unique int id : @duplication, + varchar(900) relativePath : string ref, + int equivClass : int ref); + +similarCode( + unique int id : @similarity, + varchar(900) relativePath : string ref, + int equivClass : int ref); + +@duplication_or_similarity = @duplication | @similarity; + +tokens( + int id : @duplication_or_similarity ref, + int offset : int ref, + int beginLine : int ref, + int beginColumn : int ref, + int endLine : int ref, + int endColumn : int ref); + +/** External data **/ + +externalData( + int id : @externalDataElement, + varchar(900) path : string ref, + int column: int ref, + varchar(900) value : string ref +); + +snapshotDate(unique date snapshotDate : date ref); + +sourceLocationPrefix(varchar(900) prefix : string ref); + +/** Overlay support **/ + +databaseMetadata( + string metadataKey: string ref, + string value: string ref +); + +overlayChangedFiles( + string path: string ref +); + + +/* + * XML Files + */ + +xmlEncoding( + unique int id: @file ref, + string encoding: string ref +); + +xmlDTDs( + unique int id: @xmldtd, + string root: string ref, + string publicId: string ref, + string systemId: string ref, + int fileid: @file ref +); + +xmlElements( + unique int id: @xmlelement, + string name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref +); + +xmlAttrs( + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + string name: string ref, + string value: string ref, + int idx: int ref, + int fileid: @file ref +); + +xmlNs( + int id: @xmlnamespace, + string prefixName: string ref, + string URI: string ref, + int fileid: @file ref +); + +xmlHasNs( + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref +); + +xmlComments( + unique int id: @xmlcomment, + string text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref +); + +xmlChars( + unique int id: @xmlcharacters, + string text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref +); + +@xmlparent = @file | @xmlelement; +@xmlnamespaceable = @xmlelement | @xmlattribute; + +xmllocations( + int xmlElement: @xmllocatable ref, + int location: @location_default ref +); + +@xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace; + +compilations(unique int id: @compilation, string cwd: string ref); + +#keyset[id, num] +compilation_args(int id: @compilation ref, int num: int ref, string arg: string ref); + +#keyset[id, num, kind] +compilation_time(int id: @compilation ref, int num: int ref, int kind: int ref, float secs: float ref); + +diagnostic_for(unique int diagnostic: @diagnostic ref, int compilation: @compilation ref, int file_number: int ref, int file_number_diagnostic_number: int ref); + +compilation_finished(unique int id: @compilation ref, float cpu_seconds: float ref, float elapsed_seconds: float ref); + +#keyset[id, num] +compilation_compiling_files(int id: @compilation ref, int num: int ref, int file: @file ref); + +diagnostics(unique int id: @diagnostic, int severity: int ref, string error_tag: string ref, string error_message: string ref, + string full_error_message: string ref, int location: @location ref); + +locations_default(unique int id: @location_default, int file: @file ref, int beginLine: int ref, int beginColumn: int ref, + int endLine: int ref, int endColumn: int ref); + +numlines(int element_id: @sourceline ref, int num_lines: int ref, int num_code: int ref, int num_comment: int ref); + +files(unique int id: @file, string name: string ref); + +folders(unique int id: @folder, string name: string ref); + +containerparent(int parent: @container ref, unique int child: @container ref); + +has_location(unique int locatable: @locatable ref, int location: @location ref); + +#keyset[parent, idx] +comment_groups(unique int id: @comment_group, int parent: @file ref, int idx: int ref); + +comments(unique int id: @comment, int kind: int ref, int parent: @comment_group ref, int idx: int ref, string text: string ref); + +doc_comments(unique int node: @documentable ref, int comment: @comment_group ref); + +#keyset[parent, idx] +exprs(unique int id: @expr, int kind: int ref, int parent: @exprparent ref, int idx: int ref); + +literals(unique int expr: @expr ref, string value: string ref, string raw: string ref); + +constvalues(unique int expr: @expr ref, string value: string ref, string exact: string ref); + +fields(unique int id: @field, int parent: @fieldparent ref, int idx: int ref); + +typeparamdecls(unique int id: @typeparamdecl, int parent: @typeparamdeclparent ref, int idx: int ref); + +#keyset[parent, idx] +stmts(unique int id: @stmt, int kind: int ref, int parent: @stmtparent ref, int idx: int ref); + +#keyset[parent, idx] +decls(unique int id: @decl, int kind: int ref, int parent: @declparent ref, int idx: int ref); + +#keyset[parent, idx] +specs(unique int id: @spec, int kind: int ref, int parent: @gendecl ref, int idx: int ref); + +scopes(unique int id: @scope, int kind: int ref); + +scopenesting(unique int inner: @scope ref, int outer: @scope ref); + +scopenodes(unique int node: @scopenode ref, int scope: @localscope ref); + +objects(unique int id: @object, int kind: int ref, string name: string ref); + +objectscopes(unique int object: @object ref, int scope: @scope ref); + +objecttypes(unique int object: @object ref, int tp: @type ref); + +methodreceivers(unique int method: @object ref, int receiver: @object ref); + +fieldstructs(unique int field: @object ref, int struct: @structtype ref); + +methodhosts(int method: @object ref, int host: @definedtype ref); + +defs(int ident: @ident ref, int object: @object ref); + +uses(int ident: @ident ref, int object: @object ref); + +types(unique int id: @type, int kind: int ref); + +type_of(unique int expr: @expr ref, int tp: @type ref); + +typename(unique int tp: @type ref, string name: string ref); + +key_type(unique int map: @maptype ref, int tp: @type ref); + +element_type(unique int container: @containertype ref, int tp: @type ref); + +base_type(unique int ptr: @pointertype ref, int tp: @type ref); + +underlying_type(unique int defined: @definedtype ref, int tp: @type ref); + +#keyset[parent, index] +component_types(int parent: @compositetype ref, int index: int ref, string name: string ref, int tp: @type ref); + +#keyset[parent, index] +struct_tags(int parent: @structtype ref, int index: int ref, string tag: string ref); + +#keyset[interface, index] +interface_private_method_ids(int interface: @interfacetype ref, int index: int ref, string id: string ref); + +array_length(unique int tp: @arraytype ref, string len: string ref); + +type_objects(unique int tp: @type ref, int object: @object ref); + +packages(unique int id: @package, string name: string ref, string path: string ref, int scope: @packagescope ref); + +#keyset[parent, idx] +modexprs(unique int id: @modexpr, int kind: int ref, int parent: @modexprparent ref, int idx: int ref); + +#keyset[parent, idx] +modtokens(string token: string ref, int parent: @modexpr ref, int idx: int ref); + +#keyset[package, idx] +errors(unique int id: @error, int kind: int ref, string msg: string ref, string rawpos: string ref, + string file: string ref, int line: int ref, int col: int ref, int package: @package ref, int idx: int ref); + +has_ellipsis(int id: @callorconversionexpr ref); + +variadic(int id: @signaturetype ref); + +#keyset[parent, idx, is_from_recv] +typeparam(unique int tp: @typeparamtype ref, string name: string ref, + int bound: @compositetype ref, int parent: @typeparamparentobject ref, int idx: int ref, boolean is_from_recv: boolean ref); + +@container = @file | @folder; + +@locatable = @xmllocatable | @node | @localscope; + +@node = @documentable | @exprparent | @modexprparent | @fieldparent | @stmtparent | @declparent | @typeparamdeclparent + | @scopenode | @comment_group | @comment; + +@documentable = @file | @field | @typeparamdecl | @spec | @gendecl | @funcdecl | @modexpr; + +@exprparent = @funcdef | @file | @expr | @field | @stmt | @decl | @typeparamdecl | @spec; + +@modexprparent = @file | @modexpr; + +@fieldparent = @decl | @structtypeexpr | @functypeexpr | @interfacetypeexpr; + +@stmtparent = @funcdef | @stmt | @decl; + +@declparent = @file | @declstmt; + +@typeparamdeclparent = @funcdecl | @typespec; + +@funcdef = @funclit | @funcdecl; + +@scopenode = @file | @functypeexpr | @blockstmt | @ifstmt | @caseclause | @switchstmt | @commclause | @loopstmt; + +@location = @location_default; + +@sourceline = @locatable; + +case @comment.kind of + 0 = @slashslashcomment +| 1 = @slashstarcomment; + +case @expr.kind of + 0 = @badexpr +| 1 = @ident +| 2 = @ellipsis +| 3 = @intlit +| 4 = @floatlit +| 5 = @imaglit +| 6 = @charlit +| 7 = @stringlit +| 8 = @funclit +| 9 = @compositelit +| 10 = @parenexpr +| 11 = @selectorexpr +| 12 = @indexexpr +| 13 = @genericfunctioninstantiationexpr +| 14 = @generictypeinstantiationexpr +| 15 = @sliceexpr +| 16 = @typeassertexpr +| 17 = @callorconversionexpr +| 18 = @starexpr +| 19 = @keyvalueexpr +| 20 = @arraytypeexpr +| 21 = @structtypeexpr +| 22 = @functypeexpr +| 23 = @interfacetypeexpr +| 24 = @maptypeexpr +| 25 = @typesetliteralexpr +| 26 = @plusexpr +| 27 = @minusexpr +| 28 = @notexpr +| 29 = @complementexpr +| 30 = @derefexpr +| 31 = @addressexpr +| 32 = @arrowexpr +| 33 = @lorexpr +| 34 = @landexpr +| 35 = @eqlexpr +| 36 = @neqexpr +| 37 = @lssexpr +| 38 = @leqexpr +| 39 = @gtrexpr +| 40 = @geqexpr +| 41 = @addexpr +| 42 = @subexpr +| 43 = @orexpr +| 44 = @xorexpr +| 45 = @mulexpr +| 46 = @quoexpr +| 47 = @remexpr +| 48 = @shlexpr +| 49 = @shrexpr +| 50 = @andexpr +| 51 = @andnotexpr +| 52 = @sendchantypeexpr +| 53 = @recvchantypeexpr +| 54 = @sendrcvchantypeexpr; + +@basiclit = @intlit | @floatlit | @imaglit | @charlit | @stringlit; + +@operatorexpr = @logicalexpr | @arithmeticexpr | @bitwiseexpr | @unaryexpr | @binaryexpr; + +@logicalexpr = @logicalunaryexpr | @logicalbinaryexpr; + +@arithmeticexpr = @arithmeticunaryexpr | @arithmeticbinaryexpr; + +@bitwiseexpr = @bitwiseunaryexpr | @bitwisebinaryexpr; + +@unaryexpr = @logicalunaryexpr | @bitwiseunaryexpr | @arithmeticunaryexpr | @derefexpr | @addressexpr | @arrowexpr; + +@logicalunaryexpr = @notexpr; + +@bitwiseunaryexpr = @complementexpr; + +@arithmeticunaryexpr = @plusexpr | @minusexpr; + +@binaryexpr = @logicalbinaryexpr | @bitwisebinaryexpr | @arithmeticbinaryexpr | @comparison; + +@logicalbinaryexpr = @lorexpr | @landexpr; + +@bitwisebinaryexpr = @shiftexpr | @orexpr | @xorexpr | @andexpr | @andnotexpr; + +@arithmeticbinaryexpr = @addexpr | @subexpr | @mulexpr | @quoexpr | @remexpr; + +@shiftexpr = @shlexpr | @shrexpr; + +@comparison = @equalitytest | @relationalcomparison; + +@equalitytest = @eqlexpr | @neqexpr; + +@relationalcomparison = @lssexpr | @leqexpr | @gtrexpr | @geqexpr; + +@chantypeexpr = @sendchantypeexpr | @recvchantypeexpr | @sendrcvchantypeexpr; + +case @stmt.kind of + 0 = @badstmt +| 1 = @declstmt +| 2 = @emptystmt +| 3 = @labeledstmt +| 4 = @exprstmt +| 5 = @sendstmt +| 6 = @incstmt +| 7 = @decstmt +| 8 = @gostmt +| 9 = @deferstmt +| 10 = @returnstmt +| 11 = @breakstmt +| 12 = @continuestmt +| 13 = @gotostmt +| 14 = @fallthroughstmt +| 15 = @blockstmt +| 16 = @ifstmt +| 17 = @caseclause +| 18 = @exprswitchstmt +| 19 = @typeswitchstmt +| 20 = @commclause +| 21 = @selectstmt +| 22 = @forstmt +| 23 = @rangestmt +| 24 = @assignstmt +| 25 = @definestmt +| 26 = @addassignstmt +| 27 = @subassignstmt +| 28 = @mulassignstmt +| 29 = @quoassignstmt +| 30 = @remassignstmt +| 31 = @andassignstmt +| 32 = @orassignstmt +| 33 = @xorassignstmt +| 34 = @shlassignstmt +| 35 = @shrassignstmt +| 36 = @andnotassignstmt; + +@incdecstmt = @incstmt | @decstmt; + +@assignment = @simpleassignstmt | @compoundassignstmt; + +@simpleassignstmt = @assignstmt | @definestmt; + +@compoundassignstmt = @addassignstmt | @subassignstmt | @mulassignstmt | @quoassignstmt | @remassignstmt + | @andassignstmt | @orassignstmt | @xorassignstmt | @shlassignstmt | @shrassignstmt | @andnotassignstmt; + +@branchstmt = @breakstmt | @continuestmt | @gotostmt | @fallthroughstmt; + +@switchstmt = @exprswitchstmt | @typeswitchstmt; + +@loopstmt = @forstmt | @rangestmt; + +case @decl.kind of + 0 = @baddecl +| 1 = @importdecl +| 2 = @constdecl +| 3 = @typedecl +| 4 = @vardecl +| 5 = @funcdecl; + +@gendecl = @importdecl | @constdecl | @typedecl | @vardecl; + +case @spec.kind of + 0 = @importspec +| 1 = @valuespec +| 2 = @typedefspec +| 3 = @aliasspec; + +@typespec = @typedefspec | @aliasspec; + +case @object.kind of + 0 = @pkgobject +| 1 = @decltypeobject +| 2 = @builtintypeobject +| 3 = @declconstobject +| 4 = @builtinconstobject +| 5 = @declvarobject +| 6 = @declfunctionobject +| 7 = @builtinfunctionobject +| 8 = @labelobject; + +@typeparamparentobject = @decltypeobject | @declfunctionobject; + +@declobject = @decltypeobject | @declconstobject | @declvarobject | @declfunctionobject; + +@builtinobject = @builtintypeobject | @builtinconstobject | @builtinfunctionobject; + +@typeobject = @decltypeobject | @builtintypeobject; + +@valueobject = @constobject | @varobject | @functionobject; + +@constobject = @declconstobject | @builtinconstobject; + +@varobject = @declvarobject; + +@functionobject = @declfunctionobject | @builtinfunctionobject; + +case @scope.kind of + 0 = @universescope +| 1 = @packagescope +| 2 = @localscope; + +case @type.kind of + 0 = @invalidtype +| 1 = @boolexprtype +| 2 = @inttype +| 3 = @int8type +| 4 = @int16type +| 5 = @int32type +| 6 = @int64type +| 7 = @uinttype +| 8 = @uint8type +| 9 = @uint16type +| 10 = @uint32type +| 11 = @uint64type +| 12 = @uintptrtype +| 13 = @float32type +| 14 = @float64type +| 15 = @complex64type +| 16 = @complex128type +| 17 = @stringexprtype +| 18 = @unsafepointertype +| 19 = @boolliteraltype +| 20 = @intliteraltype +| 21 = @runeliteraltype +| 22 = @floatliteraltype +| 23 = @complexliteraltype +| 24 = @stringliteraltype +| 25 = @nilliteraltype +| 26 = @typeparamtype +| 27 = @arraytype +| 28 = @slicetype +| 29 = @structtype +| 30 = @pointertype +| 31 = @interfacetype +| 32 = @tupletype +| 33 = @signaturetype +| 34 = @maptype +| 35 = @sendchantype +| 36 = @recvchantype +| 37 = @sendrcvchantype +| 38 = @definedtype +| 39 = @typesetliteraltype; + +@basictype = @booltype | @numerictype | @stringtype | @literaltype | @invalidtype | @unsafepointertype; + +@booltype = @boolexprtype | @boolliteraltype; + +@numerictype = @integertype | @floattype | @complextype; + +@integertype = @signedintegertype | @unsignedintegertype; + +@signedintegertype = @inttype | @int8type | @int16type | @int32type | @int64type | @intliteraltype | @runeliteraltype; + +@unsignedintegertype = @uinttype | @uint8type | @uint16type | @uint32type | @uint64type | @uintptrtype; + +@floattype = @float32type | @float64type | @floatliteraltype; + +@complextype = @complex64type | @complex128type | @complexliteraltype; + +@stringtype = @stringexprtype | @stringliteraltype; + +@literaltype = @boolliteraltype | @intliteraltype | @runeliteraltype | @floatliteraltype | @complexliteraltype + | @stringliteraltype | @nilliteraltype; + +@compositetype = @typeparamtype | @containertype | @structtype | @pointertype | @interfacetype | @tupletype + | @signaturetype | @definedtype | @typesetliteraltype; + +@containertype = @arraytype | @slicetype | @maptype | @chantype; + +@chantype = @sendchantype | @recvchantype | @sendrcvchantype; + +case @modexpr.kind of + 0 = @modcommentblock +| 1 = @modline +| 2 = @modlineblock +| 3 = @modlparen +| 4 = @modrparen; + +case @error.kind of + 0 = @unknownerror +| 1 = @listerror +| 2 = @parseerror +| 3 = @typeerror; + diff --git a/go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/typeparam.ql b/go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/typeparam.ql new file mode 100644 index 00000000000..00c17208e6e --- /dev/null +++ b/go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/typeparam.ql @@ -0,0 +1,15 @@ +class TypeParamType extends @typeparamtype { + string toString() { none() } +} + +class CompositeType extends @compositetype { + string toString() { none() } +} + +class TypeParamParentObject extends @typeparamparentobject { + string toString() { none() } +} + +from TypeParamType tp, string name, CompositeType bound, TypeParamParentObject parent, int idx, boolean is_from_recv +where typeparam(tp, name, bound, parent, idx, is_from_recv) +select tp, name, bound, parent, idx diff --git a/go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/upgrade.properties b/go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/upgrade.properties new file mode 100644 index 00000000000..d383bdf4ad3 --- /dev/null +++ b/go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/upgrade.properties @@ -0,0 +1,3 @@ +description: Track whether a type parameter is from a receiver +compatibility: partial +typeparam.rel: run typeparam.qlo diff --git a/go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/go.dbscheme b/go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/go.dbscheme new file mode 100644 index 00000000000..5ff5325d274 --- /dev/null +++ b/go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/go.dbscheme @@ -0,0 +1,563 @@ +/** Auto-generated dbscheme; do not edit. Run `make gen` in directory `go/` to regenerate. */ + + +/** Duplicate code **/ + +duplicateCode( + unique int id : @duplication, + varchar(900) relativePath : string ref, + int equivClass : int ref); + +similarCode( + unique int id : @similarity, + varchar(900) relativePath : string ref, + int equivClass : int ref); + +@duplication_or_similarity = @duplication | @similarity; + +tokens( + int id : @duplication_or_similarity ref, + int offset : int ref, + int beginLine : int ref, + int beginColumn : int ref, + int endLine : int ref, + int endColumn : int ref); + +/** External data **/ + +externalData( + int id : @externalDataElement, + varchar(900) path : string ref, + int column: int ref, + varchar(900) value : string ref +); + +snapshotDate(unique date snapshotDate : date ref); + +sourceLocationPrefix(varchar(900) prefix : string ref); + +/** Overlay support **/ + +databaseMetadata( + string metadataKey: string ref, + string value: string ref +); + +overlayChangedFiles( + string path: string ref +); + + +/* + * XML Files + */ + +xmlEncoding( + unique int id: @file ref, + string encoding: string ref +); + +xmlDTDs( + unique int id: @xmldtd, + string root: string ref, + string publicId: string ref, + string systemId: string ref, + int fileid: @file ref +); + +xmlElements( + unique int id: @xmlelement, + string name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref +); + +xmlAttrs( + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + string name: string ref, + string value: string ref, + int idx: int ref, + int fileid: @file ref +); + +xmlNs( + int id: @xmlnamespace, + string prefixName: string ref, + string URI: string ref, + int fileid: @file ref +); + +xmlHasNs( + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref +); + +xmlComments( + unique int id: @xmlcomment, + string text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref +); + +xmlChars( + unique int id: @xmlcharacters, + string text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref +); + +@xmlparent = @file | @xmlelement; +@xmlnamespaceable = @xmlelement | @xmlattribute; + +xmllocations( + int xmlElement: @xmllocatable ref, + int location: @location_default ref +); + +@xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace; + +compilations(unique int id: @compilation, string cwd: string ref); + +#keyset[id, num] +compilation_args(int id: @compilation ref, int num: int ref, string arg: string ref); + +#keyset[id, num, kind] +compilation_time(int id: @compilation ref, int num: int ref, int kind: int ref, float secs: float ref); + +diagnostic_for(unique int diagnostic: @diagnostic ref, int compilation: @compilation ref, int file_number: int ref, int file_number_diagnostic_number: int ref); + +compilation_finished(unique int id: @compilation ref, float cpu_seconds: float ref, float elapsed_seconds: float ref); + +#keyset[id, num] +compilation_compiling_files(int id: @compilation ref, int num: int ref, int file: @file ref); + +diagnostics(unique int id: @diagnostic, int severity: int ref, string error_tag: string ref, string error_message: string ref, + string full_error_message: string ref, int location: @location ref); + +locations_default(unique int id: @location_default, int file: @file ref, int beginLine: int ref, int beginColumn: int ref, + int endLine: int ref, int endColumn: int ref); + +numlines(int element_id: @sourceline ref, int num_lines: int ref, int num_code: int ref, int num_comment: int ref); + +files(unique int id: @file, string name: string ref); + +folders(unique int id: @folder, string name: string ref); + +containerparent(int parent: @container ref, unique int child: @container ref); + +has_location(unique int locatable: @locatable ref, int location: @location ref); + +#keyset[parent, idx] +comment_groups(unique int id: @comment_group, int parent: @file ref, int idx: int ref); + +comments(unique int id: @comment, int kind: int ref, int parent: @comment_group ref, int idx: int ref, string text: string ref); + +doc_comments(unique int node: @documentable ref, int comment: @comment_group ref); + +#keyset[parent, idx] +exprs(unique int id: @expr, int kind: int ref, int parent: @exprparent ref, int idx: int ref); + +literals(unique int expr: @expr ref, string value: string ref, string raw: string ref); + +constvalues(unique int expr: @expr ref, string value: string ref, string exact: string ref); + +fields(unique int id: @field, int parent: @fieldparent ref, int idx: int ref); + +typeparamdecls(unique int id: @typeparamdecl, int parent: @typeparamdeclparent ref, int idx: int ref); + +#keyset[parent, idx] +stmts(unique int id: @stmt, int kind: int ref, int parent: @stmtparent ref, int idx: int ref); + +#keyset[parent, idx] +decls(unique int id: @decl, int kind: int ref, int parent: @declparent ref, int idx: int ref); + +#keyset[parent, idx] +specs(unique int id: @spec, int kind: int ref, int parent: @gendecl ref, int idx: int ref); + +scopes(unique int id: @scope, int kind: int ref); + +scopenesting(unique int inner: @scope ref, int outer: @scope ref); + +scopenodes(unique int node: @scopenode ref, int scope: @localscope ref); + +objects(unique int id: @object, int kind: int ref, string name: string ref); + +objectscopes(unique int object: @object ref, int scope: @scope ref); + +objecttypes(unique int object: @object ref, int tp: @type ref); + +methodreceivers(unique int method: @object ref, int receiver: @object ref); + +fieldstructs(unique int field: @object ref, int struct: @structtype ref); + +methodhosts(int method: @object ref, int host: @definedtype ref); + +defs(int ident: @ident ref, int object: @object ref); + +uses(int ident: @ident ref, int object: @object ref); + +types(unique int id: @type, int kind: int ref); + +type_of(unique int expr: @expr ref, int tp: @type ref); + +typename(unique int tp: @type ref, string name: string ref); + +key_type(unique int map: @maptype ref, int tp: @type ref); + +element_type(unique int container: @containertype ref, int tp: @type ref); + +base_type(unique int ptr: @pointertype ref, int tp: @type ref); + +underlying_type(unique int defined: @definedtype ref, int tp: @type ref); + +#keyset[parent, index] +component_types(int parent: @compositetype ref, int index: int ref, string name: string ref, int tp: @type ref); + +#keyset[parent, index] +struct_tags(int parent: @structtype ref, int index: int ref, string tag: string ref); + +#keyset[interface, index] +interface_private_method_ids(int interface: @interfacetype ref, int index: int ref, string id: string ref); + +array_length(unique int tp: @arraytype ref, string len: string ref); + +type_objects(unique int tp: @type ref, int object: @object ref); + +packages(unique int id: @package, string name: string ref, string path: string ref, int scope: @packagescope ref); + +#keyset[parent, idx] +modexprs(unique int id: @modexpr, int kind: int ref, int parent: @modexprparent ref, int idx: int ref); + +#keyset[parent, idx] +modtokens(string token: string ref, int parent: @modexpr ref, int idx: int ref); + +#keyset[package, idx] +errors(unique int id: @error, int kind: int ref, string msg: string ref, string rawpos: string ref, + string file: string ref, int line: int ref, int col: int ref, int package: @package ref, int idx: int ref); + +has_ellipsis(int id: @callorconversionexpr ref); + +variadic(int id: @signaturetype ref); + +#keyset[parent, idx, is_from_recv] +typeparam(unique int tp: @typeparamtype ref, string name: string ref, + int bound: @compositetype ref, int parent: @typeparamparentobject ref, int idx: int ref, boolean is_from_recv: boolean ref); + +@container = @file | @folder; + +@locatable = @xmllocatable | @node | @localscope; + +@node = @documentable | @exprparent | @modexprparent | @fieldparent | @stmtparent | @declparent | @typeparamdeclparent + | @scopenode | @comment_group | @comment; + +@documentable = @file | @field | @typeparamdecl | @spec | @gendecl | @funcdecl | @modexpr; + +@exprparent = @funcdef | @file | @expr | @field | @stmt | @decl | @typeparamdecl | @spec; + +@modexprparent = @file | @modexpr; + +@fieldparent = @decl | @structtypeexpr | @functypeexpr | @interfacetypeexpr; + +@stmtparent = @funcdef | @stmt | @decl; + +@declparent = @file | @declstmt; + +@typeparamdeclparent = @funcdecl | @typespec; + +@funcdef = @funclit | @funcdecl; + +@scopenode = @file | @functypeexpr | @blockstmt | @ifstmt | @caseclause | @switchstmt | @commclause | @loopstmt; + +@location = @location_default; + +@sourceline = @locatable; + +case @comment.kind of + 0 = @slashslashcomment +| 1 = @slashstarcomment; + +case @expr.kind of + 0 = @badexpr +| 1 = @ident +| 2 = @ellipsis +| 3 = @intlit +| 4 = @floatlit +| 5 = @imaglit +| 6 = @charlit +| 7 = @stringlit +| 8 = @funclit +| 9 = @compositelit +| 10 = @parenexpr +| 11 = @selectorexpr +| 12 = @indexexpr +| 13 = @genericfunctioninstantiationexpr +| 14 = @generictypeinstantiationexpr +| 15 = @sliceexpr +| 16 = @typeassertexpr +| 17 = @callorconversionexpr +| 18 = @starexpr +| 19 = @keyvalueexpr +| 20 = @arraytypeexpr +| 21 = @structtypeexpr +| 22 = @functypeexpr +| 23 = @interfacetypeexpr +| 24 = @maptypeexpr +| 25 = @typesetliteralexpr +| 26 = @plusexpr +| 27 = @minusexpr +| 28 = @notexpr +| 29 = @complementexpr +| 30 = @derefexpr +| 31 = @addressexpr +| 32 = @arrowexpr +| 33 = @lorexpr +| 34 = @landexpr +| 35 = @eqlexpr +| 36 = @neqexpr +| 37 = @lssexpr +| 38 = @leqexpr +| 39 = @gtrexpr +| 40 = @geqexpr +| 41 = @addexpr +| 42 = @subexpr +| 43 = @orexpr +| 44 = @xorexpr +| 45 = @mulexpr +| 46 = @quoexpr +| 47 = @remexpr +| 48 = @shlexpr +| 49 = @shrexpr +| 50 = @andexpr +| 51 = @andnotexpr +| 52 = @sendchantypeexpr +| 53 = @recvchantypeexpr +| 54 = @sendrcvchantypeexpr; + +@basiclit = @intlit | @floatlit | @imaglit | @charlit | @stringlit; + +@operatorexpr = @logicalexpr | @arithmeticexpr | @bitwiseexpr | @unaryexpr | @binaryexpr; + +@logicalexpr = @logicalunaryexpr | @logicalbinaryexpr; + +@arithmeticexpr = @arithmeticunaryexpr | @arithmeticbinaryexpr; + +@bitwiseexpr = @bitwiseunaryexpr | @bitwisebinaryexpr; + +@unaryexpr = @logicalunaryexpr | @bitwiseunaryexpr | @arithmeticunaryexpr | @derefexpr | @addressexpr | @arrowexpr; + +@logicalunaryexpr = @notexpr; + +@bitwiseunaryexpr = @complementexpr; + +@arithmeticunaryexpr = @plusexpr | @minusexpr; + +@binaryexpr = @logicalbinaryexpr | @bitwisebinaryexpr | @arithmeticbinaryexpr | @comparison; + +@logicalbinaryexpr = @lorexpr | @landexpr; + +@bitwisebinaryexpr = @shiftexpr | @orexpr | @xorexpr | @andexpr | @andnotexpr; + +@arithmeticbinaryexpr = @addexpr | @subexpr | @mulexpr | @quoexpr | @remexpr; + +@shiftexpr = @shlexpr | @shrexpr; + +@comparison = @equalitytest | @relationalcomparison; + +@equalitytest = @eqlexpr | @neqexpr; + +@relationalcomparison = @lssexpr | @leqexpr | @gtrexpr | @geqexpr; + +@chantypeexpr = @sendchantypeexpr | @recvchantypeexpr | @sendrcvchantypeexpr; + +case @stmt.kind of + 0 = @badstmt +| 1 = @declstmt +| 2 = @emptystmt +| 3 = @labeledstmt +| 4 = @exprstmt +| 5 = @sendstmt +| 6 = @incstmt +| 7 = @decstmt +| 8 = @gostmt +| 9 = @deferstmt +| 10 = @returnstmt +| 11 = @breakstmt +| 12 = @continuestmt +| 13 = @gotostmt +| 14 = @fallthroughstmt +| 15 = @blockstmt +| 16 = @ifstmt +| 17 = @caseclause +| 18 = @exprswitchstmt +| 19 = @typeswitchstmt +| 20 = @commclause +| 21 = @selectstmt +| 22 = @forstmt +| 23 = @rangestmt +| 24 = @assignstmt +| 25 = @definestmt +| 26 = @addassignstmt +| 27 = @subassignstmt +| 28 = @mulassignstmt +| 29 = @quoassignstmt +| 30 = @remassignstmt +| 31 = @andassignstmt +| 32 = @orassignstmt +| 33 = @xorassignstmt +| 34 = @shlassignstmt +| 35 = @shrassignstmt +| 36 = @andnotassignstmt; + +@incdecstmt = @incstmt | @decstmt; + +@assignment = @simpleassignstmt | @compoundassignstmt; + +@simpleassignstmt = @assignstmt | @definestmt; + +@compoundassignstmt = @addassignstmt | @subassignstmt | @mulassignstmt | @quoassignstmt | @remassignstmt + | @andassignstmt | @orassignstmt | @xorassignstmt | @shlassignstmt | @shrassignstmt | @andnotassignstmt; + +@branchstmt = @breakstmt | @continuestmt | @gotostmt | @fallthroughstmt; + +@switchstmt = @exprswitchstmt | @typeswitchstmt; + +@loopstmt = @forstmt | @rangestmt; + +case @decl.kind of + 0 = @baddecl +| 1 = @importdecl +| 2 = @constdecl +| 3 = @typedecl +| 4 = @vardecl +| 5 = @funcdecl; + +@gendecl = @importdecl | @constdecl | @typedecl | @vardecl; + +case @spec.kind of + 0 = @importspec +| 1 = @valuespec +| 2 = @typedefspec +| 3 = @aliasspec; + +@typespec = @typedefspec | @aliasspec; + +case @object.kind of + 0 = @pkgobject +| 1 = @decltypeobject +| 2 = @builtintypeobject +| 3 = @declconstobject +| 4 = @builtinconstobject +| 5 = @declvarobject +| 6 = @declfunctionobject +| 7 = @builtinfunctionobject +| 8 = @labelobject; + +@typeparamparentobject = @decltypeobject | @declfunctionobject; + +@declobject = @decltypeobject | @declconstobject | @declvarobject | @declfunctionobject; + +@builtinobject = @builtintypeobject | @builtinconstobject | @builtinfunctionobject; + +@typeobject = @decltypeobject | @builtintypeobject; + +@valueobject = @constobject | @varobject | @functionobject; + +@constobject = @declconstobject | @builtinconstobject; + +@varobject = @declvarobject; + +@functionobject = @declfunctionobject | @builtinfunctionobject; + +case @scope.kind of + 0 = @universescope +| 1 = @packagescope +| 2 = @localscope; + +case @type.kind of + 0 = @invalidtype +| 1 = @boolexprtype +| 2 = @inttype +| 3 = @int8type +| 4 = @int16type +| 5 = @int32type +| 6 = @int64type +| 7 = @uinttype +| 8 = @uint8type +| 9 = @uint16type +| 10 = @uint32type +| 11 = @uint64type +| 12 = @uintptrtype +| 13 = @float32type +| 14 = @float64type +| 15 = @complex64type +| 16 = @complex128type +| 17 = @stringexprtype +| 18 = @unsafepointertype +| 19 = @boolliteraltype +| 20 = @intliteraltype +| 21 = @runeliteraltype +| 22 = @floatliteraltype +| 23 = @complexliteraltype +| 24 = @stringliteraltype +| 25 = @nilliteraltype +| 26 = @typeparamtype +| 27 = @arraytype +| 28 = @slicetype +| 29 = @structtype +| 30 = @pointertype +| 31 = @interfacetype +| 32 = @tupletype +| 33 = @signaturetype +| 34 = @maptype +| 35 = @sendchantype +| 36 = @recvchantype +| 37 = @sendrcvchantype +| 38 = @definedtype +| 39 = @typesetliteraltype; + +@basictype = @booltype | @numerictype | @stringtype | @literaltype | @invalidtype | @unsafepointertype; + +@booltype = @boolexprtype | @boolliteraltype; + +@numerictype = @integertype | @floattype | @complextype; + +@integertype = @signedintegertype | @unsignedintegertype; + +@signedintegertype = @inttype | @int8type | @int16type | @int32type | @int64type | @intliteraltype | @runeliteraltype; + +@unsignedintegertype = @uinttype | @uint8type | @uint16type | @uint32type | @uint64type | @uintptrtype; + +@floattype = @float32type | @float64type | @floatliteraltype; + +@complextype = @complex64type | @complex128type | @complexliteraltype; + +@stringtype = @stringexprtype | @stringliteraltype; + +@literaltype = @boolliteraltype | @intliteraltype | @runeliteraltype | @floatliteraltype | @complexliteraltype + | @stringliteraltype | @nilliteraltype; + +@compositetype = @typeparamtype | @containertype | @structtype | @pointertype | @interfacetype | @tupletype + | @signaturetype | @definedtype | @typesetliteraltype; + +@containertype = @arraytype | @slicetype | @maptype | @chantype; + +@chantype = @sendchantype | @recvchantype | @sendrcvchantype; + +case @modexpr.kind of + 0 = @modcommentblock +| 1 = @modline +| 2 = @modlineblock +| 3 = @modlparen +| 4 = @modrparen; + +case @error.kind of + 0 = @unknownerror +| 1 = @listerror +| 2 = @parseerror +| 3 = @typeerror; + diff --git a/go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/old.dbscheme b/go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/old.dbscheme new file mode 100644 index 00000000000..b1341734d68 --- /dev/null +++ b/go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/old.dbscheme @@ -0,0 +1,563 @@ +/** Auto-generated dbscheme; do not edit. Run `make gen` in directory `go/` to regenerate. */ + + +/** Duplicate code **/ + +duplicateCode( + unique int id : @duplication, + varchar(900) relativePath : string ref, + int equivClass : int ref); + +similarCode( + unique int id : @similarity, + varchar(900) relativePath : string ref, + int equivClass : int ref); + +@duplication_or_similarity = @duplication | @similarity; + +tokens( + int id : @duplication_or_similarity ref, + int offset : int ref, + int beginLine : int ref, + int beginColumn : int ref, + int endLine : int ref, + int endColumn : int ref); + +/** External data **/ + +externalData( + int id : @externalDataElement, + varchar(900) path : string ref, + int column: int ref, + varchar(900) value : string ref +); + +snapshotDate(unique date snapshotDate : date ref); + +sourceLocationPrefix(varchar(900) prefix : string ref); + +/** Overlay support **/ + +databaseMetadata( + string metadataKey: string ref, + string value: string ref +); + +overlayChangedFiles( + string path: string ref +); + + +/* + * XML Files + */ + +xmlEncoding( + unique int id: @file ref, + string encoding: string ref +); + +xmlDTDs( + unique int id: @xmldtd, + string root: string ref, + string publicId: string ref, + string systemId: string ref, + int fileid: @file ref +); + +xmlElements( + unique int id: @xmlelement, + string name: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int fileid: @file ref +); + +xmlAttrs( + unique int id: @xmlattribute, + int elementid: @xmlelement ref, + string name: string ref, + string value: string ref, + int idx: int ref, + int fileid: @file ref +); + +xmlNs( + int id: @xmlnamespace, + string prefixName: string ref, + string URI: string ref, + int fileid: @file ref +); + +xmlHasNs( + int elementId: @xmlnamespaceable ref, + int nsId: @xmlnamespace ref, + int fileid: @file ref +); + +xmlComments( + unique int id: @xmlcomment, + string text: string ref, + int parentid: @xmlparent ref, + int fileid: @file ref +); + +xmlChars( + unique int id: @xmlcharacters, + string text: string ref, + int parentid: @xmlparent ref, + int idx: int ref, + int isCDATA: int ref, + int fileid: @file ref +); + +@xmlparent = @file | @xmlelement; +@xmlnamespaceable = @xmlelement | @xmlattribute; + +xmllocations( + int xmlElement: @xmllocatable ref, + int location: @location_default ref +); + +@xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace; + +compilations(unique int id: @compilation, string cwd: string ref); + +#keyset[id, num] +compilation_args(int id: @compilation ref, int num: int ref, string arg: string ref); + +#keyset[id, num, kind] +compilation_time(int id: @compilation ref, int num: int ref, int kind: int ref, float secs: float ref); + +diagnostic_for(unique int diagnostic: @diagnostic ref, int compilation: @compilation ref, int file_number: int ref, int file_number_diagnostic_number: int ref); + +compilation_finished(unique int id: @compilation ref, float cpu_seconds: float ref, float elapsed_seconds: float ref); + +#keyset[id, num] +compilation_compiling_files(int id: @compilation ref, int num: int ref, int file: @file ref); + +diagnostics(unique int id: @diagnostic, int severity: int ref, string error_tag: string ref, string error_message: string ref, + string full_error_message: string ref, int location: @location ref); + +locations_default(unique int id: @location_default, int file: @file ref, int beginLine: int ref, int beginColumn: int ref, + int endLine: int ref, int endColumn: int ref); + +numlines(int element_id: @sourceline ref, int num_lines: int ref, int num_code: int ref, int num_comment: int ref); + +files(unique int id: @file, string name: string ref); + +folders(unique int id: @folder, string name: string ref); + +containerparent(int parent: @container ref, unique int child: @container ref); + +has_location(unique int locatable: @locatable ref, int location: @location ref); + +#keyset[parent, idx] +comment_groups(unique int id: @comment_group, int parent: @file ref, int idx: int ref); + +comments(unique int id: @comment, int kind: int ref, int parent: @comment_group ref, int idx: int ref, string text: string ref); + +doc_comments(unique int node: @documentable ref, int comment: @comment_group ref); + +#keyset[parent, idx] +exprs(unique int id: @expr, int kind: int ref, int parent: @exprparent ref, int idx: int ref); + +literals(unique int expr: @expr ref, string value: string ref, string raw: string ref); + +constvalues(unique int expr: @expr ref, string value: string ref, string exact: string ref); + +fields(unique int id: @field, int parent: @fieldparent ref, int idx: int ref); + +typeparamdecls(unique int id: @typeparamdecl, int parent: @typeparamdeclparent ref, int idx: int ref); + +#keyset[parent, idx] +stmts(unique int id: @stmt, int kind: int ref, int parent: @stmtparent ref, int idx: int ref); + +#keyset[parent, idx] +decls(unique int id: @decl, int kind: int ref, int parent: @declparent ref, int idx: int ref); + +#keyset[parent, idx] +specs(unique int id: @spec, int kind: int ref, int parent: @gendecl ref, int idx: int ref); + +scopes(unique int id: @scope, int kind: int ref); + +scopenesting(unique int inner: @scope ref, int outer: @scope ref); + +scopenodes(unique int node: @scopenode ref, int scope: @localscope ref); + +objects(unique int id: @object, int kind: int ref, string name: string ref); + +objectscopes(unique int object: @object ref, int scope: @scope ref); + +objecttypes(unique int object: @object ref, int tp: @type ref); + +methodreceivers(unique int method: @object ref, int receiver: @object ref); + +fieldstructs(unique int field: @object ref, int struct: @structtype ref); + +methodhosts(int method: @object ref, int host: @definedtype ref); + +defs(int ident: @ident ref, int object: @object ref); + +uses(int ident: @ident ref, int object: @object ref); + +types(unique int id: @type, int kind: int ref); + +type_of(unique int expr: @expr ref, int tp: @type ref); + +typename(unique int tp: @type ref, string name: string ref); + +key_type(unique int map: @maptype ref, int tp: @type ref); + +element_type(unique int container: @containertype ref, int tp: @type ref); + +base_type(unique int ptr: @pointertype ref, int tp: @type ref); + +underlying_type(unique int defined: @definedtype ref, int tp: @type ref); + +#keyset[parent, index] +component_types(int parent: @compositetype ref, int index: int ref, string name: string ref, int tp: @type ref); + +#keyset[parent, index] +struct_tags(int parent: @structtype ref, int index: int ref, string tag: string ref); + +#keyset[interface, index] +interface_private_method_ids(int interface: @interfacetype ref, int index: int ref, string id: string ref); + +array_length(unique int tp: @arraytype ref, string len: string ref); + +type_objects(unique int tp: @type ref, int object: @object ref); + +packages(unique int id: @package, string name: string ref, string path: string ref, int scope: @packagescope ref); + +#keyset[parent, idx] +modexprs(unique int id: @modexpr, int kind: int ref, int parent: @modexprparent ref, int idx: int ref); + +#keyset[parent, idx] +modtokens(string token: string ref, int parent: @modexpr ref, int idx: int ref); + +#keyset[package, idx] +errors(unique int id: @error, int kind: int ref, string msg: string ref, string rawpos: string ref, + string file: string ref, int line: int ref, int col: int ref, int package: @package ref, int idx: int ref); + +has_ellipsis(int id: @callorconversionexpr ref); + +variadic(int id: @signaturetype ref); + +#keyset[parent, idx] +typeparam(unique int tp: @typeparamtype ref, string name: string ref, int bound: @compositetype ref, + int parent: @typeparamparentobject ref, int idx: int ref); + +@container = @file | @folder; + +@locatable = @xmllocatable | @node | @localscope; + +@node = @documentable | @exprparent | @modexprparent | @fieldparent | @stmtparent | @declparent | @typeparamdeclparent + | @scopenode | @comment_group | @comment; + +@documentable = @file | @field | @typeparamdecl | @spec | @gendecl | @funcdecl | @modexpr; + +@exprparent = @funcdef | @file | @expr | @field | @stmt | @decl | @typeparamdecl | @spec; + +@modexprparent = @file | @modexpr; + +@fieldparent = @decl | @structtypeexpr | @functypeexpr | @interfacetypeexpr; + +@stmtparent = @funcdef | @stmt | @decl; + +@declparent = @file | @declstmt; + +@typeparamdeclparent = @funcdecl | @typespec; + +@funcdef = @funclit | @funcdecl; + +@scopenode = @file | @functypeexpr | @blockstmt | @ifstmt | @caseclause | @switchstmt | @commclause | @loopstmt; + +@location = @location_default; + +@sourceline = @locatable; + +case @comment.kind of + 0 = @slashslashcomment +| 1 = @slashstarcomment; + +case @expr.kind of + 0 = @badexpr +| 1 = @ident +| 2 = @ellipsis +| 3 = @intlit +| 4 = @floatlit +| 5 = @imaglit +| 6 = @charlit +| 7 = @stringlit +| 8 = @funclit +| 9 = @compositelit +| 10 = @parenexpr +| 11 = @selectorexpr +| 12 = @indexexpr +| 13 = @genericfunctioninstantiationexpr +| 14 = @generictypeinstantiationexpr +| 15 = @sliceexpr +| 16 = @typeassertexpr +| 17 = @callorconversionexpr +| 18 = @starexpr +| 19 = @keyvalueexpr +| 20 = @arraytypeexpr +| 21 = @structtypeexpr +| 22 = @functypeexpr +| 23 = @interfacetypeexpr +| 24 = @maptypeexpr +| 25 = @typesetliteralexpr +| 26 = @plusexpr +| 27 = @minusexpr +| 28 = @notexpr +| 29 = @complementexpr +| 30 = @derefexpr +| 31 = @addressexpr +| 32 = @arrowexpr +| 33 = @lorexpr +| 34 = @landexpr +| 35 = @eqlexpr +| 36 = @neqexpr +| 37 = @lssexpr +| 38 = @leqexpr +| 39 = @gtrexpr +| 40 = @geqexpr +| 41 = @addexpr +| 42 = @subexpr +| 43 = @orexpr +| 44 = @xorexpr +| 45 = @mulexpr +| 46 = @quoexpr +| 47 = @remexpr +| 48 = @shlexpr +| 49 = @shrexpr +| 50 = @andexpr +| 51 = @andnotexpr +| 52 = @sendchantypeexpr +| 53 = @recvchantypeexpr +| 54 = @sendrcvchantypeexpr; + +@basiclit = @intlit | @floatlit | @imaglit | @charlit | @stringlit; + +@operatorexpr = @logicalexpr | @arithmeticexpr | @bitwiseexpr | @unaryexpr | @binaryexpr; + +@logicalexpr = @logicalunaryexpr | @logicalbinaryexpr; + +@arithmeticexpr = @arithmeticunaryexpr | @arithmeticbinaryexpr; + +@bitwiseexpr = @bitwiseunaryexpr | @bitwisebinaryexpr; + +@unaryexpr = @logicalunaryexpr | @bitwiseunaryexpr | @arithmeticunaryexpr | @derefexpr | @addressexpr | @arrowexpr; + +@logicalunaryexpr = @notexpr; + +@bitwiseunaryexpr = @complementexpr; + +@arithmeticunaryexpr = @plusexpr | @minusexpr; + +@binaryexpr = @logicalbinaryexpr | @bitwisebinaryexpr | @arithmeticbinaryexpr | @comparison; + +@logicalbinaryexpr = @lorexpr | @landexpr; + +@bitwisebinaryexpr = @shiftexpr | @orexpr | @xorexpr | @andexpr | @andnotexpr; + +@arithmeticbinaryexpr = @addexpr | @subexpr | @mulexpr | @quoexpr | @remexpr; + +@shiftexpr = @shlexpr | @shrexpr; + +@comparison = @equalitytest | @relationalcomparison; + +@equalitytest = @eqlexpr | @neqexpr; + +@relationalcomparison = @lssexpr | @leqexpr | @gtrexpr | @geqexpr; + +@chantypeexpr = @sendchantypeexpr | @recvchantypeexpr | @sendrcvchantypeexpr; + +case @stmt.kind of + 0 = @badstmt +| 1 = @declstmt +| 2 = @emptystmt +| 3 = @labeledstmt +| 4 = @exprstmt +| 5 = @sendstmt +| 6 = @incstmt +| 7 = @decstmt +| 8 = @gostmt +| 9 = @deferstmt +| 10 = @returnstmt +| 11 = @breakstmt +| 12 = @continuestmt +| 13 = @gotostmt +| 14 = @fallthroughstmt +| 15 = @blockstmt +| 16 = @ifstmt +| 17 = @caseclause +| 18 = @exprswitchstmt +| 19 = @typeswitchstmt +| 20 = @commclause +| 21 = @selectstmt +| 22 = @forstmt +| 23 = @rangestmt +| 24 = @assignstmt +| 25 = @definestmt +| 26 = @addassignstmt +| 27 = @subassignstmt +| 28 = @mulassignstmt +| 29 = @quoassignstmt +| 30 = @remassignstmt +| 31 = @andassignstmt +| 32 = @orassignstmt +| 33 = @xorassignstmt +| 34 = @shlassignstmt +| 35 = @shrassignstmt +| 36 = @andnotassignstmt; + +@incdecstmt = @incstmt | @decstmt; + +@assignment = @simpleassignstmt | @compoundassignstmt; + +@simpleassignstmt = @assignstmt | @definestmt; + +@compoundassignstmt = @addassignstmt | @subassignstmt | @mulassignstmt | @quoassignstmt | @remassignstmt + | @andassignstmt | @orassignstmt | @xorassignstmt | @shlassignstmt | @shrassignstmt | @andnotassignstmt; + +@branchstmt = @breakstmt | @continuestmt | @gotostmt | @fallthroughstmt; + +@switchstmt = @exprswitchstmt | @typeswitchstmt; + +@loopstmt = @forstmt | @rangestmt; + +case @decl.kind of + 0 = @baddecl +| 1 = @importdecl +| 2 = @constdecl +| 3 = @typedecl +| 4 = @vardecl +| 5 = @funcdecl; + +@gendecl = @importdecl | @constdecl | @typedecl | @vardecl; + +case @spec.kind of + 0 = @importspec +| 1 = @valuespec +| 2 = @typedefspec +| 3 = @aliasspec; + +@typespec = @typedefspec | @aliasspec; + +case @object.kind of + 0 = @pkgobject +| 1 = @decltypeobject +| 2 = @builtintypeobject +| 3 = @declconstobject +| 4 = @builtinconstobject +| 5 = @declvarobject +| 6 = @declfunctionobject +| 7 = @builtinfunctionobject +| 8 = @labelobject; + +@typeparamparentobject = @decltypeobject | @declfunctionobject; + +@declobject = @decltypeobject | @declconstobject | @declvarobject | @declfunctionobject; + +@builtinobject = @builtintypeobject | @builtinconstobject | @builtinfunctionobject; + +@typeobject = @decltypeobject | @builtintypeobject; + +@valueobject = @constobject | @varobject | @functionobject; + +@constobject = @declconstobject | @builtinconstobject; + +@varobject = @declvarobject; + +@functionobject = @declfunctionobject | @builtinfunctionobject; + +case @scope.kind of + 0 = @universescope +| 1 = @packagescope +| 2 = @localscope; + +case @type.kind of + 0 = @invalidtype +| 1 = @boolexprtype +| 2 = @inttype +| 3 = @int8type +| 4 = @int16type +| 5 = @int32type +| 6 = @int64type +| 7 = @uinttype +| 8 = @uint8type +| 9 = @uint16type +| 10 = @uint32type +| 11 = @uint64type +| 12 = @uintptrtype +| 13 = @float32type +| 14 = @float64type +| 15 = @complex64type +| 16 = @complex128type +| 17 = @stringexprtype +| 18 = @unsafepointertype +| 19 = @boolliteraltype +| 20 = @intliteraltype +| 21 = @runeliteraltype +| 22 = @floatliteraltype +| 23 = @complexliteraltype +| 24 = @stringliteraltype +| 25 = @nilliteraltype +| 26 = @typeparamtype +| 27 = @arraytype +| 28 = @slicetype +| 29 = @structtype +| 30 = @pointertype +| 31 = @interfacetype +| 32 = @tupletype +| 33 = @signaturetype +| 34 = @maptype +| 35 = @sendchantype +| 36 = @recvchantype +| 37 = @sendrcvchantype +| 38 = @definedtype +| 39 = @typesetliteraltype; + +@basictype = @booltype | @numerictype | @stringtype | @literaltype | @invalidtype | @unsafepointertype; + +@booltype = @boolexprtype | @boolliteraltype; + +@numerictype = @integertype | @floattype | @complextype; + +@integertype = @signedintegertype | @unsignedintegertype; + +@signedintegertype = @inttype | @int8type | @int16type | @int32type | @int64type | @intliteraltype | @runeliteraltype; + +@unsignedintegertype = @uinttype | @uint8type | @uint16type | @uint32type | @uint64type | @uintptrtype; + +@floattype = @float32type | @float64type | @floatliteraltype; + +@complextype = @complex64type | @complex128type | @complexliteraltype; + +@stringtype = @stringexprtype | @stringliteraltype; + +@literaltype = @boolliteraltype | @intliteraltype | @runeliteraltype | @floatliteraltype | @complexliteraltype + | @stringliteraltype | @nilliteraltype; + +@compositetype = @typeparamtype | @containertype | @structtype | @pointertype | @interfacetype | @tupletype + | @signaturetype | @definedtype | @typesetliteraltype; + +@containertype = @arraytype | @slicetype | @maptype | @chantype; + +@chantype = @sendchantype | @recvchantype | @sendrcvchantype; + +case @modexpr.kind of + 0 = @modcommentblock +| 1 = @modline +| 2 = @modlineblock +| 3 = @modlparen +| 4 = @modrparen; + +case @error.kind of + 0 = @unknownerror +| 1 = @listerror +| 2 = @parseerror +| 3 = @typeerror; + diff --git a/go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/typeparam.ql b/go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/typeparam.ql new file mode 100644 index 00000000000..041c33577f7 --- /dev/null +++ b/go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/typeparam.ql @@ -0,0 +1,15 @@ +class TypeParamType extends @typeparamtype { + string toString() { none() } +} + +class CompositeType extends @compositetype { + string toString() { none() } +} + +class TypeParamParentObject extends @typeparamparentobject { + string toString() { none() } +} + +from TypeParamType tp, string name, CompositeType bound, TypeParamParentObject parent, int idx +where typeparam(tp, name, bound, parent, idx) +select tp, name, bound, parent, idx, false diff --git a/go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/upgrade.properties b/go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/upgrade.properties new file mode 100644 index 00000000000..d383bdf4ad3 --- /dev/null +++ b/go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/upgrade.properties @@ -0,0 +1,3 @@ +description: Track whether a type parameter is from a receiver +compatibility: partial +typeparam.rel: run typeparam.qlo From e17508ca5118b2298785fc41d4c7bcc9f91ba21d Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Mon, 13 Jul 2026 12:34:16 +0200 Subject: [PATCH 46/90] Go: Expose new TypeParamType column and use in test --- go/ql/lib/semmle/go/Types.qll | 3 + .../semmle/go/Function/TypeParamType.expected | 66 +++++++++---------- .../semmle/go/Function/TypeParamType.ql | 3 +- 3 files changed, 38 insertions(+), 34 deletions(-) diff --git a/go/ql/lib/semmle/go/Types.qll b/go/ql/lib/semmle/go/Types.qll index 217ac5d211c..e1b5c70f095 100644 --- a/go/ql/lib/semmle/go/Types.qll +++ b/go/ql/lib/semmle/go/Types.qll @@ -395,6 +395,9 @@ class TypeParamType extends @typeparamtype, CompositeType { /** Gets the index of this type parameter type. */ int getIndex() { typeparam(this, _, _, _, result, _) } + /** Holds if this type parameter type is declared as part of a receiver */ + boolean isFromReceiver() { typeparam(this, _, _, _, _, result) } + override string pp() { result = this.getParamName() } /** diff --git a/go/ql/test/library-tests/semmle/go/Function/TypeParamType.expected b/go/ql/test/library-tests/semmle/go/Function/TypeParamType.expected index bda7c151797..42be6e28ab3 100644 --- a/go/ql/test/library-tests/semmle/go/Function/TypeParamType.expected +++ b/go/ql/test/library-tests/semmle/go/Function/TypeParamType.expected @@ -20,36 +20,36 @@ numberOfTypeParameters | genericFunctions.go:152:6:152:36 | multipleAnonymousTypeParamsType | 3 | | genericFunctions.go:154:51:154:51 | f | 3 | #select -| codeql-go-tests/function.EdgeConstraint | 0 | Node | interface { } | -| codeql-go-tests/function.Element | 0 | S | interface { } | -| codeql-go-tests/function.GenericFunctionInAnotherFile | 0 | T | interface { } | -| codeql-go-tests/function.GenericFunctionOneTypeParam | 0 | T | interface { } | -| codeql-go-tests/function.GenericFunctionTwoTypeParams | 0 | K | comparable | -| codeql-go-tests/function.GenericFunctionTwoTypeParams | 1 | V | interface { int64 \| float64 } | -| codeql-go-tests/function.GenericStruct1 | 0 | T | interface { } | -| codeql-go-tests/function.GenericStruct1.f1 | 0 | TF1 | interface { } | -| codeql-go-tests/function.GenericStruct1.g1 | 0 | TG1 | interface { } | -| codeql-go-tests/function.GenericStruct2 | 0 | S | interface { } | -| codeql-go-tests/function.GenericStruct2 | 1 | T | interface { } | -| codeql-go-tests/function.GenericStruct2.f2 | 0 | SF2 | interface { } | -| codeql-go-tests/function.GenericStruct2.f2 | 1 | TF2 | interface { } | -| codeql-go-tests/function.GenericStruct2.g2 | 0 | SG2 | interface { } | -| codeql-go-tests/function.GenericStruct2.g2 | 1 | TG2 | interface { } | -| codeql-go-tests/function.Graph | 0 | Node | NodeConstraint | -| codeql-go-tests/function.Graph | 1 | Edge | EdgeConstraint | -| codeql-go-tests/function.Graph.ShortestPath | 0 | Node | NodeConstraint | -| codeql-go-tests/function.Graph.ShortestPath | 1 | Edge | EdgeConstraint | -| codeql-go-tests/function.List | 0 | T | interface { } | -| codeql-go-tests/function.List.MyLen | 0 | U | interface { } | -| codeql-go-tests/function.New | 0 | Node | NodeConstraint | -| codeql-go-tests/function.New | 1 | Edge | EdgeConstraint | -| codeql-go-tests/function.NodeConstraint | 0 | Edge | interface { } | -| codeql-go-tests/function.multipleAnonymousTypeParamsFunc | 0 | _ | interface { } | -| codeql-go-tests/function.multipleAnonymousTypeParamsFunc | 1 | _ | interface { string } | -| codeql-go-tests/function.multipleAnonymousTypeParamsFunc | 2 | _ | interface { } | -| codeql-go-tests/function.multipleAnonymousTypeParamsType | 0 | _ | interface { } | -| codeql-go-tests/function.multipleAnonymousTypeParamsType | 1 | _ | interface { string } | -| codeql-go-tests/function.multipleAnonymousTypeParamsType | 2 | _ | interface { } | -| codeql-go-tests/function.multipleAnonymousTypeParamsType.f | 0 | _ | interface { } | -| codeql-go-tests/function.multipleAnonymousTypeParamsType.f | 1 | _ | interface { string } | -| codeql-go-tests/function.multipleAnonymousTypeParamsType.f | 2 | _ | interface { } | +| codeql-go-tests/function.EdgeConstraint | 0 | false | Node | interface { } | +| codeql-go-tests/function.Element | 0 | false | S | interface { } | +| codeql-go-tests/function.GenericFunctionInAnotherFile | 0 | false | T | interface { } | +| codeql-go-tests/function.GenericFunctionOneTypeParam | 0 | false | T | interface { } | +| codeql-go-tests/function.GenericFunctionTwoTypeParams | 0 | false | K | comparable | +| codeql-go-tests/function.GenericFunctionTwoTypeParams | 1 | false | V | interface { int64 \| float64 } | +| codeql-go-tests/function.GenericStruct1 | 0 | false | T | interface { } | +| codeql-go-tests/function.GenericStruct1.f1 | 0 | true | TF1 | interface { } | +| codeql-go-tests/function.GenericStruct1.g1 | 0 | true | TG1 | interface { } | +| codeql-go-tests/function.GenericStruct2 | 0 | false | S | interface { } | +| codeql-go-tests/function.GenericStruct2 | 1 | false | T | interface { } | +| codeql-go-tests/function.GenericStruct2.f2 | 0 | true | SF2 | interface { } | +| codeql-go-tests/function.GenericStruct2.f2 | 1 | true | TF2 | interface { } | +| codeql-go-tests/function.GenericStruct2.g2 | 0 | true | SG2 | interface { } | +| codeql-go-tests/function.GenericStruct2.g2 | 1 | true | TG2 | interface { } | +| codeql-go-tests/function.Graph | 0 | false | Node | NodeConstraint | +| codeql-go-tests/function.Graph | 1 | false | Edge | EdgeConstraint | +| codeql-go-tests/function.Graph.ShortestPath | 0 | true | Node | NodeConstraint | +| codeql-go-tests/function.Graph.ShortestPath | 1 | true | Edge | EdgeConstraint | +| codeql-go-tests/function.List | 0 | false | T | interface { } | +| codeql-go-tests/function.List.MyLen | 0 | true | U | interface { } | +| codeql-go-tests/function.New | 0 | false | Node | NodeConstraint | +| codeql-go-tests/function.New | 1 | false | Edge | EdgeConstraint | +| codeql-go-tests/function.NodeConstraint | 0 | false | Edge | interface { } | +| codeql-go-tests/function.multipleAnonymousTypeParamsFunc | 0 | false | _ | interface { } | +| codeql-go-tests/function.multipleAnonymousTypeParamsFunc | 1 | false | _ | interface { string } | +| codeql-go-tests/function.multipleAnonymousTypeParamsFunc | 2 | false | _ | interface { } | +| codeql-go-tests/function.multipleAnonymousTypeParamsType | 0 | false | _ | interface { } | +| codeql-go-tests/function.multipleAnonymousTypeParamsType | 1 | false | _ | interface { string } | +| codeql-go-tests/function.multipleAnonymousTypeParamsType | 2 | false | _ | interface { } | +| codeql-go-tests/function.multipleAnonymousTypeParamsType.f | 0 | true | _ | interface { } | +| codeql-go-tests/function.multipleAnonymousTypeParamsType.f | 1 | true | _ | interface { string } | +| codeql-go-tests/function.multipleAnonymousTypeParamsType.f | 2 | true | _ | interface { } | diff --git a/go/ql/test/library-tests/semmle/go/Function/TypeParamType.ql b/go/ql/test/library-tests/semmle/go/Function/TypeParamType.ql index 02dec23d2a8..365f58493f1 100644 --- a/go/ql/test/library-tests/semmle/go/Function/TypeParamType.ql +++ b/go/ql/test/library-tests/semmle/go/Function/TypeParamType.ql @@ -11,4 +11,5 @@ where // Note that we cannot use the location of `tpt` itself as we currently fail // to extract an object for type parameters for methods on generic structs. exists(ty.getLocation()) -select ty.getQualifiedName(), tpt.getIndex(), tpt.getParamName(), tpt.getConstraint().pp() +select ty.getQualifiedName(), tpt.getIndex(), tpt.isFromReceiver(), tpt.getParamName(), + tpt.getConstraint().pp() From 9c1f5c9be2fb6e9561d014d715e21a263e03084c Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Mon, 13 Jul 2026 12:47:17 +0200 Subject: [PATCH 47/90] Go: Fix dowgrade script formatting --- .../5ff5325d274ae4f86defa195577bc7c1370b72fa/typeparam.ql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/typeparam.ql b/go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/typeparam.ql index 00c17208e6e..ad7685ac16a 100644 --- a/go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/typeparam.ql +++ b/go/downgrades/5ff5325d274ae4f86defa195577bc7c1370b72fa/typeparam.ql @@ -10,6 +10,8 @@ class TypeParamParentObject extends @typeparamparentobject { string toString() { none() } } -from TypeParamType tp, string name, CompositeType bound, TypeParamParentObject parent, int idx, boolean is_from_recv +from + TypeParamType tp, string name, CompositeType bound, TypeParamParentObject parent, int idx, + boolean is_from_recv where typeparam(tp, name, bound, parent, idx, is_from_recv) select tp, name, bound, parent, idx From 9efcc49af706de6e342e577b1d8c932d7ca89c42 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Mon, 13 Jul 2026 13:06:15 +0200 Subject: [PATCH 48/90] C++: Update the CWE tag of `cpp/new-free-mismatch` --- cpp/ql/src/Critical/NewFreeMismatch.ql | 2 +- cpp/ql/src/change-notes/2026-07-13-new-free.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 cpp/ql/src/change-notes/2026-07-13-new-free.md diff --git a/cpp/ql/src/Critical/NewFreeMismatch.ql b/cpp/ql/src/Critical/NewFreeMismatch.ql index 19b9b197214..7443ad97731 100644 --- a/cpp/ql/src/Critical/NewFreeMismatch.ql +++ b/cpp/ql/src/Critical/NewFreeMismatch.ql @@ -8,7 +8,7 @@ * @id cpp/new-free-mismatch * @tags reliability * security - * external/cwe/cwe-401 + * external/cwe/cwe-762 */ import NewDelete diff --git a/cpp/ql/src/change-notes/2026-07-13-new-free.md b/cpp/ql/src/change-notes/2026-07-13-new-free.md new file mode 100644 index 00000000000..aef5a58aa48 --- /dev/null +++ b/cpp/ql/src/change-notes/2026-07-13-new-free.md @@ -0,0 +1,4 @@ +--- +category: queryMetadata +--- +* Added the tag `external/cwe/cwe-762` to `cpp/new-free-mismatch`, and removed the tag `external/cwe/cwe-401`. This better matches the behavior of the query. From e0f0987b81a126d21a91451e2d7a0f4c774ed237 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Mon, 13 Jul 2026 13:24:23 +0200 Subject: [PATCH 49/90] Go: Make `isFromReceiver` and actual `predicate` --- go/ql/lib/semmle/go/Types.qll | 2 +- .../semmle/go/Function/TypeParamType.expected | 66 +++++++++---------- .../semmle/go/Function/TypeParamType.ql | 8 +-- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/go/ql/lib/semmle/go/Types.qll b/go/ql/lib/semmle/go/Types.qll index e1b5c70f095..1ea5b5c65c1 100644 --- a/go/ql/lib/semmle/go/Types.qll +++ b/go/ql/lib/semmle/go/Types.qll @@ -396,7 +396,7 @@ class TypeParamType extends @typeparamtype, CompositeType { int getIndex() { typeparam(this, _, _, _, result, _) } /** Holds if this type parameter type is declared as part of a receiver */ - boolean isFromReceiver() { typeparam(this, _, _, _, _, result) } + predicate isFromReceiver() { typeparam(this, _, _, _, _, true) } override string pp() { result = this.getParamName() } diff --git a/go/ql/test/library-tests/semmle/go/Function/TypeParamType.expected b/go/ql/test/library-tests/semmle/go/Function/TypeParamType.expected index 42be6e28ab3..4addfa50f34 100644 --- a/go/ql/test/library-tests/semmle/go/Function/TypeParamType.expected +++ b/go/ql/test/library-tests/semmle/go/Function/TypeParamType.expected @@ -20,36 +20,36 @@ numberOfTypeParameters | genericFunctions.go:152:6:152:36 | multipleAnonymousTypeParamsType | 3 | | genericFunctions.go:154:51:154:51 | f | 3 | #select -| codeql-go-tests/function.EdgeConstraint | 0 | false | Node | interface { } | -| codeql-go-tests/function.Element | 0 | false | S | interface { } | -| codeql-go-tests/function.GenericFunctionInAnotherFile | 0 | false | T | interface { } | -| codeql-go-tests/function.GenericFunctionOneTypeParam | 0 | false | T | interface { } | -| codeql-go-tests/function.GenericFunctionTwoTypeParams | 0 | false | K | comparable | -| codeql-go-tests/function.GenericFunctionTwoTypeParams | 1 | false | V | interface { int64 \| float64 } | -| codeql-go-tests/function.GenericStruct1 | 0 | false | T | interface { } | -| codeql-go-tests/function.GenericStruct1.f1 | 0 | true | TF1 | interface { } | -| codeql-go-tests/function.GenericStruct1.g1 | 0 | true | TG1 | interface { } | -| codeql-go-tests/function.GenericStruct2 | 0 | false | S | interface { } | -| codeql-go-tests/function.GenericStruct2 | 1 | false | T | interface { } | -| codeql-go-tests/function.GenericStruct2.f2 | 0 | true | SF2 | interface { } | -| codeql-go-tests/function.GenericStruct2.f2 | 1 | true | TF2 | interface { } | -| codeql-go-tests/function.GenericStruct2.g2 | 0 | true | SG2 | interface { } | -| codeql-go-tests/function.GenericStruct2.g2 | 1 | true | TG2 | interface { } | -| codeql-go-tests/function.Graph | 0 | false | Node | NodeConstraint | -| codeql-go-tests/function.Graph | 1 | false | Edge | EdgeConstraint | -| codeql-go-tests/function.Graph.ShortestPath | 0 | true | Node | NodeConstraint | -| codeql-go-tests/function.Graph.ShortestPath | 1 | true | Edge | EdgeConstraint | -| codeql-go-tests/function.List | 0 | false | T | interface { } | -| codeql-go-tests/function.List.MyLen | 0 | true | U | interface { } | -| codeql-go-tests/function.New | 0 | false | Node | NodeConstraint | -| codeql-go-tests/function.New | 1 | false | Edge | EdgeConstraint | -| codeql-go-tests/function.NodeConstraint | 0 | false | Edge | interface { } | -| codeql-go-tests/function.multipleAnonymousTypeParamsFunc | 0 | false | _ | interface { } | -| codeql-go-tests/function.multipleAnonymousTypeParamsFunc | 1 | false | _ | interface { string } | -| codeql-go-tests/function.multipleAnonymousTypeParamsFunc | 2 | false | _ | interface { } | -| codeql-go-tests/function.multipleAnonymousTypeParamsType | 0 | false | _ | interface { } | -| codeql-go-tests/function.multipleAnonymousTypeParamsType | 1 | false | _ | interface { string } | -| codeql-go-tests/function.multipleAnonymousTypeParamsType | 2 | false | _ | interface { } | -| codeql-go-tests/function.multipleAnonymousTypeParamsType.f | 0 | true | _ | interface { } | -| codeql-go-tests/function.multipleAnonymousTypeParamsType.f | 1 | true | _ | interface { string } | -| codeql-go-tests/function.multipleAnonymousTypeParamsType.f | 2 | true | _ | interface { } | +| codeql-go-tests/function.EdgeConstraint | 0 | | Node | interface { } | +| codeql-go-tests/function.Element | 0 | | S | interface { } | +| codeql-go-tests/function.GenericFunctionInAnotherFile | 0 | | T | interface { } | +| codeql-go-tests/function.GenericFunctionOneTypeParam | 0 | | T | interface { } | +| codeql-go-tests/function.GenericFunctionTwoTypeParams | 0 | | K | comparable | +| codeql-go-tests/function.GenericFunctionTwoTypeParams | 1 | | V | interface { int64 \| float64 } | +| codeql-go-tests/function.GenericStruct1 | 0 | | T | interface { } | +| codeql-go-tests/function.GenericStruct1.f1 | 0 | from receiver | TF1 | interface { } | +| codeql-go-tests/function.GenericStruct1.g1 | 0 | from receiver | TG1 | interface { } | +| codeql-go-tests/function.GenericStruct2 | 0 | | S | interface { } | +| codeql-go-tests/function.GenericStruct2 | 1 | | T | interface { } | +| codeql-go-tests/function.GenericStruct2.f2 | 0 | from receiver | SF2 | interface { } | +| codeql-go-tests/function.GenericStruct2.f2 | 1 | from receiver | TF2 | interface { } | +| codeql-go-tests/function.GenericStruct2.g2 | 0 | from receiver | SG2 | interface { } | +| codeql-go-tests/function.GenericStruct2.g2 | 1 | from receiver | TG2 | interface { } | +| codeql-go-tests/function.Graph | 0 | | Node | NodeConstraint | +| codeql-go-tests/function.Graph | 1 | | Edge | EdgeConstraint | +| codeql-go-tests/function.Graph.ShortestPath | 0 | from receiver | Node | NodeConstraint | +| codeql-go-tests/function.Graph.ShortestPath | 1 | from receiver | Edge | EdgeConstraint | +| codeql-go-tests/function.List | 0 | | T | interface { } | +| codeql-go-tests/function.List.MyLen | 0 | from receiver | U | interface { } | +| codeql-go-tests/function.New | 0 | | Node | NodeConstraint | +| codeql-go-tests/function.New | 1 | | Edge | EdgeConstraint | +| codeql-go-tests/function.NodeConstraint | 0 | | Edge | interface { } | +| codeql-go-tests/function.multipleAnonymousTypeParamsFunc | 0 | | _ | interface { } | +| codeql-go-tests/function.multipleAnonymousTypeParamsFunc | 1 | | _ | interface { string } | +| codeql-go-tests/function.multipleAnonymousTypeParamsFunc | 2 | | _ | interface { } | +| codeql-go-tests/function.multipleAnonymousTypeParamsType | 0 | | _ | interface { } | +| codeql-go-tests/function.multipleAnonymousTypeParamsType | 1 | | _ | interface { string } | +| codeql-go-tests/function.multipleAnonymousTypeParamsType | 2 | | _ | interface { } | +| codeql-go-tests/function.multipleAnonymousTypeParamsType.f | 0 | from receiver | _ | interface { } | +| codeql-go-tests/function.multipleAnonymousTypeParamsType.f | 1 | from receiver | _ | interface { string } | +| codeql-go-tests/function.multipleAnonymousTypeParamsType.f | 2 | from receiver | _ | interface { } | diff --git a/go/ql/test/library-tests/semmle/go/Function/TypeParamType.ql b/go/ql/test/library-tests/semmle/go/Function/TypeParamType.ql index 365f58493f1..323defa0493 100644 --- a/go/ql/test/library-tests/semmle/go/Function/TypeParamType.ql +++ b/go/ql/test/library-tests/semmle/go/Function/TypeParamType.ql @@ -5,11 +5,11 @@ query predicate numberOfTypeParameters(TypeParamParentEntity parent, int n) { n = strictcount(TypeParamType tpt | tpt.getParent() = parent) } -from TypeParamType tpt, TypeParamParentEntity ty +from TypeParamType tpt, TypeParamParentEntity ty, string fr where ty = tpt.getParent() and // Note that we cannot use the location of `tpt` itself as we currently fail // to extract an object for type parameters for methods on generic structs. - exists(ty.getLocation()) -select ty.getQualifiedName(), tpt.getIndex(), tpt.isFromReceiver(), tpt.getParamName(), - tpt.getConstraint().pp() + exists(ty.getLocation()) and + if tpt.isFromReceiver() then fr = "from receiver" else fr = "" +select ty.getQualifiedName(), tpt.getIndex(), fr, tpt.getParamName(), tpt.getConstraint().pp() From a8885aeebd465ec65db7913b452d44a8179f9d00 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Mon, 13 Jul 2026 15:18:38 +0200 Subject: [PATCH 50/90] C#: Remove the HttpRequest.RawUrl barrier model. --- csharp/ql/lib/ext/System.Web.model.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/csharp/ql/lib/ext/System.Web.model.yml b/csharp/ql/lib/ext/System.Web.model.yml index 63c539fbe5e..9e24158bc09 100644 --- a/csharp/ql/lib/ext/System.Web.model.yml +++ b/csharp/ql/lib/ext/System.Web.model.yml @@ -1,10 +1,4 @@ extensions: - - addsTo: - pack: codeql/csharp-all - extensible: barrierModel - data: - # The RawUrl property is considered to be safe for URL redirects - - ["System.Web", "HttpRequest", False, "get_RawUrl", "()", "", "ReturnValue", "url-redirection", "manual"] - addsTo: pack: codeql/csharp-all extensible: sinkModel From 03e44f548fcc56c8ccf07cf4b489e8fb63f10c57 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Mon, 13 Jul 2026 15:19:04 +0200 Subject: [PATCH 51/90] C#: Update test and expected output. --- .../Security Features/CWE-601/UrlRedirect/UrlRedirect.cs | 4 ++-- .../CWE-601/UrlRedirect/UrlRedirect.expected | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/csharp/ql/test/query-tests/Security Features/CWE-601/UrlRedirect/UrlRedirect.cs b/csharp/ql/test/query-tests/Security Features/CWE-601/UrlRedirect/UrlRedirect.cs index 498b004505e..efb19036533 100644 --- a/csharp/ql/test/query-tests/Security Features/CWE-601/UrlRedirect/UrlRedirect.cs +++ b/csharp/ql/test/query-tests/Security Features/CWE-601/UrlRedirect/UrlRedirect.cs @@ -38,8 +38,8 @@ public class UrlRedirectHandler : IHttpHandler ctx.Response.AddHeader("Location", ctx.Request.QueryString["page"]); // $ Alert=r3 Alert=r3 ctx.Response.AppendHeader("Location", ctx.Request.QueryString["page"]); // $ Alert=r4 Alert=r4 - // GOOD: Redirecting to the RawUrl only reloads the current Url - ctx.Response.Redirect(ctx.Request.RawUrl); + // BAD: The RawUrl contains the un-normalized request line. + ctx.Response.Redirect(ctx.Request.RawUrl); // $ Alert // GOOD: The attacker can only control the parameters, not the location ctx.Response.Redirect("foo.asp?param=" + url); diff --git a/csharp/ql/test/query-tests/Security Features/CWE-601/UrlRedirect/UrlRedirect.expected b/csharp/ql/test/query-tests/Security Features/CWE-601/UrlRedirect/UrlRedirect.expected index e7fced7fde3..b5ff8e88234 100644 --- a/csharp/ql/test/query-tests/Security Features/CWE-601/UrlRedirect/UrlRedirect.expected +++ b/csharp/ql/test/query-tests/Security Features/CWE-601/UrlRedirect/UrlRedirect.expected @@ -3,6 +3,7 @@ | UrlRedirect.cs:13:31:13:61 | access to indexer | UrlRedirect.cs:13:31:13:53 | access to property QueryString : NameValueCollection | UrlRedirect.cs:13:31:13:61 | access to indexer | Untrusted URL redirection due to $@. | UrlRedirect.cs:13:31:13:53 | access to property QueryString | user-provided value | | UrlRedirect.cs:38:44:38:74 | access to indexer | UrlRedirect.cs:38:44:38:66 | access to property QueryString : NameValueCollection | UrlRedirect.cs:38:44:38:74 | access to indexer | Untrusted URL redirection due to $@. | UrlRedirect.cs:38:44:38:66 | access to property QueryString | user-provided value | | UrlRedirect.cs:39:47:39:77 | access to indexer | UrlRedirect.cs:39:47:39:69 | access to property QueryString : NameValueCollection | UrlRedirect.cs:39:47:39:77 | access to indexer | Untrusted URL redirection due to $@. | UrlRedirect.cs:39:47:39:69 | access to property QueryString | user-provided value | +| UrlRedirect.cs:42:31:42:48 | access to property RawUrl | UrlRedirect.cs:42:31:42:48 | access to property RawUrl | UrlRedirect.cs:42:31:42:48 | access to property RawUrl | Untrusted URL redirection due to $@. | UrlRedirect.cs:42:31:42:48 | access to property RawUrl | user-provided value | | UrlRedirect.cs:48:29:48:31 | access to local variable url | UrlRedirect.cs:23:22:23:44 | access to property QueryString : NameValueCollection | UrlRedirect.cs:48:29:48:31 | access to local variable url | Untrusted URL redirection due to $@. | UrlRedirect.cs:23:22:23:44 | access to property QueryString | user-provided value | | UrlRedirect.cs:64:31:64:52 | $"..." | UrlRedirect.cs:23:22:23:44 | access to property QueryString : NameValueCollection | UrlRedirect.cs:64:31:64:52 | $"..." | Untrusted URL redirection due to $@. | UrlRedirect.cs:23:22:23:44 | access to property QueryString | user-provided value | | UrlRedirect.cs:70:31:70:69 | call to method Format | UrlRedirect.cs:23:22:23:44 | access to property QueryString : NameValueCollection | UrlRedirect.cs:70:31:70:69 | call to method Format | Untrusted URL redirection due to $@. | UrlRedirect.cs:23:22:23:44 | access to property QueryString | user-provided value | @@ -66,6 +67,7 @@ nodes | UrlRedirect.cs:38:44:38:74 | access to indexer | semmle.label | access to indexer | | UrlRedirect.cs:39:47:39:69 | access to property QueryString : NameValueCollection | semmle.label | access to property QueryString : NameValueCollection | | UrlRedirect.cs:39:47:39:77 | access to indexer | semmle.label | access to indexer | +| UrlRedirect.cs:42:31:42:48 | access to property RawUrl | semmle.label | access to property RawUrl | | UrlRedirect.cs:48:29:48:31 | access to local variable url | semmle.label | access to local variable url | | UrlRedirect.cs:64:31:64:52 | $"..." | semmle.label | $"..." | | UrlRedirect.cs:70:31:70:69 | call to method Format | semmle.label | call to method Format | From ae06b778e5203ee3773b987c34bfa151bce6727a Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Mon, 13 Jul 2026 15:29:18 +0200 Subject: [PATCH 52/90] C#: Add change-note. --- csharp/ql/src/change-notes/2026-07-13-CWE-601.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 csharp/ql/src/change-notes/2026-07-13-CWE-601.md diff --git a/csharp/ql/src/change-notes/2026-07-13-CWE-601.md b/csharp/ql/src/change-notes/2026-07-13-CWE-601.md new file mode 100644 index 00000000000..237ee562a52 --- /dev/null +++ b/csharp/ql/src/change-notes/2026-07-13-CWE-601.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* `System.Web.HttpRequest.RawUrl` is no longer treated as a sanitizer for `cs/web/unvalidated-url-redirection`, since it contains the un-normalized request line. This may lead to more results. From 2d0095826b8082e43b613c08779119dffe18b460 Mon Sep 17 00:00:00 2001 From: Owen Mansel-Chan Date: Mon, 13 Jul 2026 19:40:31 +0100 Subject: [PATCH 53/90] Address review comment --- .../code/java/security/PathSanitizer.qll | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/java/ql/lib/semmle/code/java/security/PathSanitizer.qll b/java/ql/lib/semmle/code/java/security/PathSanitizer.qll index 7c428a6691c..b4344c50cea 100644 --- a/java/ql/lib/semmle/code/java/security/PathSanitizer.qll +++ b/java/ql/lib/semmle/code/java/security/PathSanitizer.qll @@ -108,17 +108,19 @@ private class FileGetNameCall extends MethodCall { */ private predicate dotDotCheckGuard(Guard g, Expr e, boolean branch) { pathTraversalGuard(g, e, branch) and - ( - exists(Guard previousGuard | - previousGuard.(AllowedPrefixGuard).controls(g.getBasicBlock(), true) - or - previousGuard.(BlockListGuard).controls(g.getBasicBlock(), false) - ) + exists(Guard previousGuard | + previousGuard.(AllowedPrefixGuard).controls(g.getBasicBlock(), true) or - // `File.getName` strips any directory prefix, returning only the final path - // component. The only remaining path traversal risk is when that component is - // itself `..`, so a check for `..` components completes the sanitization. - exists(FileGetNameCall getName | localTaintFlowToPathGuard(getName, g)) + previousGuard.(BlockListGuard).controls(g.getBasicBlock(), false) + ) + or + // `File.getName` strips any directory prefix, returning only the final path + // component. The only remaining path traversal risk is when that component is + // itself `..`, so a check for `..` components on the result of `getName` + // completes the sanitization. + exists(FileGetNameCall getName | + pathTraversalGuard(g, getName, branch) and + TaintTracking::localExprTaint(getName, e) ) } From 09da46d8bd583d289ee9559deafbfbc339810b19 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Tue, 14 Jul 2026 10:23:43 +0200 Subject: [PATCH 54/90] Go: Address review comments --- go/extractor/extractor.go | 18 +++++++++--------- .../typeparam.ql | 8 +++++++- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index f421688e52a..8c217837614 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -2010,11 +2010,9 @@ func extractTypeParamDecls(tw *trap.Writer, fields *ast.FieldList, parent trap.L } func populateTypeParamParentsFromFunction(funcObj *types.Func) { - recvTypeParams := funcObj.Type().(*types.Signature).RecvTypeParams() - populateTypeParamParents(recvTypeParams, funcObj, true) - typeParams := funcObj.Type().(*types.Signature).TypeParams() - populateTypeParamParents(typeParams, funcObj, false) - + signature := funcObj.Type().(*types.Signature) + populateTypeParamParents(signature.RecvTypeParams(), funcObj, true) + populateTypeParamParents(signature.TypeParams(), funcObj, false) } // populateTypeParamParents sets `parent` as the parent of the elements of `typeparams` @@ -2053,12 +2051,14 @@ func getTypeParamParentLabel(tw *trap.Writer, tp *types.TypeParam) (trap.Label, return parentlbl, entry.isFromReceiver } -func setTypeParamParent(tp *types.TypeParam, newobj types.Object, isFromReceiver bool) { +func setTypeParamParent(tp *types.TypeParam, parent types.Object, isFromReceiver bool) { entry, exists := typeParamParent[tp] + newEntry := typeParamParentEntry{parent, isFromReceiver} if !exists { - typeParamParent[tp] = typeParamParentEntry{newobj, isFromReceiver} - } else if entry.parent != newobj { - log.Fatalf("Parent of type parameter '%s %s' being set to a different value: '%s' vs '%s'", tp.String(), tp.Constraint().String(), entry.parent, newobj) + typeParamParent[tp] = newEntry + } else if entry != newEntry { + log.Fatalf("Parent of type parameter '%s %s' being set to a different value: {'%s', %t}' vs {'%s', %t}", + tp.String(), tp.Constraint().String(), entry.parent, entry.isFromReceiver, parent, isFromReceiver) } } diff --git a/go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/typeparam.ql b/go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/typeparam.ql index 041c33577f7..05ceab70344 100644 --- a/go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/typeparam.ql +++ b/go/ql/lib/upgrades/b1341734d6870b105e5c9d168ce7dec25d7f72d0/typeparam.ql @@ -10,6 +10,12 @@ class TypeParamParentObject extends @typeparamparentobject { string toString() { none() } } +// In Go 1.26 and below, a type parameter is from a receiver exactly when its +// parent is a method. +boolean isFromReceiver(TypeParamParentObject parent) { + if methodreceivers(parent, _) then result = true else result = false +} + from TypeParamType tp, string name, CompositeType bound, TypeParamParentObject parent, int idx where typeparam(tp, name, bound, parent, idx) -select tp, name, bound, parent, idx, false +select tp, name, bound, parent, idx, isFromReceiver(parent) From 3e2daf2258da21c31f6a657295de20c4da0a9708 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Jul 2026 14:12:30 +0000 Subject: [PATCH 55/90] Support building swift-syntax-rs on macOS --- MODULE.bazel | 34 +++---- unified/swift-syntax-rs/BUILD.bazel | 53 ++++++----- unified/swift-syntax-rs/README.md | 14 ++- unified/swift-syntax-rs/build.rs | 2 +- unified/swift-syntax-rs/swift/Package.swift | 5 ++ unified/swift-syntax-rs/xcode_transition.bzl | 94 ++++++++++++++++++++ 6 files changed, 161 insertions(+), 41 deletions(-) create mode 100644 unified/swift-syntax-rs/xcode_transition.bzl diff --git a/MODULE.bazel b/MODULE.bazel index aa1f6555e0c..4c9e3316c01 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -33,6 +33,11 @@ bazel_dep(name = "googletest", version = "1.17.0.bcr.2") bazel_dep(name = "rules_rust", version = "0.69.0") bazel_dep(name = "rules_swift", version = "4.0.0-rc4") bazel_dep(name = "swift-syntax", version = "602.0.0.bcr.2") + +# Needed so we can `use_repo` `local_config_xcode` and +# `local_config_apple_cc_toolchains` below (referenced by the per-target +# Xcode-config transition in `unified/swift-syntax-rs/xcode_transition.bzl`). +bazel_dep(name = "apple_support", version = "2.6.1") bazel_dep(name = "zstd", version = "1.5.7.bcr.1") bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True) @@ -221,22 +226,11 @@ use_repo( "swift-resource-dir-macos", ) -# Hermetic Swift toolchain (from swift.org) for building the `swift-syntax` -# based Rust wrapper in `unified/swift-syntax-rs`. This is the official -# `rules_swift` standalone-toolchain extension and is independent of the -# patched prebuilt toolchain wired up via `swift_deps` above. -# -# Bazel cannot auto-select between Linux distributions, so the toolchain is -# registered explicitly for the distribution our CI runs on (ubuntu24.04, -# x86_64). Add further platforms here if CI grows to cover them. -# -# We register the `exec` toolchain (normal host compilation, targeting -# linux/x86_64) rather than the `embedded` one, which targets `os:none` -# (Embedded Swift) and would not match a normal host build. -# -# The Swift version is read from `unified/swift-syntax-rs/.swift-version`, which -# is the single source of truth shared with the local `cargo` build (via -# swiftly / any Swift install) and `swift/Package.swift`. +# Swift toolchain for building `unified/swift-syntax-rs`. On Linux we register +# a hermetic swift.org toolchain as the exec toolchain; on macOS `rules_swift` +# auto-registers `xcode_swift_toolchain` (host Xcode + OS-provided Swift +# runtime), which is not hermetic. Version comes from +# `unified/swift-syntax-rs/.swift-version`. swift = use_extension("@rules_swift//swift:extensions.bzl", "swift") swift.toolchain( name = "swift_toolchain", @@ -252,6 +246,14 @@ register_toolchains( "@swift_toolchain//:swift_toolchain_exec_ubuntu24.04", ) +# `apple_support`'s xcode_config and CC toolchains, needed by the Xcode +# transition in `unified/swift-syntax-rs/xcode_transition.bzl`. +xcode_configure = use_extension("@apple_support//xcode:xcode_configure.bzl", "xcode_configure_extension") +use_repo(xcode_configure, "local_config_xcode") + +apple_cc_configure = use_extension("@apple_support//crosstool:setup.bzl", "apple_cc_configure_extension") +use_repo(apple_cc_configure, "local_config_apple_cc_toolchains") + node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node") node.toolchain( name = "nodejs", diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index 9db40d4db30..be7d41eaf87 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -1,31 +1,36 @@ load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test") -load("@rules_swift//swift:swift.bzl", "swift_library") +load(":xcode_transition.bzl", "xcode_transition_swift_library") package(default_visibility = ["//visibility:public"]) -# The pinned Swift version, shared with the local `cargo` build and -# `swift/Package.swift`. Referenced by the `swift.toolchain` extension in -# //:MODULE.bazel via `swift_version_file`. +# Pinned Swift version, shared with `cargo` and `swift/Package.swift`. +# Referenced by `swift.toolchain(swift_version_file = ...)` in //:MODULE.bazel. exports_files([".swift-version"]) -# The Swift FFI shim: wraps swift-syntax and exposes a small C ABI -# (`ssr_parse_json` / `ssr_string_free`). Built with the hermetic Swift -# toolchain registered in //:MODULE.bazel; provides a `CcInfo` that the Rust -# targets below link against. -swift_library( +# Targets in this package require a Swift toolchain (Linux or macOS). +# `select()` gives us OR-of-OSes; other platforms get marked incompatible +# so `bazel build/test //...` skips them cleanly. +_SWIFT_SUPPORTED_PLATFORMS = select({ + "@platforms//os:linux": [], + "@platforms//os:macos": [], + "//conditions:default": ["@platforms//:incompatible"], +}) + +# Swift FFI shim: wraps swift-syntax and exposes a small C ABI. The Rust +# targets below link against its `CcInfo`. +xcode_transition_swift_library( name = "swift_syntax_ffi", srcs = ["swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift"], module_name = "SwiftSyntaxFFI", + target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, deps = [ "@swift-syntax//:SwiftParser", "@swift-syntax//:SwiftSyntax", ], ) -# Safe Rust bindings on top of the C ABI. `build.rs` is intentionally *not* -# wired up here (no `cargo_build_script`): under Bazel the Swift side is -# provided by `:swift_syntax_ffi`, whereas `build.rs` only exists to build the -# Swift shim in the local `cargo` workflow. +# Safe Rust bindings on top of the C ABI. Under Bazel the Swift side comes +# from `:swift_syntax_ffi`; `build.rs` is only used by the `cargo` workflow. rust_library( name = "swift_syntax_rs", srcs = glob( @@ -33,6 +38,7 @@ rust_library( exclude = ["src/main.rs"], ), edition = "2024", + target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, deps = [ ":swift_syntax_ffi", ], @@ -41,13 +47,16 @@ rust_library( rust_binary( name = "swift-syntax-parse", srcs = ["src/main.rs"], - # The Swift toolchain propagates a runfiles-relative RPATH to its runtime - # `.so`s via `swift_syntax_ffi`'s `CcInfo`, but (unlike `swift_binary`) - # `rust_binary` does not copy those libraries into runfiles. Add the - # toolchain files as `data` so the runtime is found at execution time. - # (rules_swift does not yet support statically linking the runtime.) - data = ["@swift_toolchain_ubuntu24.04//:files"], + # `rust_binary` doesn't copy the Swift runtime into runfiles the way + # `swift_binary` does. On Linux, ship the standalone toolchain's runtime; + # on macOS the OS provides it at `/usr/lib/swift` (rpath'd by + # `xcode_swift_toolchain`). + data = select({ + "@platforms//os:macos": [], + "@platforms//os:linux": ["@swift_toolchain_ubuntu24.04//:files"], + }), edition = "2024", + target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, deps = [":swift_syntax_rs"], ) @@ -55,6 +64,10 @@ rust_test( name = "swift_syntax_rs_test", size = "small", crate = ":swift_syntax_rs", - data = ["@swift_toolchain_ubuntu24.04//:files"], + data = select({ + "@platforms//os:macos": [], + "@platforms//os:linux": ["@swift_toolchain_ubuntu24.04//:files"], + }), edition = "2024", + target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, ) diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index a2d66495a54..6a714137d8c 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -144,10 +144,16 @@ Requirements: - **`clang`** must be installed on the runner. `rules_swift` requires the Bazel CC toolchain to use clang; the repo's `.bazelrc` already sets `--repo_env=CC=clang`, so no extra flags are needed. -- The registered Swift toolchain currently targets **ubuntu24.04 / x86_64** - only (Bazel cannot auto-select between Linux distributions). Add more - platforms in `MODULE.bazel` (`swift.toolchain` + `register_toolchains`) if CI - grows to cover them. +- The registered Swift toolchains cover **ubuntu24.04 / x86_64** and + **macOS / `xcode`** (Apple Silicon and Intel). Bazel selects the toolchain + matching the host. Targets are marked `target_compatible_with` these two + OSes, so on Windows Bazel skips them cleanly. +- **macOS only:** the Swift toolchain comes from the host Xcode installation + (`rules_swift` auto-registers `xcode_swift_toolchain`), which also needs + Xcode's CC toolchain and xcode_config; these are applied to the Swift + target via an incoming-edge Starlark transition (see + [`xcode_transition.bzl`](xcode_transition.bzl)), so other targets on macOS + keep using Bazel's default CC toolchain. The Swift compiler version is read from [`.swift-version`](.swift-version) by both the Bazel toolchain (`swift.toolchain(swift_version_file = …)`) and the diff --git a/unified/swift-syntax-rs/build.rs b/unified/swift-syntax-rs/build.rs index 0adb140140d..3cd2e285874 100644 --- a/unified/swift-syntax-rs/build.rs +++ b/unified/swift-syntax-rs/build.rs @@ -73,7 +73,7 @@ fn swift_runtime_dir() -> Option { let close = value_start.find('"')?; let resource_path = &value_start[..close]; - Some(PathBuf::from(resource_path).join("linux")) + Some(PathBuf::from(resource_path).join(if cfg!(target_os = "macos") { "macosx" } else { "linux" })) } /// The `swift` driver to invoke: `$SWIFT` if set, otherwise `swift` from `PATH`. diff --git a/unified/swift-syntax-rs/swift/Package.swift b/unified/swift-syntax-rs/swift/Package.swift index 1b0111364ce..fcd9fcc07bc 100644 --- a/unified/swift-syntax-rs/swift/Package.swift +++ b/unified/swift-syntax-rs/swift/Package.swift @@ -3,6 +3,11 @@ import PackageDescription let package = Package( name: "SwiftSyntaxFFI", + platforms: [ + // swift-syntax 602 requires macOS 10.15; declare it explicitly + // rather than relying on the swift-tools-version default (10.13). + .macOS(.v10_15), + ], products: [ // Dynamic library so the produced .so records its dependency on the // Swift runtime libraries, keeping the Rust link step simple. diff --git a/unified/swift-syntax-rs/xcode_transition.bzl b/unified/swift-syntax-rs/xcode_transition.bzl new file mode 100644 index 00000000000..c6f293a0452 --- /dev/null +++ b/unified/swift-syntax-rs/xcode_transition.bzl @@ -0,0 +1,94 @@ +"""Per-target Xcode configuration for `//unified/swift-syntax-rs` on macOS. + +`rules_swift`'s auto-registered `xcode_swift_toolchain` reads +`cc_toolchain.target_gnu_system_name`, which is literally "local" on +Bazel's built-in `local_config_cc`. To make it usable we need: + +- `--xcode_version_config=@local_config_xcode//:host_xcodes` — selects + `apple_support`'s xcode_config (matches `rules_swift`'s `system_sdk` keys). +- `--extra_toolchains=@local_config_apple_cc_toolchains//:all` — forces + `apple_support`'s CC toolchain ahead of `local_config_cc`. + +Applied via an incoming-edge Starlark transition on the `swift_library` +target only (via the `xcode_transition_swift_library` macro), so downstream +Rust targets and the rest of the repo stay on the default CC toolchain and +the `@local_config_*` repos are not materialized otherwise. No-op off macOS. +""" + +load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") +load("@rules_swift//swift:swift.bzl", "swift_library") +load("//misc/bazel:os.bzl", "os_select") + +_XCODE_VERSION_CONFIG = "//command_line_option:xcode_version_config" +_EXTRA_TOOLCHAINS = "//command_line_option:extra_toolchains" + +def _transition_impl(settings, attr): + if attr.os != "macos": + # Preserve inputs so the configuration is identical to the incoming one. + return { + _XCODE_VERSION_CONFIG: settings[_XCODE_VERSION_CONFIG], + _EXTRA_TOOLCHAINS: settings[_EXTRA_TOOLCHAINS], + } + return { + _XCODE_VERSION_CONFIG: "@local_config_xcode//:host_xcodes", + _EXTRA_TOOLCHAINS: ( + list(settings[_EXTRA_TOOLCHAINS]) + + ["@local_config_apple_cc_toolchains//:all"] + ), + } + +_xcode_transition = transition( + implementation = _transition_impl, + inputs = [_XCODE_VERSION_CONFIG, _EXTRA_TOOLCHAINS], + outputs = [_XCODE_VERSION_CONFIG, _EXTRA_TOOLCHAINS], +) + +def _wrapper_impl(ctx): + src = ctx.attr.actual[0] + # Forward the providers a downstream `rust_*` target reads from `deps`: + # `DefaultInfo`, `CcInfo` (linking info), and `OutputGroupInfo`. + providers = [src[DefaultInfo]] + for p in (CcInfo, OutputGroupInfo): + if p in src: + providers.append(src[p]) + return providers + +_xcode_transition_swift_library_rule = rule( + implementation = _wrapper_impl, + attrs = { + "actual": attr.label( + mandatory = True, + cfg = _xcode_transition, + providers = [CcInfo], + ), + "os": attr.string(mandatory = True), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + }, +) + +def xcode_transition_swift_library(name, visibility = None, tags = None, target_compatible_with = None, **kwargs): + """`swift_library` wrapped in the macOS Xcode-config transition. + + Emits a private inner `_impl_` `swift_library` (tagged `manual`) + and a public `name` that applies the transition on macOS and forwards + the inner target's providers. Downstream `rust_*` targets depend on + `name` as usual; only the `swift_library` sub-graph flips toolchain. + """ + inner_name = "_impl_%s" % name + swift_library( + name = inner_name, + visibility = ["//visibility:private"], + tags = (tags or []) + ["manual"], + target_compatible_with = target_compatible_with, + **kwargs + ) + _xcode_transition_swift_library_rule( + name = name, + visibility = visibility, + tags = tags, + target_compatible_with = target_compatible_with, + actual = ":" + inner_name, + os = os_select(linux = "linux", macos = "macos", default = "other"), + ) From b15c243ecad2acab4cb1ee1e2c33e4a7c2f15fa2 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 15 Jul 2026 08:36:49 +0200 Subject: [PATCH 56/90] C#: Exclude cs/useless-assignment-to-local from the code-quality suite. --- csharp/ql/src/codeql-suites/csharp-code-quality.qls | 3 +++ 1 file changed, 3 insertions(+) diff --git a/csharp/ql/src/codeql-suites/csharp-code-quality.qls b/csharp/ql/src/codeql-suites/csharp-code-quality.qls index 2074f9378cf..26c7e7437ee 100644 --- a/csharp/ql/src/codeql-suites/csharp-code-quality.qls +++ b/csharp/ql/src/codeql-suites/csharp-code-quality.qls @@ -1,3 +1,6 @@ - queries: . - apply: code-quality-selectors.yml from: codeql/suite-helpers +- exclude: + id: + - cs/useless-assignment-to-local From 2395053ed6e9c2a14b06986276e53f8aedca9238 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 15 Jul 2026 08:54:56 +0200 Subject: [PATCH 57/90] C#: Update integration test expected output. --- .../posix/query-suite/csharp-code-quality.qls.expected | 1 - 1 file changed, 1 deletion(-) diff --git a/csharp/ql/integration-tests/posix/query-suite/csharp-code-quality.qls.expected b/csharp/ql/integration-tests/posix/query-suite/csharp-code-quality.qls.expected index 893eaeb7560..b944b848df8 100644 --- a/csharp/ql/integration-tests/posix/query-suite/csharp-code-quality.qls.expected +++ b/csharp/ql/integration-tests/posix/query-suite/csharp-code-quality.qls.expected @@ -22,7 +22,6 @@ ql/csharp/ql/src/Concurrency/FutileSyncOnField.ql ql/csharp/ql/src/Concurrency/LockOrder.ql ql/csharp/ql/src/Concurrency/LockThis.ql ql/csharp/ql/src/Concurrency/LockedWait.ql -ql/csharp/ql/src/Dead Code/DeadStoreOfLocal.ql ql/csharp/ql/src/Language Abuse/CastThisToTypeParameter.ql ql/csharp/ql/src/Language Abuse/CatchOfGenericException.ql ql/csharp/ql/src/Language Abuse/DubiousDowncastOfThis.ql From cb3f8a8394825fb42b6e6b7f5e98033e4d6b04b9 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 15 Jul 2026 09:03:34 +0200 Subject: [PATCH 58/90] C#: Add change-note. --- .../2026-07-15-code-quality-useless-assignment.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 csharp/ql/src/change-notes/2026-07-15-code-quality-useless-assignment.md diff --git a/csharp/ql/src/change-notes/2026-07-15-code-quality-useless-assignment.md b/csharp/ql/src/change-notes/2026-07-15-code-quality-useless-assignment.md new file mode 100644 index 00000000000..d1fa51ea3bf --- /dev/null +++ b/csharp/ql/src/change-notes/2026-07-15-code-quality-useless-assignment.md @@ -0,0 +1,4 @@ +--- +category: queryMetadata +--- +* The query `cs/useless-assignment-to-local` has been removed from the `code-quality` suite, but it remains in the `code-quality-extended` suite. From 574ef4d1aca5ddce03cb4167a8e524b74077f357 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Wed, 15 Jul 2026 10:17:58 +0200 Subject: [PATCH 59/90] Kotlin: Support Kotlin 2.4.10 --- docs/codeql/reusables/supported-versions-compilers.rst | 2 +- java/kotlin-extractor/dev/wrapper.py | 2 +- .../diagnostics/kotlin-version-too-new/diagnostics.expected | 2 +- java/ql/lib/change-notes/2026-07-15-kotlin-2.4.10.md | 4 ++++ 4 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 java/ql/lib/change-notes/2026-07-15-kotlin-2.4.10.md diff --git a/docs/codeql/reusables/supported-versions-compilers.rst b/docs/codeql/reusables/supported-versions-compilers.rst index 7a88dd035c2..69def9ffeb1 100644 --- a/docs/codeql/reusables/supported-versions-compilers.rst +++ b/docs/codeql/reusables/supported-versions-compilers.rst @@ -21,7 +21,7 @@ Java,"Java 7 to 26 [6]_","javac (OpenJDK and Oracle JDK), Eclipse compiler for Java (ECJ) [7]_",``.java`` - Kotlin,"Kotlin 1.8.0 to 2.4.0\ *x*","kotlinc",``.kt`` + Kotlin,"Kotlin 1.8.0 to 2.4.1\ *x*","kotlinc",``.kt`` JavaScript,ECMAScript 2022 or lower,Not applicable,"``.js``, ``.jsx``, ``.mjs``, ``.es``, ``.es6``, ``.htm``, ``.html``, ``.xhtm``, ``.xhtml``, ``.vue``, ``.hbs``, ``.ejs``, ``.njk``, ``.json``, ``.yaml``, ``.yml``, ``.raml``, ``.xml`` [8]_" Python [9]_,"2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13",Not applicable,``.py`` Ruby [10]_,"up to 3.3",Not applicable,"``.rb``, ``.erb``, ``.gemspec``, ``Gemfile``" diff --git a/java/kotlin-extractor/dev/wrapper.py b/java/kotlin-extractor/dev/wrapper.py index 34b9d6b9425..1b29de23f76 100755 --- a/java/kotlin-extractor/dev/wrapper.py +++ b/java/kotlin-extractor/dev/wrapper.py @@ -27,7 +27,7 @@ import shutil import io import os -DEFAULT_VERSION = "2.4.0" +DEFAULT_VERSION = "2.4.10" def options(): diff --git a/java/ql/integration-tests/kotlin/all-platforms/diagnostics/kotlin-version-too-new/diagnostics.expected b/java/ql/integration-tests/kotlin/all-platforms/diagnostics/kotlin-version-too-new/diagnostics.expected index 33ef093cb9a..09429027c5d 100644 --- a/java/ql/integration-tests/kotlin/all-platforms/diagnostics/kotlin-version-too-new/diagnostics.expected +++ b/java/ql/integration-tests/kotlin/all-platforms/diagnostics/kotlin-version-too-new/diagnostics.expected @@ -1,5 +1,5 @@ { - "markdownMessage": "The Kotlin version installed (`999.999.999`) is too recent for this version of CodeQL. Install a version lower than 2.4.10.", + "markdownMessage": "The Kotlin version installed (`999.999.999`) is too recent for this version of CodeQL. Install a version lower than 2.4.20.", "severity": "error", "source": { "extractorName": "java", diff --git a/java/ql/lib/change-notes/2026-07-15-kotlin-2.4.10.md b/java/ql/lib/change-notes/2026-07-15-kotlin-2.4.10.md new file mode 100644 index 00000000000..43aa417d959 --- /dev/null +++ b/java/ql/lib/change-notes/2026-07-15-kotlin-2.4.10.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* Kotlin versions up to 2.4.10 are now supported. From 1577d82ebd9d4f528ebf5ca579b1af76956dac4f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:45:46 +0000 Subject: [PATCH 60/90] Upgrade swift-syntax-rs to swift-syntax 603.0.2 / Swift 6.3.2 --- MODULE.bazel | 2 +- unified/swift-syntax-rs/.swift-version | 2 +- unified/swift-syntax-rs/README.md | 2 +- unified/swift-syntax-rs/swift/Package.resolved | 6 +++--- unified/swift-syntax-rs/swift/Package.swift | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 4c9e3316c01..37a71583198 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -32,7 +32,7 @@ bazel_dep(name = "rules_dotnet", version = "0.21.5-codeql.1") bazel_dep(name = "googletest", version = "1.17.0.bcr.2") bazel_dep(name = "rules_rust", version = "0.69.0") bazel_dep(name = "rules_swift", version = "4.0.0-rc4") -bazel_dep(name = "swift-syntax", version = "602.0.0.bcr.2") +bazel_dep(name = "swift-syntax", version = "603.0.2") # Needed so we can `use_repo` `local_config_xcode` and # `local_config_apple_cc_toolchains` below (referenced by the per-target diff --git a/unified/swift-syntax-rs/.swift-version b/unified/swift-syntax-rs/.swift-version index 42cc526d6ca..91e4a9f2622 100644 --- a/unified/swift-syntax-rs/.swift-version +++ b/unified/swift-syntax-rs/.swift-version @@ -1 +1 @@ -6.2.4 +6.3.2 diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index 6a714137d8c..9b14d4026f3 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -98,7 +98,7 @@ The build does not depend on any particular version manager. You need: - **Rust** — pinned to `1.88` by the repo-root [`rust-toolchain.toml`](../../rust-toolchain.toml), which `rustup` picks up automatically. - **Swift** — pinned to the version in [`.swift-version`](.swift-version) - (currently `6.2.4`), used to build `swift-syntax` `602.0.0`. Install it any way + (currently `6.3.2`), used to build `swift-syntax` `603.0.2`. Install it any way you like — [swift.org](https://www.swift.org/install/) or [swiftly](https://www.swift.org/swiftly/) (which reads `.swift-version`), or a system package. Just make sure `swift` is on your `PATH` (or point `build.rs` diff --git a/unified/swift-syntax-rs/swift/Package.resolved b/unified/swift-syntax-rs/swift/Package.resolved index 9cebf34ec54..dafd46e961b 100644 --- a/unified/swift-syntax-rs/swift/Package.resolved +++ b/unified/swift-syntax-rs/swift/Package.resolved @@ -1,13 +1,13 @@ { - "originHash" : "ad84ad340ee01aa6e38b877dd29e38fa9cb40603dfdefa4e6a69e27f2db208d1", + "originHash" : "169957f8fbd882866f8f9f1c68cde813dca3560577939a8ffec366e3d77548e4", "pins" : [ { "identity" : "swift-syntax", "kind" : "remoteSourceControl", "location" : "https://github.com/swiftlang/swift-syntax.git", "state" : { - "revision" : "4799286537280063c85a32f09884cfbca301b1a1", - "version" : "602.0.0" + "revision" : "79e4b74a295b6eb74a8b585e3a39d29e70c1dbd1", + "version" : "603.0.2" } } ], diff --git a/unified/swift-syntax-rs/swift/Package.swift b/unified/swift-syntax-rs/swift/Package.swift index fcd9fcc07bc..a58b7c1a479 100644 --- a/unified/swift-syntax-rs/swift/Package.swift +++ b/unified/swift-syntax-rs/swift/Package.swift @@ -4,7 +4,7 @@ import PackageDescription let package = Package( name: "SwiftSyntaxFFI", platforms: [ - // swift-syntax 602 requires macOS 10.15; declare it explicitly + // swift-syntax 603 requires macOS 10.15; declare it explicitly // rather than relying on the swift-tools-version default (10.13). .macOS(.v10_15), ], @@ -20,7 +20,7 @@ let package = Package( dependencies: [ .package( url: "https://github.com/swiftlang/swift-syntax.git", - exact: "602.0.0" + exact: "603.0.2" ) ], targets: [ From a701922dbd0678aba6cc091b543873c5beaef448 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Tue, 14 Jul 2026 21:58:36 +0200 Subject: [PATCH 61/90] Swift: Update to Swift 6.3.3 --- swift/ql/lib/change-notes/2026-07-15-swift-6.3.3.md | 4 ++++ swift/third_party/resources/resource-dir-linux.zip | 4 ++-- swift/third_party/resources/resource-dir-macos.zip | 4 ++-- swift/third_party/resources/swift-prebuilt-linux.tar.zst | 4 ++-- swift/third_party/resources/swift-prebuilt-macos.tar.zst | 4 ++-- 5 files changed, 12 insertions(+), 8 deletions(-) create mode 100644 swift/ql/lib/change-notes/2026-07-15-swift-6.3.3.md diff --git a/swift/ql/lib/change-notes/2026-07-15-swift-6.3.3.md b/swift/ql/lib/change-notes/2026-07-15-swift-6.3.3.md new file mode 100644 index 00000000000..1542f2f68dc --- /dev/null +++ b/swift/ql/lib/change-notes/2026-07-15-swift-6.3.3.md @@ -0,0 +1,4 @@ +--- +category: majorAnalysis +--- +* Upgraded to allow analysis of Swift 6.3.3. diff --git a/swift/third_party/resources/resource-dir-linux.zip b/swift/third_party/resources/resource-dir-linux.zip index da93aefcc70..1fbff22aa74 100644 --- a/swift/third_party/resources/resource-dir-linux.zip +++ b/swift/third_party/resources/resource-dir-linux.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bd132a4fb44688913eff72f94110e2745048ceda3354ba199d8338750881e0e5 -size 408312701 +oid sha256:68da3c1158d6682f09644e4557b7bdbe74e3e7110d1a64cb61ec26b3a91d0b0e +size 408306334 diff --git a/swift/third_party/resources/resource-dir-macos.zip b/swift/third_party/resources/resource-dir-macos.zip index 11ac3ddf0d7..2a606b4771f 100644 --- a/swift/third_party/resources/resource-dir-macos.zip +++ b/swift/third_party/resources/resource-dir-macos.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fd4eaa3a688849279e990da69768da796f6a130ca6e01572d022142ff09e4868 -size 548155566 +oid sha256:da5cbdd44ee4728606f2dc0dd0718e3c7c8748ae1ec8d6261365282d54ba8db9 +size 548170081 diff --git a/swift/third_party/resources/swift-prebuilt-linux.tar.zst b/swift/third_party/resources/swift-prebuilt-linux.tar.zst index 2adf23cda68..95eabd7a2a9 100644 --- a/swift/third_party/resources/swift-prebuilt-linux.tar.zst +++ b/swift/third_party/resources/swift-prebuilt-linux.tar.zst @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e00f464ad0b793c8e14df844aecbeec5cdd587e8f34a85915593a470dba1beda -size 143508114 +oid sha256:e79d3f333cbbbeb39c226f5ea605f6a88f4f32f83e38b8c0a082d18d9caf095d +size 143468582 diff --git a/swift/third_party/resources/swift-prebuilt-macos.tar.zst b/swift/third_party/resources/swift-prebuilt-macos.tar.zst index f63c93f1caa..1914d1048a0 100644 --- a/swift/third_party/resources/swift-prebuilt-macos.tar.zst +++ b/swift/third_party/resources/swift-prebuilt-macos.tar.zst @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:212229f7f0545aab03e3034f55d13a665bc568633a5defc72a662c83a06e90cc -size 125133134 +oid sha256:f5fddabc442a5c7664be4f022f09889177f420477060c1199eb714394987ad93 +size 125119625 From 3eb353ecf76a7f61b6193129e64a9c803ecb7605 Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 15 Jul 2026 12:10:50 +0000 Subject: [PATCH 62/90] swift-syntax-rs: Address PR review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - BUILD.bazel: require x86_64 on the Linux branch of the compatibility select. The only registered standalone Linux Swift toolchain is x86_64-only, so without the CPU constraint Linux/aarch64 targets stayed "compatible" and then failed toolchain resolution instead of being skipped cleanly. - build.rs: emitting `rerun-if-changed` disables Cargo's default whole-package scan, so list every input to `swift build` — the package manifests, `Package.resolved`, `.swift-version`, and the whole Sources tree — so stale shim/toolchain output can't linger. (`.build/` is left unwatched, as it is this build's own output.) - src/lib.rs: a null result from the shim is not a parse failure — SwiftParser recovers from invalid syntax and always yields a tree — so it means the shim failed to serialize/allocate the JSON. Fix the error message and the variant doc accordingly. - README.md: `.swift-version` is not honored by the macOS Bazel build (which uses the host Xcode toolchain); document that it pins the Linux and local builds only. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- unified/swift-syntax-rs/BUILD.bazel | 15 ++++++++++++++- unified/swift-syntax-rs/README.md | 13 +++++++++---- unified/swift-syntax-rs/build.rs | 24 ++++++++++++++---------- unified/swift-syntax-rs/src/lib.rs | 9 +++++++-- 4 files changed, 44 insertions(+), 17 deletions(-) diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index be7d41eaf87..fed7df4e611 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -10,8 +10,21 @@ exports_files([".swift-version"]) # Targets in this package require a Swift toolchain (Linux or macOS). # `select()` gives us OR-of-OSes; other platforms get marked incompatible # so `bazel build/test //...` skips them cleanly. +# +# The Linux branch additionally requires x86_64: the only registered standalone +# Linux Swift toolchain (`swift_toolchain_exec_ubuntu24.04`, see //:MODULE.bazel) +# is x86_64-only. Without the CPU constraint, Linux/aarch64 targets would stay +# "compatible" and then fail toolchain resolution instead of being skipped. +config_setting( + name = "linux_x86_64", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], +) + _SWIFT_SUPPORTED_PLATFORMS = select({ - "@platforms//os:linux": [], + ":linux_x86_64": [], "@platforms//os:macos": [], "//conditions:default": ["@platforms//:incompatible"], }) diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index 9b14d4026f3..990af77d26f 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -155,10 +155,15 @@ Requirements: [`xcode_transition.bzl`](xcode_transition.bzl)), so other targets on macOS keep using Bazel's default CC toolchain. -The Swift compiler version is read from [`.swift-version`](.swift-version) by -both the Bazel toolchain (`swift.toolchain(swift_version_file = …)`) and the -local build, and is kept in sync with the `swift-syntax` release pinned in -`swift/Package.swift`, so local and CI builds behave identically. +The Swift compiler version in [`.swift-version`](.swift-version) is kept in sync +with the `swift-syntax` release pinned in `swift/Package.swift`. It is honored on +**Linux**, where the hermetic swift.org Bazel toolchain +(`swift.toolchain(swift_version_file = …)`) and the local `cargo`/`swift build` +both read it. On **macOS** it is *not* honored by the Bazel build: `rules_swift` +auto-registers the host `xcode_swift_toolchain`, which uses whichever Swift ships +with the installed Xcode and ignores `.swift-version`. So the pinned version +governs Linux (and local) builds, while the macOS compiler version depends on the +host Xcode. ## Usage diff --git a/unified/swift-syntax-rs/build.rs b/unified/swift-syntax-rs/build.rs index 3cd2e285874..c599cdf4e70 100644 --- a/unified/swift-syntax-rs/build.rs +++ b/unified/swift-syntax-rs/build.rs @@ -6,16 +6,20 @@ fn main() { let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); let swift_dir = manifest_dir.join("swift"); - println!( - "cargo:rerun-if-changed={}", - swift_dir.join("Package.swift").display() - ); - println!( - "cargo:rerun-if-changed={}", - swift_dir - .join("Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift") - .display() - ); + // Emitting any `rerun-if-changed` disables Cargo's default whole-package + // scan, so we must list every input to the `swift build` below. Watch the + // package manifests, the pinned dependency lockfile, the pinned compiler + // version, and the entire Swift sources tree (a directory is scanned + // recursively). `.build/` is intentionally *not* watched (it is this + // build's output; watching it would cause perpetual rebuilds). + for input in [ + swift_dir.join("Package.swift"), + swift_dir.join("Package.resolved"), + swift_dir.join("Sources"), + manifest_dir.join(".swift-version"), + ] { + println!("cargo:rerun-if-changed={}", input.display()); + } println!("cargo:rerun-if-env-changed=SWIFT"); println!("cargo:rerun-if-env-changed=SWIFTC"); diff --git a/unified/swift-syntax-rs/src/lib.rs b/unified/swift-syntax-rs/src/lib.rs index 2544c0bc408..2a8fe0411af 100644 --- a/unified/swift-syntax-rs/src/lib.rs +++ b/unified/swift-syntax-rs/src/lib.rs @@ -29,7 +29,10 @@ unsafe extern "C" { pub enum ParseError { /// The provided source contained an interior NUL byte. NulByte, - /// The Swift side failed to produce a result. + /// The Swift shim returned no result. `SwiftParser` recovers from invalid + /// syntax (it always produces a tree, possibly with error nodes), so this + /// does *not* indicate a syntax error in the source — it means the shim + /// failed to serialize the tree to JSON or to allocate the result string. SwiftFailure, } @@ -37,7 +40,9 @@ impl std::fmt::Display for ParseError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { ParseError::NulByte => write!(f, "source contained an interior NUL byte"), - ParseError::SwiftFailure => write!(f, "swift-syntax failed to parse the source"), + ParseError::SwiftFailure => { + write!(f, "the swift-syntax shim failed to produce a JSON result") + } } } } From 46d6a118f4598833a89be79cdf6f06816146500b Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 15 Jul 2026 12:10:50 +0000 Subject: [PATCH 63/90] yeast: Clarify Ast::with_schema registration doc The doc claimed the schema passed to `Ast::with_schema` must already have every node kind and field name registered, contradicting the registration methods just below and the swift-syntax adapter, which starts from `Schema::new()` and registers names on demand during construction. Document that up-front registration is optional. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- shared/yeast/src/lib.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index 89facfaea99..6719a5498e2 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -358,8 +358,14 @@ impl Ast { /// programmatically from a source other than a tree-sitter parse (e.g. an /// external parser's output). Populate it with [`Ast::create_node`] / /// [`Ast::create_node_with_range`] and then designate the root with - /// [`Ast::set_root`]. The `schema` must already have every node kind and - /// field name registered (see [`schema::Schema`]). + /// [`Ast::set_root`]. + /// + /// Node kind and field names may be registered in `schema` up front, or on + /// demand while building via [`Ast::register_kind`], + /// [`Ast::register_unnamed_kind`], and [`Ast::register_field`] (which return + /// the ids to pass to [`Ast::create_node_with_range`]). Passing a fresh + /// [`schema::Schema::new`] and registering names during construction is + /// therefore fine — see the swift-syntax adapter for an example. pub fn with_schema(schema: schema::Schema) -> Self { Self { root: Id(0), From 205c9a934674b327dccaca3ef4d09ba9c5ff3c11 Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 15 Jul 2026 12:19:25 +0000 Subject: [PATCH 64/90] swift-syntax-rs: Fix bazel formatting aCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- unified/swift-syntax-rs/xcode_transition.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/unified/swift-syntax-rs/xcode_transition.bzl b/unified/swift-syntax-rs/xcode_transition.bzl index c6f293a0452..b1612e762b1 100644 --- a/unified/swift-syntax-rs/xcode_transition.bzl +++ b/unified/swift-syntax-rs/xcode_transition.bzl @@ -45,6 +45,7 @@ _xcode_transition = transition( def _wrapper_impl(ctx): src = ctx.attr.actual[0] + # Forward the providers a downstream `rust_*` target reads from `deps`: # `DefaultInfo`, `CcInfo` (linking info), and `OutputGroupInfo`. providers = [src[DefaultInfo]] From 7474a33132cec2331e75c278567416f11f300606 Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 15 Jul 2026 12:47:26 +0000 Subject: [PATCH 65/90] unified: Hardcode swift_version It seems that using swift_version_file has some issues when `codeql` is consumed as a dependency module. I'm hoping hardcoding the version instead will fix this, but ideally we should find a more robust solution. --- MODULE.bazel | 13 ++++++++++--- unified/swift-syntax-rs/BUILD.bazel | 4 ---- unified/swift-syntax-rs/README.md | 23 ++++++++++++++--------- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 37a71583198..3638a07289b 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -229,12 +229,19 @@ use_repo( # Swift toolchain for building `unified/swift-syntax-rs`. On Linux we register # a hermetic swift.org toolchain as the exec toolchain; on macOS `rules_swift` # auto-registers `xcode_swift_toolchain` (host Xcode + OS-provided Swift -# runtime), which is not hermetic. Version comes from -# `unified/swift-syntax-rs/.swift-version`. +# runtime), which is not hermetic. +# +# The version is pinned as a literal rather than read from +# `unified/swift-syntax-rs/.swift-version` via `swift_version_file`: the latter +# makes the extension `module_ctx.read` a `//unified/...` label, which fails to +# resolve when this repo is consumed as a dependency module (`@@ql+`) whose +# `unified/swift-syntax-rs` package is not loadable in that context. Keep this +# in sync with `unified/swift-syntax-rs/.swift-version` (used by the `cargo` +# build) and the `swift-syntax` release in `swift/Package.swift`. swift = use_extension("@rules_swift//swift:extensions.bzl", "swift") swift.toolchain( name = "swift_toolchain", - swift_version_file = "//unified/swift-syntax-rs:.swift-version", + swift_version = "6.3.2", ) use_repo( swift, diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index fed7df4e611..02fda50cf50 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -3,10 +3,6 @@ load(":xcode_transition.bzl", "xcode_transition_swift_library") package(default_visibility = ["//visibility:public"]) -# Pinned Swift version, shared with `cargo` and `swift/Package.swift`. -# Referenced by `swift.toolchain(swift_version_file = ...)` in //:MODULE.bazel. -exports_files([".swift-version"]) - # Targets in this package require a Swift toolchain (Linux or macOS). # `select()` gives us OR-of-OSes; other platforms get marked incompatible # so `bazel build/test //...` skips them cleanly. diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index 990af77d26f..d43969199f9 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -155,15 +155,20 @@ Requirements: [`xcode_transition.bzl`](xcode_transition.bzl)), so other targets on macOS keep using Bazel's default CC toolchain. -The Swift compiler version in [`.swift-version`](.swift-version) is kept in sync -with the `swift-syntax` release pinned in `swift/Package.swift`. It is honored on -**Linux**, where the hermetic swift.org Bazel toolchain -(`swift.toolchain(swift_version_file = …)`) and the local `cargo`/`swift build` -both read it. On **macOS** it is *not* honored by the Bazel build: `rules_swift` -auto-registers the host `xcode_swift_toolchain`, which uses whichever Swift ships -with the installed Xcode and ignores `.swift-version`. So the pinned version -governs Linux (and local) builds, while the macOS compiler version depends on the -host Xcode. +The Swift compiler version is kept in sync across three places: the +[`.swift-version`](.swift-version) file (read by the local `cargo`/`swift build` +and by [swiftly](https://www.swift.org/swiftly/)), the literal `swift_version` +pinned on `swift.toolchain(...)` in the root `MODULE.bazel` (the hermetic +swift.org **Linux** Bazel toolchain), and the `swift-syntax` release in +`swift/Package.swift`. On **macOS** the version is *not* pinned by the Bazel +build: `rules_swift` auto-registers the host `xcode_swift_toolchain`, which uses +whichever Swift ships with the installed Xcode. So the pin governs Linux (and +local) builds, while the macOS compiler version depends on the host Xcode. + +(The Bazel toolchain pins a literal rather than reading `.swift-version` via +`swift_version_file`, because the latter makes the module extension read a +`//unified/...` label, which fails when this repo is consumed as a dependency +module.) ## Usage From da1bbb7fac53fb8097e317ae4f2c22ffaec6fc7d Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 15 Jul 2026 13:07:06 +0000 Subject: [PATCH 66/90] Bazel: regenerate vendored cargo dependencies Registers the `unified/swift-syntax-rs` workspace member (added in "unified: Add swift-syntax-rs") in the tree-sitter extractors crate universe. It has no external Rust dependencies, so its dependency entries are empty. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../tree_sitter_extractors_deps/BUILD.bazel | 12 ++++++++ .../tree_sitter_extractors_deps/defs.bzl | 28 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel index bb32aa97a5e..ce1d79a772b 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel @@ -553,6 +553,18 @@ alias( tags = ["manual"], ) +alias( + name = "swift-syntax-rs-0.1.0", + actual = "@vendor_ts__swift-syntax-rs-0.1.0//:swift_syntax_rs", + tags = ["manual"], +) + +alias( + name = "swift-syntax-rs", + actual = "@vendor_ts__swift-syntax-rs-0.1.0//:swift_syntax_rs", + tags = ["manual"], +) + alias( name = "syn-2.0.106", actual = "@vendor_ts__syn-2.0.106//:syn", diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl index 7fbdfc4bbd4..f3b5edb59cd 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl @@ -429,6 +429,10 @@ _NORMAL_DEPENDENCIES = { "tree-sitter-language": Label("@vendor_ts__tree-sitter-language-0.1.5//:tree_sitter_language"), }, }, + "unified/swift-syntax-rs": { + _COMMON_CONDITION: { + }, + }, } _NORMAL_ALIASES = { @@ -475,6 +479,10 @@ _NORMAL_ALIASES = { _COMMON_CONDITION: { }, }, + "unified/swift-syntax-rs": { + _COMMON_CONDITION: { + }, + }, } _NORMAL_DEV_DEPENDENCIES = { @@ -505,6 +513,8 @@ _NORMAL_DEV_DEPENDENCIES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _NORMAL_DEV_ALIASES = { @@ -532,6 +542,8 @@ _NORMAL_DEV_ALIASES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _PROC_MACRO_DEPENDENCIES = { @@ -557,6 +569,8 @@ _PROC_MACRO_DEPENDENCIES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _PROC_MACRO_ALIASES = { @@ -582,6 +596,8 @@ _PROC_MACRO_ALIASES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _PROC_MACRO_DEV_DEPENDENCIES = { @@ -607,6 +623,8 @@ _PROC_MACRO_DEV_DEPENDENCIES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _PROC_MACRO_DEV_ALIASES = { @@ -634,6 +652,8 @@ _PROC_MACRO_DEV_ALIASES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _BUILD_DEPENDENCIES = { @@ -663,6 +683,8 @@ _BUILD_DEPENDENCIES = { "tree-sitter-generate": Label("@vendor_ts__tree-sitter-generate-0.26.8//:tree_sitter_generate"), }, }, + "unified/swift-syntax-rs": { + }, } _BUILD_ALIASES = { @@ -690,6 +712,8 @@ _BUILD_ALIASES = { _COMMON_CONDITION: { }, }, + "unified/swift-syntax-rs": { + }, } _BUILD_PROC_MACRO_DEPENDENCIES = { @@ -715,6 +739,8 @@ _BUILD_PROC_MACRO_DEPENDENCIES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _BUILD_PROC_MACRO_ALIASES = { @@ -740,6 +766,8 @@ _BUILD_PROC_MACRO_ALIASES = { }, "unified/extractor/tree-sitter-swift": { }, + "unified/swift-syntax-rs": { + }, } _CONDITIONS = { From e9e360ae445c4a7cbaf00d006db7867fef2128bc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Jul 2026 11:18:13 +0000 Subject: [PATCH 67/90] update codeql documentation --- .../codeql-changelog/codeql-cli-2.26.1.rst | 98 +++++++++++++++++++ .../codeql-changelog/index.rst | 1 + 2 files changed, 99 insertions(+) create mode 100644 docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.26.1.rst diff --git a/docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.26.1.rst b/docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.26.1.rst new file mode 100644 index 00000000000..47d9bf78148 --- /dev/null +++ b/docs/codeql/codeql-overview/codeql-changelog/codeql-cli-2.26.1.rst @@ -0,0 +1,98 @@ +.. _codeql-cli-2.26.1: + +========================== +CodeQL 2.26.1 (2026-07-15) +========================== + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: none + +This is an overview of changes in the CodeQL CLI and relevant CodeQL query and library packs. For additional updates on changes to the CodeQL code scanning experience, check out the `code scanning section on the GitHub blog `__, `relevant GitHub Changelog updates `__, `changes in the CodeQL extension for Visual Studio Code `__, and the `CodeQL Action changelog `__. + +Security Coverage +----------------- + +CodeQL 2.26.1 runs a total of 497 security queries when configured with the Default suite (covering 170 CWE). The Extended suite enables an additional 131 queries (covering 32 more CWE). + +CodeQL CLI +---------- + +There are no user-facing CLI changes in this release. + +Query Packs +----------- + +Minor Analysis Improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Rust +"""" + +* The :code:`rust/hard-coded-cryptographic-value` query now treats arithmetic and bitwise operations, including string append operations, as barriers. This addresses false positive results where hard-coded constants are combined with non-constant data, such as incrementing a nonce or appending variable data to a constant prefix. + +Query Metadata Changes +~~~~~~~~~~~~~~~~~~~~~~ + +C/C++ +""""" + +* Added the tags :code:`external/cwe/cwe-073` and :code:`external/cwe/cwe-078` to :code:`cpp/uncontrolled-process-operation`. + +C# +"" + +* Added the tag :code:`external/cwe/cwe-073` to :code:`cs/assembly-path-injection`. + +Language Libraries +------------------ + +Breaking Changes +~~~~~~~~~~~~~~~~ + +C/C++ +""""" + +* Removed support for using variables as sources and sinks in models-as-data. Users of this feature should convert such sources and sinks to models defined using the QL language. + +Major Analysis Improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +C# +"" + +* Simplified and streamlined the use of NuGet sources when downloading dependencies via :code:`[mono] nuget.exe` in :code:`build-mode: none`\ : NuGet sources are now supplied via the :code:`-Source` flag instead of moving or creating :code:`nuget.config` files in the checked-out repository, private registries are used if configured, and only reachable feeds are used when NuGet feed checking is enabled (the default). + +Minor Analysis Improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Golang +"""""" + +* Improved models for the :code:`log/slog` package (Go 1.21+), including :code:`*slog.Logger` methods, :code:`With`\ /\ :code:`WithGroup`, and :code:`Attr`\ /\ :code:`Value` helpers, improving coverage for the :code:`go/log-injection` and :code:`go/clear-text-logging` queries. + +Java/Kotlin +""""""""""" + +* Regular expression checks via annotation with :code:`@javax.validation.constraints.Pattern` are now recognized as sanitizers for :code:`java/path-injection`. +* Added summary and LLM-generated source and sink models for :code:`org.apache.poi`. +* The first argument of the :code:`uri` method of :code:`WebClient$UriSpec` in :code:`org.springframework.web.reactive.function.client` is now considered a request forgery sink. Previously only the first arguments of the :code:`WebClient.create` and :code:`WebClient$Builder.baseUrl` methods were considered. This may lead to more alerts for the query :code:`java/ssrf` (Server-side request forgery). + +JavaScript/TypeScript +""""""""""""""""""""" + +* Added support for Angular's :code:`@HostListener('window:message', ...)` and :code:`@HostListener('document:message', ...)` decorators as :code:`postMessage` event handlers. The decorated method's event parameter is now recognized as a client-side remote flow source, and is considered by the :code:`js/missing-origin-check` query. + +Python +"""""" + +* :code:`Flask::FlaskApp::instance()` will now also return instances of subclasses defined in the source tree. Previously, these were filtered out. :code:`Flask::FlaskApp::classRef()` has been deprecated in favor of :code:`Flask::FlaskApp::subclassRef()` since it already returned some subclasses. + +Deprecated APIs +~~~~~~~~~~~~~~~ + +C/C++ +""""" + +* Models-as-data flow summaries now use fully qualified field names (for example, :code:`MyNamespace::MyStruct::myField`) instead of unqualified field names such as :code:`myField`. We recommend updating existing flow summaries to use fully qualified field names. Unqualified field names are still supported, but that support will be removed in a future release. diff --git a/docs/codeql/codeql-overview/codeql-changelog/index.rst b/docs/codeql/codeql-overview/codeql-changelog/index.rst index 301adddeedb..837fa757368 100644 --- a/docs/codeql/codeql-overview/codeql-changelog/index.rst +++ b/docs/codeql/codeql-overview/codeql-changelog/index.rst @@ -11,6 +11,7 @@ A list of queries for each suite and language `is available here Date: Fri, 17 Jul 2026 17:22:11 +0200 Subject: [PATCH 68/90] Shared: Add missing QLdoc --- shared/util/codeql/util/Strings.qll | 1 + 1 file changed, 1 insertion(+) diff --git a/shared/util/codeql/util/Strings.qll b/shared/util/codeql/util/Strings.qll index c82c23a9988..b032bb73672 100644 --- a/shared/util/codeql/util/Strings.qll +++ b/shared/util/codeql/util/Strings.qll @@ -1,3 +1,4 @@ +/** Provides predicates for working with strings. */ overlay[local?] module; From f384e291d70fea0f021f4f5522a7f3adad70c99d Mon Sep 17 00:00:00 2001 From: Taus Date: Mon, 20 Jul 2026 13:10:02 +0200 Subject: [PATCH 69/90] unified: Clean up build Co-authored-by: Jeroen Ketema <93738568+jketema@users.noreply.github.com> --- unified/swift-syntax-rs/BUILD.bazel | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index 02fda50cf50..216c232c888 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -7,23 +7,11 @@ package(default_visibility = ["//visibility:public"]) # `select()` gives us OR-of-OSes; other platforms get marked incompatible # so `bazel build/test //...` skips them cleanly. # -# The Linux branch additionally requires x86_64: the only registered standalone -# Linux Swift toolchain (`swift_toolchain_exec_ubuntu24.04`, see //:MODULE.bazel) -# is x86_64-only. Without the CPU constraint, Linux/aarch64 targets would stay -# "compatible" and then fail toolchain resolution instead of being skipped. -config_setting( - name = "linux_x86_64", - constraint_values = [ - "@platforms//os:linux", - "@platforms//cpu:x86_64", - ], -) - -_SWIFT_SUPPORTED_PLATFORMS = select({ - ":linux_x86_64": [], - "@platforms//os:macos": [], - "//conditions:default": ["@platforms//:incompatible"], -}) +_SWIFT_SUPPORTED_PLATFORMS = select({ + "@platforms//os:linux": [], + "@platforms//os:macos": [], + "//conditions:default": ["@platforms//:incompatible"], +}) # Swift FFI shim: wraps swift-syntax and exposes a small C ABI. The Rust # targets below link against its `CcInfo`. From 0a1139c2307232ab1df24b2dc9731213806a5ee3 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Tue, 21 Jul 2026 10:49:18 +0200 Subject: [PATCH 70/90] Fix Bazel formatting --- unified/swift-syntax-rs/BUILD.bazel | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index 216c232c888..58c34d80b78 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -7,11 +7,11 @@ package(default_visibility = ["//visibility:public"]) # `select()` gives us OR-of-OSes; other platforms get marked incompatible # so `bazel build/test //...` skips them cleanly. # -_SWIFT_SUPPORTED_PLATFORMS = select({ - "@platforms//os:linux": [], - "@platforms//os:macos": [], - "//conditions:default": ["@platforms//:incompatible"], -}) +_SWIFT_SUPPORTED_PLATFORMS = select({ + "@platforms//os:linux": [], + "@platforms//os:macos": [], + "//conditions:default": ["@platforms//:incompatible"], +}) # Swift FFI shim: wraps swift-syntax and exposes a small C ABI. The Rust # targets below link against its `CcInfo`. From bf575868bce83ad64fa43f03766b3362dc27e1b8 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Tue, 21 Jul 2026 10:53:03 +0200 Subject: [PATCH 71/90] Simplify Xcode transition per review comments --- unified/swift-syntax-rs/xcode_transition.bzl | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/unified/swift-syntax-rs/xcode_transition.bzl b/unified/swift-syntax-rs/xcode_transition.bzl index b1612e762b1..9783c0fc871 100644 --- a/unified/swift-syntax-rs/xcode_transition.bzl +++ b/unified/swift-syntax-rs/xcode_transition.bzl @@ -24,22 +24,19 @@ _EXTRA_TOOLCHAINS = "//command_line_option:extra_toolchains" def _transition_impl(settings, attr): if attr.os != "macos": - # Preserve inputs so the configuration is identical to the incoming one. + return {} + else: return { - _XCODE_VERSION_CONFIG: settings[_XCODE_VERSION_CONFIG], - _EXTRA_TOOLCHAINS: settings[_EXTRA_TOOLCHAINS], + _XCODE_VERSION_CONFIG: "@local_config_xcode//:host_xcodes", + _EXTRA_TOOLCHAINS: ( + list(settings[_EXTRA_TOOLCHAINS]) + + ["@local_config_apple_cc_toolchains//:all"] + ), } - return { - _XCODE_VERSION_CONFIG: "@local_config_xcode//:host_xcodes", - _EXTRA_TOOLCHAINS: ( - list(settings[_EXTRA_TOOLCHAINS]) + - ["@local_config_apple_cc_toolchains//:all"] - ), - } _xcode_transition = transition( implementation = _transition_impl, - inputs = [_XCODE_VERSION_CONFIG, _EXTRA_TOOLCHAINS], + inputs = [_EXTRA_TOOLCHAINS], outputs = [_XCODE_VERSION_CONFIG, _EXTRA_TOOLCHAINS], ) From cf8eeb8e44a353411f4e2ed58aa7025093ac7045 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Tue, 21 Jul 2026 11:01:37 +0200 Subject: [PATCH 72/90] Only depend on the runtime libraries from the Swift toolchain --- unified/swift-syntax-rs/BUILD.bazel | 10 ++++++++-- unified/swift-syntax-rs/swift_runtime.bzl | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 unified/swift-syntax-rs/swift_runtime.bzl diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index 58c34d80b78..34bf97a313f 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -1,8 +1,14 @@ load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test") +load(":swift_runtime.bzl", "swift_runtime_libs") load(":xcode_transition.bzl", "xcode_transition_swift_library") package(default_visibility = ["//visibility:public"]) +swift_runtime_libs( + name = "swift_runtime_libs", + toolchain = "@swift_toolchain_ubuntu24.04//:files", +) + # Targets in this package require a Swift toolchain (Linux or macOS). # `select()` gives us OR-of-OSes; other platforms get marked incompatible # so `bazel build/test //...` skips them cleanly. @@ -50,7 +56,7 @@ rust_binary( # `xcode_swift_toolchain`). data = select({ "@platforms//os:macos": [], - "@platforms//os:linux": ["@swift_toolchain_ubuntu24.04//:files"], + "@platforms//os:linux": [":swift_runtime_libs"], }), edition = "2024", target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, @@ -63,7 +69,7 @@ rust_test( crate = ":swift_syntax_rs", data = select({ "@platforms//os:macos": [], - "@platforms//os:linux": ["@swift_toolchain_ubuntu24.04//:files"], + "@platforms//os:linux": [":swift_runtime_libs"], }), edition = "2024", target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, diff --git a/unified/swift-syntax-rs/swift_runtime.bzl b/unified/swift-syntax-rs/swift_runtime.bzl new file mode 100644 index 00000000000..899d97d7116 --- /dev/null +++ b/unified/swift-syntax-rs/swift_runtime.bzl @@ -0,0 +1,19 @@ +"""Filter the Swift toolchain down to just its Linux runtime shared objects. + +The standalone toolchain's `:files` bundles the compiler, host libraries, clang +runtime, plugins, etc. For running a Swift-linked binary we only need the +runtime shared objects in `usr/lib/swift/linux/`. +""" + +def _swift_runtime_libs_impl(ctx): + libs = [ + f + for f in ctx.files.toolchain + if "/usr/lib/swift/linux/" in f.path and f.path.endswith(".so") + ] + return [DefaultInfo(files = depset(libs))] + +swift_runtime_libs = rule( + implementation = _swift_runtime_libs_impl, + attrs = {"toolchain": attr.label(allow_files = True)}, +) From 38c9c84dcbf34489c3ee06db986aa0d866ad6992 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Tue, 21 Jul 2026 11:08:46 +0200 Subject: [PATCH 73/90] Use 22.04 Swift toolchain --- MODULE.bazel | 4 ++-- unified/swift-syntax-rs/BUILD.bazel | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 3638a07289b..bf2446dcc72 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -246,11 +246,11 @@ swift.toolchain( use_repo( swift, "swift_toolchain", - "swift_toolchain_ubuntu24.04", + "swift_toolchain_ubuntu22.04", ) register_toolchains( - "@swift_toolchain//:swift_toolchain_exec_ubuntu24.04", + "@swift_toolchain//:swift_toolchain_exec_ubuntu22.04", ) # `apple_support`'s xcode_config and CC toolchains, needed by the Xcode diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index 34bf97a313f..5c2360d4deb 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -6,7 +6,7 @@ package(default_visibility = ["//visibility:public"]) swift_runtime_libs( name = "swift_runtime_libs", - toolchain = "@swift_toolchain_ubuntu24.04//:files", + toolchain = "@swift_toolchain_ubuntu22.04//:files", ) # Targets in this package require a Swift toolchain (Linux or macOS). From 661463d4433d0d3fc03121bab7167b7057fefdeb Mon Sep 17 00:00:00 2001 From: yoff Date: Mon, 20 Jul 2026 12:47:12 +0200 Subject: [PATCH 74/90] python: remove non-actionable change notes --- python/ql/lib/change-notes/2026-05-19-add-shared-cfg.md | 4 ---- .../lib/change-notes/2026-06-04-cfg-parameter-annotations.md | 4 ---- 2 files changed, 8 deletions(-) delete mode 100644 python/ql/lib/change-notes/2026-05-19-add-shared-cfg.md delete mode 100644 python/ql/lib/change-notes/2026-06-04-cfg-parameter-annotations.md diff --git a/python/ql/lib/change-notes/2026-05-19-add-shared-cfg.md b/python/ql/lib/change-notes/2026-05-19-add-shared-cfg.md deleted file mode 100644 index 913f95320d8..00000000000 --- a/python/ql/lib/change-notes/2026-05-19-add-shared-cfg.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -category: minorAnalysis ---- -* A new Python control flow graph implementation has been added under `semmle.python.controlflow.internal.Cfg` (backed by `AstNodeImpl.qll`), built on the shared `codeql.controlflow.ControlFlowGraph` library. It is not yet used by the dataflow library or any production query; the legacy CFG in `semmle/python/Flow.qll` remains the default. The new library is exposed for tests and for upcoming migrations. diff --git a/python/ql/lib/change-notes/2026-06-04-cfg-parameter-annotations.md b/python/ql/lib/change-notes/2026-06-04-cfg-parameter-annotations.md deleted file mode 100644 index 96ba81e1610..00000000000 --- a/python/ql/lib/change-notes/2026-06-04-cfg-parameter-annotations.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -category: minorAnalysis ---- -* The new (shared-CFG-based) Python control flow graph now visits parameter and return type annotations as CFG nodes for function definitions, matching the legacy CFG. This restores annotation-based type tracking through framework models such as FastAPI's `Depends()`, Pydantic request models, Starlette `WebSocket` handlers, and any other models that flow a class reference through `Parameter.getAnnotation()` to identify instances of the annotated class. From cc0156d8ace667e96fa9679e3cd82ff3266149b0 Mon Sep 17 00:00:00 2001 From: yoff Date: Tue, 21 Jul 2026 11:56:03 +0200 Subject: [PATCH 75/90] python: remove the use of toAst - AST navigation such as `getObject` and `getChild` now only moves through nodes identified via `injects` - identification such as `isSubscript` only holds for injected nodes - subclasses such as `AttrNode` are injected nodes (no, say after-nodes) --- .../python/controlflow/internal/Cfg.qll | 321 +++++++----------- 1 file changed, 130 insertions(+), 191 deletions(-) diff --git a/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll b/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll index 2d39ae8450e..ac8493f98ae 100644 --- a/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll +++ b/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll @@ -39,17 +39,6 @@ module CfgForBb implements BB::CfgSig { predicate dominatingEdge = CfgImpl::Cfg::dominatingEdge/2; } -/** - * Gets the Python AST node corresponding to CFG node `n`, if any. - * - * Multiple CFG nodes may map to the same AST node (e.g. `TBeforeNode(Call)` - * and `TAstNode(Call)` both map to `Py::Call`). This is a pure translation; - * uniqueness constraints are enforced at the dataflow layer where needed. - */ -private Py::AstNode toAst(CfgImpl::ControlFlowNode n) { - result = CfgImpl::astNodeToPyNode(n.getAstNode()) -} - /** * A control flow node. * @@ -64,7 +53,11 @@ private Py::AstNode toAst(CfgImpl::ControlFlowNode n) { */ class ControlFlowNode extends CfgImpl::ControlFlowNode { /** Gets the syntactic element corresponding to this flow node, if any. */ - Py::AstNode getNode() { result = toAst(this) } + Py::AstNode getNode() { + exists(CfgImpl::Ast::AstNode n | this.injects(n) | result = CfgImpl::astNodeToPyNode(n)) + } + + Py::Expr asPyExpr() { result = this.getNode() } /** Gets a predecessor of this flow node. */ ControlFlowNode getAPredecessor() { this = result.getASuccessor() } @@ -145,18 +138,16 @@ class ControlFlowNode extends CfgImpl::ControlFlowNode { * which holds on the destination of compound and unary assignments * even though the destination is also a write. */ - predicate isLoad() { exists(Py::Expr e | e = toAst(this) | py_expr_contexts(_, 3, e)) } + predicate isLoad() { py_expr_contexts(_, 3, this.asPyExpr()) } /** Holds if this flow node is a store (including those in augmented assignments). */ - predicate isStore() { - exists(Py::Expr e | e = toAst(this) | py_expr_contexts(_, 5, e) or augstore(_, this)) - } + predicate isStore() { py_expr_contexts(_, 5, this.asPyExpr()) or augstore(_, this) } /** Holds if this flow node is a delete. */ - predicate isDelete() { exists(Py::Expr e | e = toAst(this) | py_expr_contexts(_, 2, e)) } + predicate isDelete() { py_expr_contexts(_, 2, this.asPyExpr()) } /** Holds if this flow node is a parameter. */ - predicate isParameter() { exists(Py::Expr e | e = toAst(this) | py_expr_contexts(_, 4, e)) } + predicate isParameter() { py_expr_contexts(_, 4, this.asPyExpr()) } /** Holds if this flow node is a store in an augmented assignment. */ predicate isAugStore() { augstore(_, this) } @@ -166,45 +157,45 @@ class ControlFlowNode extends CfgImpl::ControlFlowNode { /** Holds if this flow node corresponds to a literal. */ predicate isLiteral() { - toAst(this) instanceof Py::Bytes or - toAst(this) instanceof Py::Dict or - toAst(this) instanceof Py::DictComp or - toAst(this) instanceof Py::Set or - toAst(this) instanceof Py::SetComp or - toAst(this) instanceof Py::Ellipsis or - toAst(this) instanceof Py::GeneratorExp or - toAst(this) instanceof Py::Lambda or - toAst(this) instanceof Py::ListComp or - toAst(this) instanceof Py::List or - toAst(this) instanceof Py::Num or - toAst(this) instanceof Py::Tuple or - toAst(this) instanceof Py::Unicode or - toAst(this) instanceof Py::NameConstant + this.getNode() instanceof Py::Bytes or + this.getNode() instanceof Py::Dict or + this.getNode() instanceof Py::DictComp or + this.getNode() instanceof Py::Set or + this.getNode() instanceof Py::SetComp or + this.getNode() instanceof Py::Ellipsis or + this.getNode() instanceof Py::GeneratorExp or + this.getNode() instanceof Py::Lambda or + this.getNode() instanceof Py::ListComp or + this.getNode() instanceof Py::List or + this.getNode() instanceof Py::Num or + this.getNode() instanceof Py::Tuple or + this.getNode() instanceof Py::Unicode or + this.getNode() instanceof Py::NameConstant } /** Holds if this flow node corresponds to an attribute expression. */ - predicate isAttribute() { toAst(this) instanceof Py::Attribute } + predicate isAttribute() { this.getNode() instanceof Py::Attribute } /** Holds if this flow node corresponds to a subscript expression. */ - predicate isSubscript() { toAst(this) instanceof Py::Subscript } + predicate isSubscript() { this.getNode() instanceof Py::Subscript } /** Holds if this flow node corresponds to an import member. */ - predicate isImportMember() { toAst(this) instanceof Py::ImportMember } + predicate isImportMember() { this.getNode() instanceof Py::ImportMember } /** Holds if this flow node corresponds to a call. */ - predicate isCall() { toAst(this) instanceof Py::Call } + predicate isCall() { this.getNode() instanceof Py::Call } /** Holds if this flow node corresponds to an import. */ - predicate isImport() { toAst(this) instanceof Py::ImportExpr } + predicate isImport() { this.getNode() instanceof Py::ImportExpr } /** Holds if this flow node corresponds to a conditional expression. */ - predicate isIfExp() { toAst(this) instanceof Py::IfExp } + predicate isIfExp() { this.getNode() instanceof Py::IfExp } /** Holds if this flow node corresponds to a function definition expression. */ - predicate isFunction() { toAst(this) instanceof Py::FunctionExpr } + predicate isFunction() { this.getNode() instanceof Py::FunctionExpr } /** Holds if this flow node corresponds to a class definition expression. */ - predicate isClass() { toAst(this) instanceof Py::ClassExpr } + predicate isClass() { this.getNode() instanceof Py::ClassExpr } /** * Holds if this flow node is a branch (i.e. has both a true and a @@ -223,7 +214,7 @@ class ControlFlowNode extends CfgImpl::ControlFlowNode { */ pragma[nomagic] ControlFlowNode getAChild() { - toAst(this).(Py::Expr).getAChildNode() = toAst(result) and + this.getNode().(Py::Expr).getAChildNode() = result.getNode() and result.getBasicBlock().dominates(this.getBasicBlock()) and not this instanceof UnaryExprNode } @@ -247,7 +238,7 @@ class ControlFlowNode extends CfgImpl::ControlFlowNode { * target's canonical node. */ private predicate augstore(ControlFlowNode load, ControlFlowNode store) { - exists(Py::AugAssign aa | aa.getTarget() = toAst(load)) and + exists(Py::AugAssign aa | aa.getTarget() = load.getNode()) and load = store } @@ -402,9 +393,9 @@ ControlFlowNode astExprToCfg(Py::Expr e) { result.getNode() = e } /** A control flow node corresponding to a `Name` or `PlaceHolder` expression. */ class NameNode extends ControlFlowNode { NameNode() { - toAst(this) instanceof Py::Name + this.getNode() instanceof Py::Name or - toAst(this) instanceof Py::PlaceHolder + this.getNode() instanceof Py::PlaceHolder } /** @@ -416,26 +407,26 @@ class NameNode extends ControlFlowNode { * semantics where compound assignments register both a write * (`VarWrite`) and a read (`VarRead`) on the destination. */ - predicate defines(Py::Variable v) { exists(Py::Name n | n = toAst(this) and n.defines(v)) } + predicate defines(Py::Variable v) { exists(Py::Name n | n = this.getNode() and n.defines(v)) } /** Holds if this flow node deletes the variable `v`. */ - predicate deletes(Py::Variable v) { exists(Py::Name n | n = toAst(this) and n.deletes(v)) } + predicate deletes(Py::Variable v) { exists(Py::Name n | n = this.getNode() and n.deletes(v)) } /** Holds if this flow node uses the variable `v`. */ predicate uses(Py::Variable v) { this.isLoad() and - exists(Py::Name u | u = toAst(this) and u.uses(v)) + exists(Py::Name u | u = this.getNode() and u.uses(v)) or exists(Py::PlaceHolder u | - u = toAst(this) and u.getVariable() = v and u.getCtx() instanceof Py::Load + u = this.getNode() and u.getVariable() = v and u.getCtx() instanceof Py::Load ) } /** Gets the identifier of this name node. */ string getId() { - result = toAst(this).(Py::Name).getId() + result = this.getNode().(Py::Name).getId() or - result = toAst(this).(Py::PlaceHolder).getId() + result = this.getNode().(Py::PlaceHolder).getId() } /** Holds if this is a use of a local variable. */ @@ -469,42 +460,35 @@ class NameNode extends ControlFlowNode { /** A control flow node corresponding to a named constant (`None`, `True`, `False`). */ class NameConstantNode extends NameNode { - NameConstantNode() { toAst(this) instanceof Py::NameConstant } + NameConstantNode() { this.getNode() instanceof Py::NameConstant } } /** A control flow node corresponding to a call. */ class CallNode extends ControlFlowNode { - CallNode() { toAst(this) instanceof Py::Call } + CallNode() { super.getNode() instanceof Py::Call } override Py::Call getNode() { result = super.getNode() } /** Gets the underlying Python `Call`. */ - Py::Call getCall() { result = toAst(this) } + Py::Call getCall() { result = this.getNode() } /** Gets the flow node for the function component of this call. */ ControlFlowNode getFunction() { - exists(Py::Call c | - c = toAst(this) and - c.getFunc() = toAst(result) and - result.getBasicBlock().dominates(this.getBasicBlock()) - ) + this.getCall().getFunc() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) } /** Gets the flow node for the `n`th positional argument. */ ControlFlowNode getArg(int n) { - exists(Py::Call c | - c = toAst(this) and - c.getArg(n) = toAst(result) and - result.getBasicBlock().dominates(this.getBasicBlock()) - ) + this.getCall().getArg(n) = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) } /** Gets the flow node for the named argument with name `name`. */ ControlFlowNode getArgByName(string name) { - exists(Py::Call c, Py::Keyword k | - c = toAst(this) and - k = c.getANamedArg() and - k.getValue() = toAst(result) and + exists(Py::Keyword k | + k = this.getCall().getANamedArg() and + k.getValue() = result.getNode() and k.getArg() = name and result.getBasicBlock().dominates(this.getBasicBlock()) ) @@ -515,20 +499,14 @@ class CallNode extends ControlFlowNode { /** Gets the first tuple (`*args`) argument, if any. */ ControlFlowNode getStarArg() { - exists(Py::Call c | - c = toAst(this) and - c.getStarArg() = toAst(result) and - result.getBasicBlock().dominates(this.getBasicBlock()) - ) + this.getCall().getStarArg() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) } /** Gets a dictionary (`**kwargs`) argument, if any. */ ControlFlowNode getKwargs() { - exists(Py::Call c | - c = toAst(this) and - c.getKwargs() = toAst(result) and - result.getBasicBlock().dominates(this.getBasicBlock()) - ) + this.getCall().getKwargs() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) } /** Holds if this call is a decorator call applied to a class or a function. */ @@ -536,62 +514,55 @@ class CallNode extends ControlFlowNode { /** Holds if this call is a decorator call applied to a class. */ predicate isClassDecoratorCall() { - exists(Py::ClassExpr cls | toAst(this) = cls.getADecoratorCall()) + exists(Py::ClassExpr cls | this.getNode() = cls.getADecoratorCall()) } /** Holds if this call is a decorator call applied to a function. */ predicate isFunctionDecoratorCall() { - exists(Py::FunctionExpr func | toAst(this) = func.getADecoratorCall()) + exists(Py::FunctionExpr func | this.getNode() = func.getADecoratorCall()) } } /** A control flow node corresponding to an attribute expression. */ class AttrNode extends ControlFlowNode { - AttrNode() { toAst(this) instanceof Py::Attribute } + AttrNode() { super.getNode() instanceof Py::Attribute } override Py::Attribute getNode() { result = super.getNode() } /** Gets the flow node for the object of the attribute expression. */ ControlFlowNode getObject() { - exists(Py::Attribute a | - a = toAst(this) and - a.getObject() = toAst(result) and - result.getBasicBlock().dominates(this.getBasicBlock()) - ) + this.getNode().getObject() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) } /** Gets the flow node for the object of this attribute expression, with the matching name. */ ControlFlowNode getObject(string name) { - exists(Py::Attribute a | - a = toAst(this) and - a.getObject() = toAst(result) and - a.getName() = name and - result.getBasicBlock().dominates(this.getBasicBlock()) - ) + this.getName() = name and + result = this.getObject() } /** Gets the attribute name. */ - string getName() { exists(Py::Attribute a | a = toAst(this) and a.getName() = result) } + string getName() { result = this.getNode().getName() } } /** A control flow node corresponding to an import statement (`import x`). */ class ImportExprNode extends ControlFlowNode { - ImportExprNode() { toAst(this) instanceof Py::ImportExpr } + ImportExprNode() { super.getNode() instanceof Py::ImportExpr } override Py::ImportExpr getNode() { result = super.getNode() } } /** A control flow node corresponding to a `from ... import name` expression. */ class ImportMemberNode extends ControlFlowNode { - ImportMemberNode() { toAst(this) instanceof Py::ImportMember } + ImportMemberNode() { super.getNode() instanceof Py::ImportMember } override Py::ImportMember getNode() { result = super.getNode() } /** Gets the flow node for the module being imported from, with the matching name. */ ControlFlowNode getModule(string name) { exists(Py::ImportMember i | - i = toAst(this) and - i.getModule() = toAst(result) and + i = this.getNode() and + i.getModule() = result.getNode() and i.getName() = name and result.getBasicBlock().dominates(this.getBasicBlock()) ) @@ -600,55 +571,46 @@ class ImportMemberNode extends ControlFlowNode { /** A control flow node corresponding to a `from ... import *` statement. */ class ImportStarNode extends ControlFlowNode { - ImportStarNode() { toAst(this) instanceof Py::ImportStar } + ImportStarNode() { super.getNode() instanceof Py::ImportStar } override Py::ImportStar getNode() { result = super.getNode() } /** Gets the flow node for the module being imported from. */ ControlFlowNode getModule() { - exists(Py::ImportStar i | - i = toAst(this) and - i.getModuleExpr() = toAst(result) and - result.getBasicBlock().dominates(this.getBasicBlock()) - ) + this.getNode().getModuleExpr() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) } } /** A control flow node corresponding to a subscript expression. */ class SubscriptNode extends ControlFlowNode { - SubscriptNode() { toAst(this) instanceof Py::Subscript } + SubscriptNode() { super.getNode() instanceof Py::Subscript } override Py::Subscript getNode() { result = super.getNode() } /** Gets the flow node for the value being subscripted. */ ControlFlowNode getObject() { - exists(Py::Subscript s | - s = toAst(this) and - s.getObject() = toAst(result) and - result.getBasicBlock().dominates(this.getBasicBlock()) - ) + this.getNode().getObject() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) } /** Gets the flow node for the index expression. */ ControlFlowNode getIndex() { - exists(Py::Subscript s | - s = toAst(this) and - s.getIndex() = toAst(result) and - result.getBasicBlock().dominates(this.getBasicBlock()) - ) + this.getNode().getIndex() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) } } /** A control flow node corresponding to a comparison operation. */ class CompareNode extends ControlFlowNode { - CompareNode() { toAst(this) instanceof Py::Compare } + CompareNode() { super.getNode() instanceof Py::Compare } override Py::Compare getNode() { result = super.getNode() } /** Holds if `left` and `right` are a pair of operands for this comparison. */ predicate operands(ControlFlowNode left, Py::Cmpop op, ControlFlowNode right) { exists(Py::Compare c, Py::Expr eleft, Py::Expr eright | - c = toAst(this) and eleft = toAst(left) and eright = toAst(right) + c = this.getNode() and eleft = left.getNode() and eright = right.getNode() | eleft = c.getLeft() and eright = c.getComparator(0) and op = c.getOp(0) or @@ -663,67 +625,55 @@ class CompareNode extends ControlFlowNode { /** A control flow node corresponding to a conditional expression (`x if c else y`). */ class IfExprNode extends ControlFlowNode { - IfExprNode() { toAst(this) instanceof Py::IfExp } + IfExprNode() { super.getNode() instanceof Py::IfExp } override Py::IfExp getNode() { result = super.getNode() } /** Gets the flow node for one of the value operands (true-branch or false-branch). */ ControlFlowNode getAnOperand() { exists(Py::IfExp ie | - ie = toAst(this) and - (toAst(result) = ie.getBody() or toAst(result) = ie.getOrelse()) + ie = this.getNode() and + (result.getNode() = ie.getBody() or result.getNode() = ie.getOrelse()) ) } } /** A control flow node corresponding to an assignment expression (walrus `:=`). */ class AssignmentExprNode extends ControlFlowNode { - AssignmentExprNode() { toAst(this) instanceof Py::AssignExpr } + AssignmentExprNode() { super.getNode() instanceof Py::AssignExpr } override Py::AssignExpr getNode() { result = super.getNode() } /** Gets the flow node for the left-hand side. */ ControlFlowNode getTarget() { - exists(Py::AssignExpr a | - a = toAst(this) and - a.getTarget() = toAst(result) and - result.getBasicBlock().dominates(this.getBasicBlock()) - ) + this.getNode().getTarget() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) } /** Gets the flow node for the right-hand side. */ ControlFlowNode getValue() { - exists(Py::AssignExpr a | - a = toAst(this) and - a.getValue() = toAst(result) and - result.getBasicBlock().dominates(this.getBasicBlock()) - ) + this.getNode().getValue() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) } } /** A control flow node corresponding to a binary expression (`a + b` etc.). */ class BinaryExprNode extends ControlFlowNode { - BinaryExprNode() { toAst(this) instanceof Py::BinaryExpr } + BinaryExprNode() { super.getNode() instanceof Py::BinaryExpr } override Py::BinaryExpr getNode() { result = super.getNode() } ControlFlowNode getLeft() { - exists(Py::BinaryExpr be | - be = toAst(this) and - be.getLeft() = toAst(result) and - result.getBasicBlock().dominates(this.getBasicBlock()) - ) + this.getNode().getLeft() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) } ControlFlowNode getRight() { - exists(Py::BinaryExpr be | - be = toAst(this) and - be.getRight() = toAst(result) and - result.getBasicBlock().dominates(this.getBasicBlock()) - ) + this.getNode().getRight() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) } - Py::Operator getOp() { result = toAst(this).(Py::BinaryExpr).getOp() } + Py::Operator getOp() { result = this.getNode().(Py::BinaryExpr).getOp() } /** Holds if `left` and `right` are the operands and `op` is the operator. */ predicate operands(ControlFlowNode left, Py::Operator op, ControlFlowNode right) { @@ -736,36 +686,28 @@ class BinaryExprNode extends ControlFlowNode { /** A control flow node corresponding to a boolean expression (`a and b`, `a or b`). */ class BoolExprNode extends ControlFlowNode { - BoolExprNode() { toAst(this) instanceof Py::BoolExpr } + BoolExprNode() { super.getNode() instanceof Py::BoolExpr } override Py::BoolExpr getNode() { result = super.getNode() } - Py::Boolop getOp() { result = toAst(this).(Py::BoolExpr).getOp() } + Py::Boolop getOp() { result = this.getNode().(Py::BoolExpr).getOp() } /** Gets any operand of this boolean expression. */ - ControlFlowNode getAnOperand() { - exists(Py::BoolExpr be | - be = toAst(this) and - be.getAValue() = toAst(result) - ) - } + ControlFlowNode getAnOperand() { this.getNode().getAValue() = result.getNode() } } /** A control flow node corresponding to a unary expression (`-x`, `not x`, etc.). */ class UnaryExprNode extends ControlFlowNode { - UnaryExprNode() { toAst(this) instanceof Py::UnaryExpr } + UnaryExprNode() { super.getNode() instanceof Py::UnaryExpr } override Py::UnaryExpr getNode() { result = super.getNode() } ControlFlowNode getOperand() { - exists(Py::UnaryExpr u | - u = toAst(this) and - u.getOperand() = toAst(result) and - result.getBasicBlock().dominates(this.getBasicBlock()) - ) + this.getNode().getOperand() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) } - Py::Unaryop getOp() { result = toAst(this).(Py::UnaryExpr).getOp() } + Py::Unaryop getOp() { result = this.getNode().(Py::UnaryExpr).getOp() } } /** @@ -783,12 +725,12 @@ class DefinitionNode extends ControlFlowNode { // is no AST node holding the result of `iter(next(seq))`; we use // the iter expression's CFG node as the stand-in. exists(Py::For f | - f.getTarget() = toAst(this) and - toAst(result) = f.getIter() + f.getTarget() = this.getNode() and + result.getNode() = f.getIter() ) or - exists(Py::AstNode value | value = assignedValue(toAst(this)) | - toAst(result) = value and + exists(Py::AstNode value | value = assignedValue(this.getNode()) | + result.getNode() = value and ( result.getBasicBlock().dominates(this.getBasicBlock()) or @@ -797,7 +739,7 @@ class DefinitionNode extends ControlFlowNode { // The default value for a parameter is evaluated in the same basic block as // the function definition, but the parameter belongs to the basic block of the // function, so there is no dominance relationship between the two. - exists(Py::Parameter param | toAst(this) = param.asName()) + exists(Py::Parameter param | this.getNode() = param.asName()) ) ) } @@ -857,7 +799,7 @@ class DeletionNode extends ControlFlowNode { /** A control flow node corresponding to a `for` loop target. */ class ForNode extends ControlFlowNode { - ForNode() { exists(Py::For f | toAst(this) = f.getIter()) } + ForNode() { exists(Py::For f | this.getNode() = f.getIter()) } /** Gets the iterable expression. */ ControlFlowNode getIter() { @@ -870,8 +812,8 @@ class ForNode extends ControlFlowNode { /** Gets the target (loop variable) of the `for` loop. */ ControlFlowNode getTarget() { exists(Py::For f | - f.getIter() = toAst(this) and - f.getTarget() = toAst(result) + f.getIter() = this.getNode() and + f.getTarget() = result.getNode() ) } @@ -883,29 +825,26 @@ class ForNode extends ControlFlowNode { /** A control flow node corresponding to a `raise` statement. */ class RaiseStmtNode extends ControlFlowNode { - RaiseStmtNode() { toAst(this) instanceof Py::Raise } + RaiseStmtNode() { super.getNode() instanceof Py::Raise } override Py::Raise getNode() { result = super.getNode() } /** Gets the exception expression, if any. */ ControlFlowNode getException() { - exists(Py::Raise r | - r = toAst(this) and - r.getException() = toAst(result) and - result.getBasicBlock().dominates(this.getBasicBlock()) - ) + this.getNode().getException() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) } } /** A control flow node corresponding to a starred expression (`*x`). */ class StarredNode extends ControlFlowNode { - StarredNode() { toAst(this) instanceof Py::Starred } + StarredNode() { this.getNode() instanceof Py::Starred } /** Gets the value being starred. */ ControlFlowNode getValue() { exists(Py::Starred s | - s = toAst(this) and - s.getValue() = toAst(result) and + s = this.getNode() and + s.getValue() = result.getNode() and result.getBasicBlock().dominates(this.getBasicBlock()) ) } @@ -913,7 +852,7 @@ class StarredNode extends ControlFlowNode { /** A control flow node corresponding to an `except` clause's name binding. */ class ExceptFlowNode extends ControlFlowNode { - ExceptFlowNode() { exists(Py::ExceptStmt e | toAst(this) = e.getName()) } + ExceptFlowNode() { exists(Py::ExceptStmt e | this.getNode() = e.getName()) } /** Gets the CFG node for the bound `as`-name itself. */ ControlFlowNode getName() { result = this } @@ -921,8 +860,8 @@ class ExceptFlowNode extends ControlFlowNode { /** Gets the type expression of this exception handler. */ ControlFlowNode getType() { exists(Py::ExceptStmt e | - e.getName() = toAst(this) and - e.getType() = toAst(result) and + e.getName() = this.getNode() and + e.getType() = result.getNode() and result.getBasicBlock().dominates(this.getBasicBlock()) ) } @@ -930,7 +869,7 @@ class ExceptFlowNode extends ControlFlowNode { /** A control flow node corresponding to an `except*` clause's name binding. */ class ExceptGroupFlowNode extends ControlFlowNode { - ExceptGroupFlowNode() { exists(Py::ExceptGroupStmt e | toAst(this) = e.getName()) } + ExceptGroupFlowNode() { exists(Py::ExceptGroupStmt e | this.getNode() = e.getName()) } /** Gets the CFG node for the bound `as`-name itself. */ ControlFlowNode getName() { result = this } @@ -947,12 +886,12 @@ abstract class SequenceNode extends ControlFlowNode { /** A control flow node corresponding to a tuple literal. */ class TupleNode extends SequenceNode { - TupleNode() { toAst(this) instanceof Py::Tuple } + TupleNode() { this.getNode() instanceof Py::Tuple } override ControlFlowNode getElement(int n) { exists(Py::Tuple t | - t = toAst(this) and - t.getElt(n) = toAst(result) and + t = this.getNode() and + t.getElt(n) = result.getNode() and result.getBasicBlock().dominates(this.getBasicBlock()) ) } @@ -960,12 +899,12 @@ class TupleNode extends SequenceNode { /** A control flow node corresponding to a list literal. */ class ListNode extends SequenceNode { - ListNode() { toAst(this) instanceof Py::List } + ListNode() { this.getNode() instanceof Py::List } override ControlFlowNode getElement(int n) { exists(Py::List l | - l = toAst(this) and - l.getElt(n) = toAst(result) and + l = this.getNode() and + l.getElt(n) = result.getNode() and result.getBasicBlock().dominates(this.getBasicBlock()) ) } @@ -973,13 +912,13 @@ class ListNode extends SequenceNode { /** A control flow node corresponding to a set literal. */ class SetNode extends ControlFlowNode { - SetNode() { toAst(this) instanceof Py::Set } + SetNode() { this.getNode() instanceof Py::Set } /** Gets the flow node for an element of the set. */ ControlFlowNode getAnElement() { exists(Py::Set s | - s = toAst(this) and - s.getAnElt() = toAst(result) and + s = this.getNode() and + s.getAnElt() = result.getNode() and result.getBasicBlock().dominates(this.getBasicBlock()) ) } @@ -987,13 +926,13 @@ class SetNode extends ControlFlowNode { /** A control flow node corresponding to a dict literal. */ class DictNode extends ControlFlowNode { - DictNode() { toAst(this) instanceof Py::Dict } + DictNode() { this.getNode() instanceof Py::Dict } /** Gets the flow node for a key of the dict. */ ControlFlowNode getAKey() { exists(Py::Dict d | - d = toAst(this) and - d.getAKey() = toAst(result) and + d = this.getNode() and + d.getAKey() = result.getNode() and result.getBasicBlock().dominates(this.getBasicBlock()) ) } @@ -1001,8 +940,8 @@ class DictNode extends ControlFlowNode { /** Gets the flow node for a value of the dict. */ ControlFlowNode getAValue() { exists(Py::Dict d | - d = toAst(this) and - d.getAValue() = toAst(result) and + d = this.getNode() and + d.getAValue() = result.getNode() and result.getBasicBlock().dominates(this.getBasicBlock()) ) } From a683b641ccb99fd23d0625049ae5d8a081974818 Mon Sep 17 00:00:00 2001 From: yoff Date: Tue, 21 Jul 2026 12:05:37 +0200 Subject: [PATCH 76/90] python: removed superflous module We have CfgImpl::Cfg already. --- .../semmle/python/controlflow/internal/Cfg.qll | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll b/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll index ac8493f98ae..ec8b44d5a82 100644 --- a/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll +++ b/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll @@ -20,24 +20,6 @@ module; private import python as Py private import semmle.python.controlflow.internal.AstNodeImpl as CfgImpl private import codeql.controlflow.SuccessorType -private import codeql.controlflow.BasicBlock as BB - -/** - * A nested sub-module that explicitly implements `BB::CfgSig`, so this - * `Cfg` facade can be passed to parameterised shared modules such as - * `codeql.dataflow.VariableCapture::Flow`. The sub-module - * exposes the *raw* shared-CFG types from `AstNodeImpl.qll` (where the - * signature is satisfied natively), not the facade's wrapped types. - */ -module CfgForBb implements BB::CfgSig { - class ControlFlowNode = CfgImpl::ControlFlowNode; - - class BasicBlock = CfgImpl::BasicBlock; - - class EntryBasicBlock = CfgImpl::Cfg::EntryBasicBlock; - - predicate dominatingEdge = CfgImpl::Cfg::dominatingEdge/2; -} /** * A control flow node. From c6448548a3d6f260b4bf6301a073b2749d03c912 Mon Sep 17 00:00:00 2001 From: Scott Talbot Date: Wed, 22 Jul 2026 17:42:39 +1000 Subject: [PATCH 77/90] JS: Recognize @fastify/rate-limit as a rate limiter --- .../javascript/security/dataflow/MissingRateLimiting.qll | 4 +++- .../ql/src/change-notes/2026-07-22-fastify-rate-limit.md | 4 ++++ .../CWE-770/MissingRateLimit/MissingRateLimiting.expected | 3 ++- .../query-tests/Security/CWE-770/MissingRateLimit/tst.js | 8 +++++++- 4 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 javascript/ql/src/change-notes/2026-07-22-fastify-rate-limit.md diff --git a/javascript/ql/lib/semmle/javascript/security/dataflow/MissingRateLimiting.qll b/javascript/ql/lib/semmle/javascript/security/dataflow/MissingRateLimiting.qll index 8dd9c483144..5cac49d6dd6 100644 --- a/javascript/ql/lib/semmle/javascript/security/dataflow/MissingRateLimiting.qll +++ b/javascript/ql/lib/semmle/javascript/security/dataflow/MissingRateLimiting.qll @@ -189,7 +189,9 @@ class RouteHandlerLimitedByRateLimiterFlexible extends RateLimitingMiddleware in { } private class FastifyRateLimiter extends RateLimitingMiddleware { - FastifyRateLimiter() { this = DataFlow::moduleImport("fastify-rate-limit") } + FastifyRateLimiter() { + this = DataFlow::moduleImport(["fastify-rate-limit", "@fastify/rate-limit"]) + } } /** diff --git a/javascript/ql/src/change-notes/2026-07-22-fastify-rate-limit.md b/javascript/ql/src/change-notes/2026-07-22-fastify-rate-limit.md new file mode 100644 index 00000000000..55e23b9dc11 --- /dev/null +++ b/javascript/ql/src/change-notes/2026-07-22-fastify-rate-limit.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* The `js/missing-rate-limiting` query now recognizes the `@fastify/rate-limit` package as a rate limiter. diff --git a/javascript/ql/test/query-tests/Security/CWE-770/MissingRateLimit/MissingRateLimiting.expected b/javascript/ql/test/query-tests/Security/CWE-770/MissingRateLimit/MissingRateLimiting.expected index 8d197d6e37f..5e2265f64b4 100644 --- a/javascript/ql/test/query-tests/Security/CWE-770/MissingRateLimit/MissingRateLimiting.expected +++ b/javascript/ql/test/query-tests/Security/CWE-770/MissingRateLimit/MissingRateLimiting.expected @@ -9,4 +9,5 @@ | tst.js:64:25:64:63 | functio ... req); } | This route handler performs $@, but is not rate-limited. | tst.js:64:46:64:60 | verifyUser(req) | authorization | | tst.js:76:25:76:53 | catchAs ... ndler1) | This route handler performs $@, but is not rate-limited. | tst.js:14:40:14:46 | login() | authorization | | tst.js:88:24:88:40 | expensiveHandler1 | This route handler performs $@, but is not rate-limited. | tst.js:14:40:14:46 | login() | authorization | -| tst.js:112:28:112:44 | expensiveHandler1 | This route handler performs $@, but is not rate-limited. | tst.js:14:40:14:46 | login() | authorization | +| tst.js:111:28:111:44 | expensiveHandler1 | This route handler performs $@, but is not rate-limited. | tst.js:14:40:14:46 | login() | authorization | +| tst.js:116:39:116:55 | expensiveHandler1 | This route handler performs $@, but is not rate-limited. | tst.js:14:40:14:46 | login() | authorization | diff --git a/javascript/ql/test/query-tests/Security/CWE-770/MissingRateLimit/tst.js b/javascript/ql/test/query-tests/Security/CWE-770/MissingRateLimit/tst.js index 5b4312bbbe0..7ff0c067fb5 100644 --- a/javascript/ql/test/query-tests/Security/CWE-770/MissingRateLimit/tst.js +++ b/javascript/ql/test/query-tests/Security/CWE-770/MissingRateLimit/tst.js @@ -91,7 +91,6 @@ fastifyApp.get('/bar', expensiveHandler1); // Fastify per-route rate limiting via config.rateLimit const fastifyApp2 = require('fastify')(); -fastifyApp2.register(require('@fastify/rate-limit')); fastifyApp2.post('/login', { config: { @@ -110,3 +109,10 @@ fastifyApp2.post('/signup', { }, expensiveHandler1); // OK - has per-route rateLimit directly in options fastifyApp2.post('/other', expensiveHandler1); // $ Alert - no rate limiting + +// rate limiting using the scoped package name +const fastifyApp3 = require('fastify')(); + +fastifyApp3.get('/before-rate-limit', expensiveHandler1); // $ Alert +fastifyApp3.register(require('@fastify/rate-limit')); +fastifyApp3.get('/after-rate-limit', expensiveHandler1); From 8facacd6a106dfc6d9351877a77d3c934ad91815 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 12:14:51 +0000 Subject: [PATCH 78/90] build(deps): bump rules_nodejs from 6.7.3 to 6.7.5 Bumps [rules_nodejs](https://github.com/bazel-contrib/rules_nodejs) from 6.7.3 to 6.7.5. - [Release notes](https://github.com/bazel-contrib/rules_nodejs/releases) - [Changelog](https://github.com/bazel-contrib/rules_nodejs/blob/main/CHANGELOG.md) - [Commits](https://github.com/bazel-contrib/rules_nodejs/compare/v6.7.3...v6.7.5) --- updated-dependencies: - dependency-name: rules_nodejs dependency-version: 6.7.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 24b1c757e37..7781b42d151 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -19,7 +19,7 @@ bazel_dep(name = "rules_cc", version = "0.2.17") bazel_dep(name = "rules_go", version = "0.60.0") bazel_dep(name = "rules_java", version = "9.6.1") bazel_dep(name = "rules_pkg", version = "1.2.0") -bazel_dep(name = "rules_nodejs", version = "6.7.3") +bazel_dep(name = "rules_nodejs", version = "6.7.5") bazel_dep(name = "rules_python", version = "1.9.0") bazel_dep(name = "rules_shell", version = "0.7.1") bazel_dep(name = "bazel_skylib", version = "1.9.0") From 90c1ddfd497d7397679edec95338f410999a6583 Mon Sep 17 00:00:00 2001 From: yoff Date: Mon, 27 Jul 2026 16:24:46 +0200 Subject: [PATCH 79/90] Update python/ql/lib/ide-contextual-queries/printCfg.ql Co-authored-by: Tom Hvitved --- python/ql/lib/ide-contextual-queries/printCfg.ql | 1 - 1 file changed, 1 deletion(-) diff --git a/python/ql/lib/ide-contextual-queries/printCfg.ql b/python/ql/lib/ide-contextual-queries/printCfg.ql index 6e325e84bb7..819022c750c 100644 --- a/python/ql/lib/ide-contextual-queries/printCfg.ql +++ b/python/ql/lib/ide-contextual-queries/printCfg.ql @@ -8,7 +8,6 @@ */ import semmle.python.Files as Files -// import semmle.python.Scope import semmle.python.controlflow.internal.AstNodeImpl external string selectedSourceFile(); From 1068add0ed97454bb72e980cd4b21d742a3a55c2 Mon Sep 17 00:00:00 2001 From: yoff Date: Tue, 21 Jul 2026 13:39:48 +0200 Subject: [PATCH 80/90] Split tests with disjoint concerns --- .../ql/test/library-tests/ControlFlow/bindings/starred.py | 4 ++++ .../ControlFlow/bindings/{walrus_starred.py => walrus.py} | 7 +------ 2 files changed, 5 insertions(+), 6 deletions(-) create mode 100644 python/ql/test/library-tests/ControlFlow/bindings/starred.py rename python/ql/test/library-tests/ControlFlow/bindings/{walrus_starred.py => walrus.py} (62%) diff --git a/python/ql/test/library-tests/ControlFlow/bindings/starred.py b/python/ql/test/library-tests/ControlFlow/bindings/starred.py new file mode 100644 index 00000000000..ac38c74ef12 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/starred.py @@ -0,0 +1,4 @@ +# Starred-target edge cases — wired in the new CFG. + +# Starred target in a Tuple LHS. +*head, tail = [1, 2, 3] # $ cfgdefines=head cfgdefines=tail diff --git a/python/ql/test/library-tests/ControlFlow/bindings/walrus_starred.py b/python/ql/test/library-tests/ControlFlow/bindings/walrus.py similarity index 62% rename from python/ql/test/library-tests/ControlFlow/bindings/walrus_starred.py rename to python/ql/test/library-tests/ControlFlow/bindings/walrus.py index 5c0c1bd8319..d370fd342dc 100644 --- a/python/ql/test/library-tests/ControlFlow/bindings/walrus_starred.py +++ b/python/ql/test/library-tests/ControlFlow/bindings/walrus.py @@ -1,4 +1,4 @@ -# Walrus and starred-target edge cases — wired in the new CFG. +# Walrus edge cases — wired in the new CFG. # Walrus in expression context. if (y := 5) > 0: # $ cfgdefines=y @@ -7,8 +7,3 @@ if (y := 5) > 0: # $ cfgdefines=y # Walrus in a comprehension. The comprehension introduces a synthetic # `.0` parameter bound to the iterable. _ = [w for _ in range(3) if (w := 1)] # $ cfgdefines=_ cfgdefines=w cfgdefines=.0 - -# Starred target in a Tuple LHS. -*head, tail = [1, 2, 3] # $ cfgdefines=head cfgdefines=tail - - From 2c0ed98cb2a3fa90a1e31612d3795cfb829460c7 Mon Sep 17 00:00:00 2001 From: yoff Date: Tue, 21 Jul 2026 13:41:29 +0200 Subject: [PATCH 81/90] python: remove uninformative headers --- .../evaluation-order/NewCfgBasicBlockAnnotationGap.ql | 1 - .../ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.ql | 1 - .../ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.ql | 1 - .../ControlFlow/evaluation-order/NewCfgNeverReachable.ql | 1 - .../ControlFlow/evaluation-order/NewCfgNoBackwardFlow.ql | 1 - .../ControlFlow/evaluation-order/NewCfgNoSharedReachable.ql | 1 - .../ControlFlow/evaluation-order/NewCfgStrictForward.ql | 1 - 7 files changed, 7 deletions(-) diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.ql index 80dd759a365..52fbda8cfaa 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.ql @@ -1,7 +1,6 @@ /** * New-CFG version of BasicBlockAnnotationGap. * - * Original: * Checks that within a basic block, if a node is annotated then its * successor is also annotated (or excluded). A gap in annotations * within a basic block indicates a missing annotation, since there diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.ql index f06d08d937e..27288dcaa67 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.ql @@ -1,7 +1,6 @@ /** * New-CFG version of BasicBlockOrdering. * - * Original: * Checks that within a single basic block, annotations appear in * increasing minimum-timestamp order. */ diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.ql index 8e52663d6ea..657fc80437c 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.ql @@ -1,7 +1,6 @@ /** * New-CFG version of ConsecutiveTimestamps. * - * Original: * Checks that consecutive annotated nodes have consecutive timestamps: * for each annotation with timestamp `a`, some CFG node for that annotation * must have a next annotation containing `a + 1`. diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.ql index 6949b2cc6e9..f10df59c34e 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.ql @@ -1,7 +1,6 @@ /** * New-CFG version of NeverReachable. * - * Original: * Checks that expressions annotated with `t.never` either have no CFG * node, or if they do, that the node is not reachable from its scope's * entry (including within the same basic block). diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.ql index 442ca5f5456..a45e01b30c1 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.ql @@ -1,7 +1,6 @@ /** * New-CFG version of NoBackwardFlow. * - * Original: * Checks that time never flows backward between consecutive timer annotations * in the CFG. For each pair of consecutive annotated nodes (A -> B), there must * exist timestamps a in A and b in B with a < b. diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.ql index 5a1a1aba2a7..30d250d32ab 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.ql @@ -1,7 +1,6 @@ /** * New-CFG version of NoSharedReachable. * - * Original: * Checks that two annotations sharing a timestamp value are on * mutually exclusive CFG paths (neither can reach the other). */ diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.ql index ebbc60346db..c93d181d852 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.ql @@ -1,7 +1,6 @@ /** * New-CFG version of StrictForward. * - * Original: * Stronger version of NoBackwardFlow: for consecutive annotated nodes * A -> B that both have a single timestamp (non-loop code) and B does * NOT dominate A (forward edge), requires max(A) < min(B). From aa305c20b12e835820e751c874159ba91b0110e2 Mon Sep 17 00:00:00 2001 From: yoff Date: Thu, 23 Jul 2026 18:52:24 +0200 Subject: [PATCH 82/90] Python: mark definition of type ascription as `SPURIOUS` --- .../ql/test/library-tests/ControlFlow/bindings/type_params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ql/test/library-tests/ControlFlow/bindings/type_params.py b/python/ql/test/library-tests/ControlFlow/bindings/type_params.py index 2bd34dc3f0e..ae48ccf130b 100644 --- a/python/ql/test/library-tests/ControlFlow/bindings/type_params.py +++ b/python/ql/test/library-tests/ControlFlow/bindings/type_params.py @@ -8,7 +8,7 @@ def func[T](x: T) -> T: # $ cfgdefines=func cfgdefines=x class Box[T]: # $ cfgdefines=Box - item: T # $ cfgdefines=item + item: T # $ SPURIOUS: cfgdefines=item # Multi-parameter, with bound and variadics. From 05aa8debc442151180fefd7c1daf4a99258e09a4 Mon Sep 17 00:00:00 2001 From: yoff Date: Mon, 27 Jul 2026 16:23:30 +0200 Subject: [PATCH 83/90] python: make `LoopStmt` final --- .../python/controlflow/internal/AstNodeImpl.qll | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll b/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll index 8199008e88c..891ba8c1318 100644 --- a/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll +++ b/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll @@ -593,8 +593,10 @@ module Ast implements AstSig { } /** A loop statement. */ - class LoopStmt extends Stmt { - LoopStmt() { + final class LoopStmt = LoopStmtImpl; + + abstract private class LoopStmtImpl extends Stmt { + LoopStmtImpl() { this = TPyStmt(any(Py::While w)) or this = TPyStmt(any(Py::For f)) @@ -605,7 +607,7 @@ module Ast implements AstSig { } /** A `while` loop statement. */ - class WhileStmt extends LoopStmt { + class WhileStmt extends LoopStmtImpl { private Py::While whileStmt; WhileStmt() { this = TPyStmt(whileStmt) } @@ -630,21 +632,21 @@ module Ast implements AstSig { /** * A `do-while` loop statement. Python has no do-while construct. */ - class DoStmt extends LoopStmt { + class DoStmt extends LoopStmtImpl { DoStmt() { none() } Expr getCondition() { none() } } /** An `until` loop. Python has no `until` loop. */ - class UntilStmt extends LoopStmt { + class UntilStmt extends LoopStmtImpl { UntilStmt() { none() } Expr getCondition() { none() } } /** A C-style `for` loop. Python has no C-style for loop. */ - class ForStmt extends LoopStmt { + class ForStmt extends LoopStmtImpl { ForStmt() { none() } AstNode getInit(int index) { none() } @@ -655,7 +657,7 @@ module Ast implements AstSig { } /** A for-each loop (`for x in iterable:`). */ - class ForeachStmt extends LoopStmt { + class ForeachStmt extends LoopStmtImpl { private Py::For forStmt; ForeachStmt() { this = TPyStmt(forStmt) } From fa94dc9a329d593c8f8069b551138d3219ba7e5b Mon Sep 17 00:00:00 2001 From: yoff Date: Mon, 27 Jul 2026 16:31:41 +0200 Subject: [PATCH 84/90] Python: remove reference to line numbers --- .../ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll b/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll index 891ba8c1318..31bcf19b434 100644 --- a/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll +++ b/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll @@ -156,7 +156,7 @@ module Ast implements AstSig { /** * A parameter of a callable. * - * Modelled per the C# template (`csharp/.../ControlFlowGraph.qll:147-156`): + * Modelled per the C# template (`csharp/.../ControlFlowGraph.qll`): * each Python parameter (the `Py::Parameter` AST node, which is a `Name` * or — Python 2 only — a `Tuple` in store context) becomes a CFG node * at a stable position in the enclosing callable's entry sequence. From 665e046ae6d505e4e0d2ce80c1530815c91d258f Mon Sep 17 00:00:00 2001 From: yoff Date: Mon, 27 Jul 2026 17:00:35 +0200 Subject: [PATCH 85/90] Python: add overlay annotations to inlined predicates --- python/ql/lib/semmle/python/controlflow/internal/Cfg.qll | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll b/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll index ec8b44d5a82..4dc6ac20d04 100644 --- a/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll +++ b/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll @@ -85,10 +85,12 @@ class ControlFlowNode extends CfgImpl::ControlFlowNode { } /** Holds if this strictly dominates `other`. */ + overlay[caller?] pragma[inline] predicate strictlyDominates(ControlFlowNode other) { super.strictlyDominates(other) } /** Holds if this dominates `other` (reflexively). */ + overlay[caller?] pragma[inline] predicate dominates(ControlFlowNode other) { super.dominates(other) } From 3206ab0be19038c6d91bc34493d6c3c4101067c2 Mon Sep 17 00:00:00 2001 From: yoff Date: Mon, 27 Jul 2026 17:37:36 +0200 Subject: [PATCH 86/90] Python: address CodeQL alerts --- .../controlflow/internal/AstNodeImpl.qll | 18 +++++++++--------- .../semmle/python/controlflow/internal/Cfg.qll | 2 +- .../evaluation-order/NewCfgAllLiveReachable.ql | 1 - .../NewCfgAnnotationHasCfgNode.ql | 1 - .../NewCfgBasicBlockAnnotationGap.ql | 1 - .../NewCfgBasicBlockOrdering.ql | 1 - .../evaluation-order/NewCfgBranchTimestamps.ql | 3 +-- .../NewCfgConsecutivePredecessorTimestamps.ql | 1 - .../NewCfgConsecutiveTimestamps.ql | 1 - .../evaluation-order/NewCfgImpl.qll | 4 +--- .../evaluation-order/NewCfgNeverReachable.ql | 1 - .../evaluation-order/NewCfgNoBackwardFlow.ql | 1 - .../evaluation-order/NewCfgNoBasicBlock.ql | 1 - .../NewCfgNoSharedReachable.ql | 1 - .../evaluation-order/NewCfgStrictForward.ql | 1 - .../evaluation-order/OldCfgImpl.qll | 8 ++++---- 16 files changed, 16 insertions(+), 30 deletions(-) diff --git a/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll b/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll index 31bcf19b434..21dbad68b81 100644 --- a/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll +++ b/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll @@ -156,7 +156,7 @@ module Ast implements AstSig { /** * A parameter of a callable. * - * Modelled per the C# template (`csharp/.../ControlFlowGraph.qll`): + * modeled per the C# template (`csharp/.../ControlFlowGraph.qll`): * each Python parameter (the `Py::Parameter` AST node, which is a `Name` * or — Python 2 only — a `Tuple` in store context) becomes a CFG node * at a stable position in the enclosing callable's entry sequence. @@ -766,7 +766,7 @@ module Ast implements AstSig { /** * A `from m import *` statement. Evaluates the module expression but * binds no name (the bindings happen by side-effect at runtime, which - * is not modelled at the CFG level). + * is not modeled at the CFG level). */ additional class ImportStarStmt extends Stmt { private Py::ImportStar imp; @@ -921,7 +921,7 @@ module Ast implements AstSig { * latter flattens a tuple of exception types (`except (A, B):`) into * its individual elements, which would yield several patterns for one * handler and violate the shared CFG's single-pattern-per-catch - * contract. The raw child is the one tuple node, modelling the + * contract. The raw child is the one tuple node, modeling the * runtime's single `isinstance(exc, (A, B))` test. */ AstNode getPattern() { @@ -1231,7 +1231,7 @@ module Ast implements AstSig { } /** - * An `import x.y` module expression. Modelled as a leaf — the dotted + * An `import x.y` module expression. modeled as a leaf — the dotted * name is just a string. */ additional class ImportExpression extends Expr { @@ -1629,9 +1629,9 @@ private module Input implements InputSig1, InputSig2 { private string assertThrowTag() { result = "[assert-throw]" } /** - * Holds if the AST node `n` may raise an exception at runtime as part of + * Holds if the expression node `e` may raise an exception at runtime as part of * its normal evaluation (not via an explicit `raise`/`assert`, which are - * modelled separately). + * modeled separately). * * The set mirrors what the legacy CFG used to flag implicitly: function * calls (anything can raise), attribute access (`AttributeError`), @@ -1640,7 +1640,7 @@ private module Input implements InputSig1, InputSig2 { * (`ImportError`/`ModuleNotFoundError`), and generator/coroutine * suspension points (`await`/`yield`/`yield from`). * - * Bare `Name` reads are intentionally excluded — modelling every name + * Bare `Name` reads are intentionally excluded — modeling every name * read as `mayThrow` would explode CFG edge count for negligible * analysis value. `BoolExpr`/`IfExp` containers are also excluded; the * operands they evaluate contribute their own exception edges. @@ -1677,7 +1677,7 @@ private module Input implements InputSig1, InputSig2 { private predicate stmtMayThrow(Py::Stmt s) { s instanceof Py::ImportStar } /** - * Holds if `n` is syntactically inside the body, handlers, `else`, or + * Holds if `py` is syntactically inside the body, handlers, `else`, or * `finally` of a `try` statement (or the body of a `with` statement, * which compiles to an implicit try/finally for `__exit__`) in the * same scope. @@ -1701,7 +1701,7 @@ private module Input implements InputSig1, InputSig2 { * `exprMayThrow` and `stmtMayThrow` for the included AST classes. * * Restricted to nodes inside a `try`/`with` statement: matches Java's - * approach of only modelling exception flow where it can be observed + * approach of only modeling exception flow where it can be observed * by local handling. */ private predicate mayThrow(Ast::AstNode n) { diff --git a/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll b/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll index 4dc6ac20d04..b102c71391f 100644 --- a/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll +++ b/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll @@ -859,7 +859,7 @@ class ExceptGroupFlowNode extends ControlFlowNode { ControlFlowNode getName() { result = this } } -/** Abstract base class for sequence nodes (tuple, list). */ +/** A control flow node corresponding to a sequence (tuple, list). */ abstract class SequenceNode extends ControlFlowNode { /** Gets the `n`th element of this sequence. */ abstract ControlFlowNode getElement(int n); diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAllLiveReachable.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAllLiveReachable.ql index 75f02d14a9c..883b266c82e 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAllLiveReachable.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAllLiveReachable.ql @@ -1,7 +1,6 @@ /** New-CFG version of AllLiveReachable. */ import python -import TimerUtils import NewCfgImpl private module Utils = EvalOrderCfgUtils; diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAnnotationHasCfgNode.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAnnotationHasCfgNode.ql index 4b1d82e27e6..bf97a079322 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAnnotationHasCfgNode.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAnnotationHasCfgNode.ql @@ -5,7 +5,6 @@ */ import python -import TimerUtils import NewCfgImpl private module Utils = EvalOrderCfgUtils; diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.ql index 52fbda8cfaa..e12ab8ff5ef 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.ql @@ -11,7 +11,6 @@ */ import python -import TimerUtils import NewCfgImpl private module Utils = EvalOrderCfgUtils; diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.ql index 27288dcaa67..b8254a8e864 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.ql @@ -6,7 +6,6 @@ */ import python -import TimerUtils import NewCfgImpl private module Utils = EvalOrderCfgUtils; diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBranchTimestamps.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBranchTimestamps.ql index cfd8ffb4e4b..2dc8ae144dc 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBranchTimestamps.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBranchTimestamps.ql @@ -16,7 +16,7 @@ * * `match` statements: each `case` body is a syntactically distinct * sub-tree, and the branches don't reconverge through a common * annotation point in the timeline; - * * `try` / `with` and `raise` / `assert`: exception edges are modelled + * * `try` / `with` and `raise` / `assert`: exception edges are modeled * as true/false but flow to syntactically distinct handlers, with no * reconvergence in the linear annotation order; * * short-circuit `and` / `or` (`BoolExpr`): the branches reconverge at @@ -33,7 +33,6 @@ */ import python -import TimerUtils import NewCfgImpl private module Utils = EvalOrderCfgUtils; diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.ql index 3feacae264e..0eb8ad5a414 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.ql @@ -8,7 +8,6 @@ */ import python -import TimerUtils import NewCfgImpl private module Utils = EvalOrderCfgUtils; diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.ql index 657fc80437c..8fc13c9acba 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.ql @@ -14,7 +14,6 @@ */ import python -import TimerUtils import NewCfgImpl private module Utils = EvalOrderCfgUtils; diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgImpl.qll b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgImpl.qll index cbecb2da19d..ae27e8f16a0 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgImpl.qll +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgImpl.qll @@ -74,9 +74,7 @@ module NewCfg implements EvalOrderCfgSig { Py::Scope getScope() { result = NewControlFlowNode.super.getEnclosingCallable().asScope() } - BasicBlock getBasicBlock() { - exists(NewBasicBlock bb, int i | bb.getNode(i) = this and result = bb) - } + BasicBlock getBasicBlock() { exists(NewBasicBlock bb | bb.getNode(_) = this and result = bb) } } /** diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.ql index f10df59c34e..db324e6ee76 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.ql @@ -7,7 +7,6 @@ */ import python -import TimerUtils import NewCfgImpl private module Utils = EvalOrderCfgUtils; diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.ql index a45e01b30c1..8cb0dfb143b 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.ql @@ -7,7 +7,6 @@ */ import python -import TimerUtils import NewCfgImpl private module Utils = EvalOrderCfgUtils; diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBasicBlock.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBasicBlock.ql index e07890f7250..c0fc86b5130 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBasicBlock.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBasicBlock.ql @@ -5,7 +5,6 @@ */ import python -import TimerUtils import NewCfgImpl private module Utils = EvalOrderCfgUtils; diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.ql index 30d250d32ab..dd174d6d913 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.ql @@ -6,7 +6,6 @@ */ import python -import TimerUtils import NewCfgImpl private module Utils = EvalOrderCfgUtils; diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.ql index c93d181d852..e7f4a12ff81 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.ql +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.ql @@ -7,7 +7,6 @@ */ import python -import TimerUtils import NewCfgImpl private module Utils = EvalOrderCfgUtils; diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/OldCfgImpl.qll b/python/ql/test/library-tests/ControlFlow/evaluation-order/OldCfgImpl.qll index fc52c8dd3ed..cb7bbb495b8 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/OldCfgImpl.qll +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/OldCfgImpl.qll @@ -3,14 +3,14 @@ * Python control flow graph. */ -private import python as Py +private import python as PY import TimerUtils /** Existing Python CFG implementation of the evaluation-order signature. */ module OldCfg implements EvalOrderCfgSig { - class CfgNode = Py::ControlFlowNode; + class CfgNode = PY::ControlFlowNode; - class BasicBlock = Py::BasicBlock; + class BasicBlock = PY::BasicBlock; - CfgNode scopeGetEntryNode(Py::Scope s) { result = s.getEntryNode() } + CfgNode scopeGetEntryNode(PY::Scope s) { result = s.getEntryNode() } } From 9b4de2bfb693688c81b55a088b6cb733f4641449 Mon Sep 17 00:00:00 2001 From: yoff Date: Mon, 27 Jul 2026 17:52:24 +0200 Subject: [PATCH 87/90] Python: add tests for short-circuiting comparisons - Both the old and new CFG model this incorrectly right now. - Added ConsecutivePredecessorTimestamps.ql for the old CFG for symmetry. --- .../ConsecutivePredecessorTimestamps.expected | 13 +++++++ .../ConsecutivePredecessorTimestamps.ql | 22 +++++++++++ .../ConsecutiveTimestamps.expected | 1 + ...gConsecutivePredecessorTimestamps.expected | 2 +- .../NewCfgConsecutiveTimestamps.expected | 1 + .../evaluation-order/test_comparison.py | 37 +++++++++++++++++++ 6 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutivePredecessorTimestamps.expected create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutivePredecessorTimestamps.ql create mode 100644 python/ql/test/library-tests/ControlFlow/evaluation-order/test_comparison.py diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutivePredecessorTimestamps.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutivePredecessorTimestamps.expected new file mode 100644 index 00000000000..9dc28649061 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutivePredecessorTimestamps.expected @@ -0,0 +1,13 @@ +| test_boolean.py:9:10:9:43 | BoolExpr | $@ in $@ has no consecutive predecessor (expected 1) | test_boolean.py:9:59:9:59 | IntegerLiteral | Timestamp 2 | test_boolean.py:7:1:7:27 | Function test_and_both_sides | test_and_both_sides | +| test_boolean.py:15:10:15:43 | BoolExpr | $@ in $@ has no consecutive predecessor (expected 0) | test_boolean.py:15:50:15:50 | IntegerLiteral | Timestamp 1 | test_boolean.py:13:1:13:30 | Function test_and_short_circuit | test_and_short_circuit | +| test_boolean.py:21:10:21:42 | BoolExpr | $@ in $@ has no consecutive predecessor (expected 0) | test_boolean.py:21:49:21:49 | IntegerLiteral | Timestamp 1 | test_boolean.py:19:1:19:29 | Function test_or_short_circuit | test_or_short_circuit | +| test_boolean.py:27:10:27:34 | BoolExpr | $@ in $@ has no consecutive predecessor (expected 1) | test_boolean.py:27:50:27:50 | IntegerLiteral | Timestamp 2 | test_boolean.py:25:1:25:26 | Function test_or_both_sides | test_or_both_sides | +| test_boolean.py:40:10:40:61 | BoolExpr | $@ in $@ has no consecutive predecessor (expected 2) | test_boolean.py:40:86:40:86 | IntegerLiteral | Timestamp 3 | test_boolean.py:38:1:38:24 | Function test_chained_and | test_chained_and | +| test_boolean.py:46:10:46:61 | BoolExpr | $@ in $@ has no consecutive predecessor (expected 2) | test_boolean.py:46:86:46:86 | IntegerLiteral | Timestamp 3 | test_boolean.py:44:1:44:23 | Function test_chained_or | test_chained_or | +| test_boolean.py:52:10:52:95 | BoolExpr | $@ in $@ has no consecutive predecessor (expected 3) | test_boolean.py:52:120:52:120 | IntegerLiteral | Timestamp 4 | test_boolean.py:50:1:50:25 | Function test_mixed_and_or | test_mixed_and_or | +| test_boolean.py:52:11:52:47 | BoolExpr | $@ in $@ has no consecutive predecessor (expected 1) | test_boolean.py:52:63:52:63 | IntegerLiteral | Timestamp 2 | test_boolean.py:50:1:50:25 | Function test_mixed_and_or | test_mixed_and_or | +| test_boolean.py:52:78:52:79 | IntegerLiteral | $@ in $@ has no consecutive predecessor (expected 2) | test_boolean.py:52:85:52:85 | IntegerLiteral | Timestamp 3 | test_boolean.py:50:1:50:25 | Function test_mixed_and_or | test_mixed_and_or | +| test_comparison.py:37:6:37:41 | Compare | $@ in $@ has no consecutive predecessor (expected 1) | test_comparison.py:37:48:37:48 | IntegerLiteral | Timestamp 2 | test_comparison.py:32:1:32:32 | Function test_three_short_circuit | test_three_short_circuit | +| test_if.py:96:9:96:9 | x | $@ in $@ has no consecutive predecessor (expected 1) | test_if.py:96:15:96:15 | IntegerLiteral | Timestamp 2 | test_if.py:93:1:93:34 | Function test_if_compound_condition | test_if_compound_condition | +| test_if.py:96:9:96:29 | BoolExpr | $@ in $@ has no consecutive predecessor (expected 3) | test_if.py:96:36:96:36 | IntegerLiteral | Timestamp 4 | test_if.py:93:1:93:34 | Function test_if_compound_condition | test_if_compound_condition | +| test_if.py:99:13:99:13 | IntegerLiteral | $@ in $@ has no consecutive predecessor (expected 4) | test_if.py:99:19:99:19 | IntegerLiteral | Timestamp 5 | test_if.py:93:1:93:34 | Function test_if_compound_condition | test_if_compound_condition | diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutivePredecessorTimestamps.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutivePredecessorTimestamps.ql new file mode 100644 index 00000000000..6d90605d685 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutivePredecessorTimestamps.ql @@ -0,0 +1,22 @@ +/** + * Checks that each annotated node (except the minimum timestamp) has a + * predecessor annotation with timestamp `a - 1`. This is the reverse of + * ConsecutiveTimestamps: it catches nodes that are reachable but arrived + * at from the wrong place (skipping an intermediate node). + * + * Only applies to functions where all annotations are in the function's + * own scope (excludes tests with generators, async, comprehensions, or + * lambdas that have annotations in nested scopes). + */ + +import OldCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from TimerAnnotation ann, int a +where consecutivePredecessorTimestamps(ann, a) +select ann, "$@ in $@ has no consecutive predecessor (expected " + (a - 1) + ")", + ann.getTimestampExpr(a), "Timestamp " + a, ann.getTestFunction(), ann.getTestFunction().getName() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutiveTimestamps.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutiveTimestamps.expected index ed22c971ecb..c354396586e 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutiveTimestamps.expected +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutiveTimestamps.expected @@ -7,6 +7,7 @@ | test_boolean.py:52:11:52:47 | BoolExpr | $@ in $@ has no consecutive successor (expected 3) | test_boolean.py:52:63:52:63 | IntegerLiteral | Timestamp 2 | test_boolean.py:50:1:50:25 | Function test_mixed_and_or | test_mixed_and_or | | test_boolean.py:52:27:52:31 | False | $@ in $@ has no consecutive successor (expected 2) | test_boolean.py:52:37:52:37 | IntegerLiteral | Timestamp 1 | test_boolean.py:50:1:50:25 | Function test_mixed_and_or | test_mixed_and_or | | test_boolean.py:52:78:52:79 | IntegerLiteral | $@ in $@ has no consecutive successor (expected 4) | test_boolean.py:52:85:52:85 | IntegerLiteral | Timestamp 3 | test_boolean.py:50:1:50:25 | Function test_mixed_and_or | test_mixed_and_or | +| test_comparison.py:37:17:37:17 | IntegerLiteral | $@ in $@ has no consecutive successor (expected 2) | test_comparison.py:37:23:37:23 | IntegerLiteral | Timestamp 1 | test_comparison.py:32:1:32:32 | Function test_three_short_circuit | test_three_short_circuit | | test_if.py:95:9:95:13 | False | $@ in $@ has no consecutive successor (expected 2) | test_if.py:95:19:95:19 | IntegerLiteral | Timestamp 1 | test_if.py:93:1:93:34 | Function test_if_compound_condition | test_if_compound_condition | | test_if.py:96:9:96:29 | BoolExpr | $@ in $@ has no consecutive successor (expected 5) | test_if.py:96:36:96:36 | IntegerLiteral | Timestamp 4 | test_if.py:93:1:93:34 | Function test_if_compound_condition | test_if_compound_condition | | test_if.py:96:22:96:22 | y | $@ in $@ has no consecutive successor (expected 4) | test_if.py:96:28:96:28 | IntegerLiteral | Timestamp 3 | test_if.py:93:1:93:34 | Function test_if_compound_condition | test_if_compound_condition | diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.expected index 8b137891791..0c9e31f3771 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.expected +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.expected @@ -1 +1 @@ - +| test_comparison.py:37:6:37:41 | Compare | $@ in $@ has no consecutive predecessor (expected 1) | test_comparison.py:37:48:37:48 | IntegerLiteral | Timestamp 2 | test_comparison.py:32:1:32:32 | Function test_three_short_circuit | test_three_short_circuit | diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.expected index e69de29bb2d..6fdcfc40c4a 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.expected +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.expected @@ -0,0 +1 @@ +| test_comparison.py:37:17:37:17 | IntegerLiteral | $@ in $@ has no consecutive successor (expected 2) | test_comparison.py:37:23:37:23 | IntegerLiteral | Timestamp 1 | test_comparison.py:32:1:32:32 | Function test_three_short_circuit | test_three_short_circuit | diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/test_comparison.py b/python/ql/test/library-tests/ControlFlow/evaluation-order/test_comparison.py new file mode 100644 index 00000000000..97866317166 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/test_comparison.py @@ -0,0 +1,37 @@ +"""Comparisons and evaluation order. + +Comparison operands are evaluated left-to-right, followed by the comparison +node itself. Annotations record the *run-time* evaluation order, so this file +self-validates under CPython (``python3 test_comparison.py``). + +Python short-circuits *chained* comparisons (in ``a < b < c`` the operand ``c`` +is skipped once ``a < b`` is false), whereas the control-flow graph +conservatively evaluates every operand. That divergence cannot be expressed in +a single annotation, so the affected CFG queries report it (captured in the +ConsecutiveTimestamps and NewCfgConsecutivePredecessorTimestamps .expected +files). +""" + +from timer import test, dead + + +@test +def test_two(t): + # Both operands, then the comparison. + (1 @ t[0] < 0 @ t[1]) @ t[2] + + +@test +def test_three(t): + # Chained comparison, all comparisons hold: operands left-to-right, then + # the comparison. Run time and CFG agree. + (1 @ t[0] < 2 @ t[1] < 3 @ t[2]) @ t[3] + + +@test +def test_three_short_circuit(t): + # ``1 > 2`` is false, so at run time Python short-circuits and never + # evaluates ``3``; the comparison is reached at timestamp 2. The CFG still + # evaluates ``3`` (dead at run time), which is why the CFG queries report a + # gap around this comparison. + (1 @ t[0] > 2 @ t[1] < 3 @ t[dead(2)]) @ t[2] From c11b4914e93bd6742dd3bc26d1565691e3586df4 Mon Sep 17 00:00:00 2001 From: yoff Date: Tue, 28 Jul 2026 14:42:04 +0200 Subject: [PATCH 88/90] Update python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll Co-authored-by: Anders Schack-Mulligen --- python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll | 1 - 1 file changed, 1 deletion(-) diff --git a/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll b/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll index 21dbad68b81..bfbd6e29bdd 100644 --- a/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll +++ b/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll @@ -1755,7 +1755,6 @@ private module Input implements InputSig1, InputSig2 { } } -import CfgCachedStage import Public /** From c516d1f0b951aa973686ed3383066c1057f12d57 Mon Sep 17 00:00:00 2001 From: yoff Date: Tue, 28 Jul 2026 14:49:18 +0200 Subject: [PATCH 89/90] python: forward rather implement subtle predicates --- .../python/controlflow/internal/Cfg.qll | 26 ++----------------- 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll b/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll index b102c71391f..3441ad0203c 100644 --- a/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll +++ b/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll @@ -280,14 +280,7 @@ class BasicBlock extends CfgImpl::BasicBlock { * doesn't currently expose a `dominanceFrontier` predicate at this * level. */ - predicate inDominanceFrontier(BasicBlock df) { - this = df.getAPredecessor() and not this = df.getImmediateDominator() - or - exists(BasicBlock prev | prev.inDominanceFrontier(df) | - this = prev.getImmediateDominator() and - not this = df.getImmediateDominator() - ) - } + predicate inDominanceFrontier(BasicBlock df) { super.inDominanceFrontier(df) } /** Holds if this basic block strictly reaches `other`. */ predicate strictlyReaches(BasicBlock other) { super.getASuccessor+() = other } @@ -326,22 +319,7 @@ class BasicBlock extends CfgImpl::BasicBlock { * This mirrors the legacy `ConditionBlock.controls(BB, branch)`. */ predicate controls(BasicBlock other, boolean branch) { - exists(BasicBlock succ | - branch = true and succ = this.getATrueSuccessor() - or - branch = false and succ = this.getAFalseSuccessor() - | - succ.dominates(other) and - // The other branch must not also reach `other` — otherwise - // `other` is not actually controlled by `branch`. - not exists(BasicBlock otherSucc | - branch = true and otherSucc = this.getAFalseSuccessor() - or - branch = false and otherSucc = this.getATrueSuccessor() - | - otherSucc.reaches(other) - ) - ) + super.edgeDominates(other, any(BooleanSuccessor t | t.getValue() = branch)) } } From 3ad48615a7cd2aa70513b49d56883c13fb54b024 Mon Sep 17 00:00:00 2001 From: yoff Date: Tue, 28 Jul 2026 14:54:09 +0200 Subject: [PATCH 90/90] Python: fix replace of upper-case characters --- .../ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll b/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll index bfbd6e29bdd..78218f48921 100644 --- a/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll +++ b/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll @@ -156,7 +156,7 @@ module Ast implements AstSig { /** * A parameter of a callable. * - * modeled per the C# template (`csharp/.../ControlFlowGraph.qll`): + * Modeled per the C# template (`csharp/.../ControlFlowGraph.qll`): * each Python parameter (the `Py::Parameter` AST node, which is a `Name` * or — Python 2 only — a `Tuple` in store context) becomes a CFG node * at a stable position in the enclosing callable's entry sequence. @@ -1231,7 +1231,7 @@ module Ast implements AstSig { } /** - * An `import x.y` module expression. modeled as a leaf — the dotted + * An `import x.y` module expression. Modeled as a leaf — the dotted * name is just a string. */ additional class ImportExpression extends Expr {