upgrading to 2.17.5

javascript/BUILD.bazel

@@ -1,6 +1,5 @@
 load("@rules_pkg//pkg:mappings.bzl", "pkg_files")
-load("@semmle_code//:dist.bzl", "dist")
-load("@semmle_code//buildutils-internal:zipmerge.bzl", "zipmerge")
+load("//misc/bazel:pkg.bzl", "codeql_pack")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -23,26 +22,25 @@ pkg_files(
     strip_prefix = None,
 )
 
-dist(
-    name = "javascript-extractor-pack",
+# We have to use a zip of the typescript parser wrapper, as it's generated by a genrule
+# and we don't know a list of its output files.
+codeql_pack(
+    name = "javascript",
     srcs = [
         ":dbscheme-group",
         "//javascript/downgrades",
         "//javascript/externs",
         "//javascript/extractor:tools-extractor",
-        "@semmle_code//language-packs/javascript:resources",
+        "//javascript/resources",
     ],
-    prefix = "javascript",
+    visibility = ["//visibility:public"],
+    zips = {"//javascript/extractor/lib/typescript": "tools"},
 )
 
-# We have to zipmerge in the typescript parser wrapper, as it's generated by a genrule
-# and we don't know a list of its output files. Therefore, we sidestep the
-# rules_pkg tooling here, and generate the zip for the language pack manually.
-zipmerge(
-    name = "javascript",
-    srcs = [
-        ":javascript-extractor-pack.zip",
-        "//javascript/extractor/lib/typescript",
-    ],
-    out = "javascript.zip",
+# TODO copy for internal repository backward compatibility
+genrule(
+    name = "javascript.zip",
+    srcs = [":javascript-generic-zip"],
+    outs = ["javascript.zip"],
+    cmd = "cp $< $@",
 )

javascript/downgrades/BUILD.bazel

@@ -1,11 +1,12 @@
-load("@semmle_code//:dist.bzl", "pack_zip")
+load("//misc/bazel:pkg.bzl", "codeql_pkg_files", "strip_prefix")
 
-pack_zip(
+codeql_pkg_files(
     name = "downgrades",
     srcs = glob(
         ["**/*"],
         exclude = ["BUILD.bazel"],
     ),
     prefix = "downgrades",
-    visibility = ["//visibility:public"],
+    strip_prefix = strip_prefix.from_pkg(),
+    visibility = ["//javascript:__pkg__"],
 )

javascript/externs/BUILD.bazel

@@ -1,11 +1,12 @@
-load("@semmle_code//:dist.bzl", "pack_zip")
+load("//misc/bazel:pkg.bzl", "codeql_pkg_files", "strip_prefix")
 
-pack_zip(
+codeql_pkg_files(
     name = "externs",
     srcs = glob(
         ["**/*"],
         exclude = ["BUILD.bazel"],
     ),
     prefix = "tools/data/externs",
-    visibility = ["//visibility:public"],
+    strip_prefix = strip_prefix.from_pkg(),
+    visibility = ["//javascript:__pkg__"],
 )

javascript/extractor/lib/typescript/BUILD.bazel

@@ -1,5 +1,3 @@
-load("@semmle_code//:common.bzl", "on_windows")
-
 # Builds a zip file of the compiled typscript-parser-wrapper and its dependencies.
 genrule(
     name = "typescript",
@@ -33,19 +31,16 @@ genrule(
         # Install again with only runtime deps
         "$$NPM install --prod",
         "mv node_modules build/",
-        "mkdir -p javascript/tools/typescript-parser-wrapper",
-        "mv build/* javascript/tools/typescript-parser-wrapper",
+        "mkdir -p typescript-parser-wrapper",
+        "mv build/* typescript-parser-wrapper",
+        "OUT=$$BAZEL_ROOT/$@",
+        "case $$OSTYPE in",
+        "  cygwin|msys|win32) OUT=$$(cygpath -w $$OUT);;",
+        "esac",
         "",
-    ]) + on_windows(
-        " && ".join([
-            "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$(cygpath -w $$BAZEL_ROOT/$@) $$(find javascript -name '*' -print)",
-            "rm -rf $$TEMP",
-        ]),
-        " && ".join([
-            "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$BAZEL_ROOT/$@ $$(find javascript -name '*' -print)",
-            "rm -rf $$TEMP",
-        ]),
-    ),
+        "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$OUT $$(find typescript-parser-wrapper -name '*' -print)",
+        "rm -rf $$TEMP",
+    ]),
     tools = [
         "@bazel_tools//tools/zip:zipper",
         "@nodejs//:node_bin",

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -159,6 +159,7 @@ import com.semmle.util.trap.TrapWriter;
  *   <li>Files with base name "package.json" or "tsconfig.json", and files whose base name
  *       is of the form "codeql-javascript-*.json".
  *   <li>JavaScript, JSON or YAML files whose base name starts with ".eslintrc".
+ *   <li>JSON files whose base name is ".xsaccess".
  *   <li>All extension-less files.
  * </ul>
  *
@@ -393,9 +394,10 @@ public class AutoBuild {
     for (FileType filetype : defaultExtract)
       for (String extension : filetype.getExtensions()) patterns.add("**/*" + extension);
 
-    // include .eslintrc files, package.json files, tsconfig.json files, and
-    // codeql-javascript-*.json files
+    // include .eslintrc files, .xsaccess files, package.json files, 
+    // tsconfig.json files, and codeql-javascript-*.json files
     patterns.add("**/.eslintrc*");
+    patterns.add("**/.xsaccess");
     patterns.add("**/package.json");
     patterns.add("**/tsconfig*.json");
     patterns.add("**/codeql-javascript-*.json");

javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java

@@ -184,8 +184,8 @@ public class FileExtractor {
         if (super.contains(f, lcExt, config)) return true;
 
         // detect JSON-encoded configuration files whose name starts with `.` and ends with `rc`
-        // (e.g., `.eslintrc` or `.babelrc`)
-        if (f.isFile() && f.getName().matches("\\..*rc")) {
+        // (e.g., `.eslintrc` or `.babelrc`) as well as `.xsaccess` files
+        if (f.isFile() && f.getName().matches("\\..*rc|\\.xsaccess")) {
           try (BufferedReader br = new BufferedReader(new FileReader(f))) {
             // check whether the first two non-empty lines look like the start of a JSON object
             // (two lines because the opening brace is usually on a line by itself)

javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java

@@ -50,7 +50,7 @@ public class AllTests {
         entry = zis.getNextEntry();
       }
     }
-    Path tsWrapper = tempDir.resolve("javascript/tools/typescript-parser-wrapper/main.js");
+    Path tsWrapper = tempDir.resolve("typescript-parser-wrapper/main.js");
     if (!Files.exists(tsWrapper)) {
       throw new RuntimeException("Could not find ts-wrapper at " + tsWrapper);
     }

javascript/extractor/tests/xsaccess/input/.xsaccess

@@ -0,0 +1,3 @@
+{
+  "exposed": true // Expose data via http
+}

javascript/extractor/tests/xsaccess/output/trap/.xsaccess.trap

@@ -0,0 +1,22 @@
+#10000=@"/.xsaccess;sourcefile"
+files(#10000,"/.xsaccess")
+#10001=@"/;folder"
+folders(#10001,"/")
+containerparent(#10001,#10000)
+#10002=@"loc,{#10000},0,0,0,0"
+locations_default(#10002,#10000,0,0,0,0)
+hasLocation(#10000,#10002)
+#20000=*
+json(#20000,5,#10000,0,"{\n  ""ex ...  http\n}")
+#20001=@"loc,{#10000},1,1,3,1"
+locations_default(#20001,#10000,1,1,3,1)
+json_locations(#20000,#20001)
+#20002=*
+json(#20002,1,#20000,0,"true")
+#20003=@"loc,{#10000},2,14,2,17"
+locations_default(#20003,#10000,2,14,2,17)
+json_locations(#20002,#20003)
+json_literals("true","true",#20002)
+json_properties(#20000,"exposed",#20002)
+numlines(#10000,3,0,0)
+filetype(#10000,"json")

javascript/ql/integration-tests/legacy

@@ -0,0 +1 @@
+These tests are still run with the legacy test runner

javascript/ql/lib/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.0.1
+
+No user-facing changes.
+
 ## 1.0.0
 
 ### Breaking Changes

javascript/ql/lib/change-notes/released/1.0.1.md

@@ -0,0 +1,3 @@
+## 1.0.1
+
+No user-facing changes.

javascript/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.0.0
+lastReleaseVersion: 1.0.1

javascript/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/javascript-all
-version: 1.0.0
+version: 1.0.1
 groups: javascript
 dbscheme: semmlecode.javascript.dbscheme
 extractor: javascript

javascript/ql/lib/semmle/javascript/dataflow/internal/DataFlowNode.qll

@@ -33,4 +33,10 @@ newtype TNode =
   TExceptionalInvocationReturnNode(InvokeExpr e) or
   TGlobalAccessPathRoot() or
   TTemplatePlaceholderTag(Templating::TemplatePlaceholderTag tag) or
-  TReflectiveParametersNode(Function f)
+  TReflectiveParametersNode(Function f) or
+  TForbiddenRecursionGuard() {
+    none() and
+    // We want to prune irrelevant models before materialising data flow nodes, so types contributed
+    // directly from CodeQL must expose their pruning info without depending on data flow nodes.
+    (any(ModelInput::TypeModel tm).isTypeUsed("") implies any())
+  }

javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModels.qll

@@ -168,9 +168,20 @@ module ModelInput {
    * A unit class for adding additional type model rows from CodeQL models.
    */
   class TypeModel extends Unit {
+    /**
+     * Holds if any of the other predicates in this class might have a result
+     * for the given `type`.
+     *
+     * The implementation of this predicate should not depend on `DataFlow::Node`.
+     */
+    bindingset[type]
+    predicate isTypeUsed(string type) { none() }
+
     /**
      * Gets a data-flow node that is a source of the given `type`.
      *
+     * Note that `type` should also be included in `isTypeUsed`.
+     *
      * This must not depend on API graphs, but ensures that an API node is generated for
      * the source.
      */
@@ -180,6 +191,8 @@ module ModelInput {
      * Gets a data-flow node that is a sink of the given `type`,
      * usually because it is an argument passed to a parameter of that type.
      *
+     * Note that `type` should also be included in `isTypeUsed`.
+     *
      * This must not depend on API graphs, but ensures that an API node is generated for
      * the sink.
      */
@@ -188,6 +201,8 @@ module ModelInput {
     /**
      * Gets an API node that is a source or sink of the given `type`.
      *
+     * Note that `type` should also be included in `isTypeUsed`.
+     *
      * Unlike `getASource` and `getASink`, this may depend on API graphs.
      */
     API::Node getAnApiNode(string type) { none() }
@@ -354,6 +369,28 @@ private predicate typeVariableModel(string name, string path) {
   Extensions::typeVariableModel(name, path)
 }
 
+/**
+ * Holds if the given extension tuple `madId` should pretty-print as `model`.
+ *
+ * This predicate should only be used in tests.
+ */
+predicate interpretModelForTest(QlBuiltins::ExtensionId madId, string model) {
+  exists(string type, string path, string kind |
+    Extensions::sourceModel(type, path, kind, madId) and
+    model = "Source: " + type + "; " + path + "; " + kind
+  )
+  or
+  exists(string type, string path, string kind |
+    Extensions::sinkModel(type, path, kind, madId) and
+    model = "Sink: " + type + "; " + path + "; " + kind
+  )
+  or
+  exists(string type, string path, string input, string output, string kind |
+    Extensions::summaryModel(type, path, input, output, kind, madId) and
+    model = "Summary: " + type + "; " + path + "; " + input + "; " + output + "; " + kind
+  )
+}
+
 /**
  * Holds if rows involving `type` might be relevant for the analysis of this database.
  */
@@ -367,6 +404,8 @@ predicate isRelevantType(string type) {
   (
     Specific::isTypeUsed(type)
     or
+    any(TypeModel model).isTypeUsed(type)
+    or
     exists(TestAllModels t)
   )
   or

javascript/ql/src/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.0.1
+
+No user-facing changes.
+
 ## 1.0.0
 
 ### Breaking Changes

javascript/ql/src/change-notes/released/1.0.1.md

@@ -0,0 +1,3 @@
+## 1.0.1
+
+No user-facing changes.

javascript/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.0.0
+lastReleaseVersion: 1.0.1

javascript/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/javascript-queries
-version: 1.0.0
+version: 1.0.1
 groups:
   - javascript
   - queries

javascript/ql/test/library-tests/frameworks/data/test.expected

@@ -79,6 +79,7 @@ taintFlow
 | test.js:269:10:269:31 | this.ba ... ource() | test.js:269:10:269:31 | this.ba ... ource() |
 | test.js:272:6:272:40 | new MyS ... ource() | test.js:272:6:272:40 | new MyS ... ource() |
 | test.js:274:6:274:39 | testlib ... eName() | test.js:274:6:274:39 | testlib ... eName() |
+| test.js:277:8:277:31 | "danger ... .danger | test.js:277:8:277:31 | "danger ... .danger |
 isSink
 | test.js:54:18:54:25 | source() | test-sink |
 | test.js:55:22:55:29 | source() | test-sink |

javascript/ql/test/library-tests/frameworks/data/test.ext.yml

@@ -11,6 +11,7 @@ extensions:
       - ['testlib', 'Member[ParamDecoratorSource].DecoratedParameter', 'test-source']
       - ['testlib', 'Member[getSource].ReturnValue', 'test-source']
       - ['(testlib)', 'Member[parenthesizedPackageName].ReturnValue', 'test-source']
+      - ['danger-constant', 'Member[danger]', 'test-source']
 
   - addsTo:
       pack: codeql/javascript-all

javascript/ql/test/library-tests/frameworks/data/test.js

@@ -272,3 +272,9 @@ class MySubclass2 extends MySubclass {
 sink(new MySubclass2().baseclassSource()); // NOT OK
 
 sink(testlib.parenthesizedPackageName()); // NOT OK
+
+function dangerConstant() {
+  sink("danger-constant".danger); // NOT OK
+  sink("danger-constant".safe); // OK
+  sink("danger-constant"); // OK
+}

javascript/ql/test/library-tests/frameworks/data/test.ql

@@ -2,6 +2,15 @@ import javascript
 import testUtilities.ConsistencyChecking
 import semmle.javascript.frameworks.data.internal.ApiGraphModels as ApiGraphModels
 
+class TypeModelFromCodeQL extends ModelInput::TypeModel {
+  override predicate isTypeUsed(string type) { type = "danger-constant" }
+
+  override DataFlow::Node getASource(string type) {
+    type = "danger-constant" and
+    result.getStringValue() = "danger-constant"
+  }
+}
+
 class BasicTaintTracking extends TaintTracking::Configuration {
   BasicTaintTracking() { this = "BasicTaintTracking" }
 

javascript/resources/BUILD.bazel

@@ -0,0 +1,15 @@
+load("//misc/bazel:pkg.bzl", "codeql_pkg_files")
+
+codeql_pkg_files(
+    name = "resources",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "tools/*.sh",
+            "BUILD.bazel",
+        ],
+    ),
+    exes = glob(["tools/*.sh"]),
+    strip_prefix = "",
+    visibility = ["//javascript:__pkg__"],
+)

javascript/resources/codeql-extractor.yml

@@ -0,0 +1,94 @@
+name: "javascript"
+aliases:
+  - javascript-typescript
+  - typescript
+display_name: "JavaScript/TypeScript"
+version: 1.22.1
+column_kind: "utf16"
+unicode_newlines: true
+build_modes:
+  - none
+file_coverage_languages:
+  - name: javascript
+    display_name: JavaScript
+    scc_languages:
+      - JavaScript
+  - name: typescript
+    display_name: TypeScript
+    scc_languages:
+      - TypeScript
+      - TypeScript Typings
+github_api_languages:
+  - JavaScript
+  - TypeScript
+scc_languages:
+  - JavaScript
+  - TypeScript
+  - TypeScript Typings
+file_types:
+  - name: javascript
+    display_name: JavaScript
+    extensions:
+      - .js
+      - .jsx
+  - name: ecmascript
+    display_name: ECMAScript
+    extensions:
+      - .es
+      - .es6
+      - .mjs
+  - name: typescript
+    display_name: TypeScript
+    extensions:
+      - .ts
+      - .tsx
+  - name: html
+    display_name: HTML
+    extensions:
+      - .html
+      - .htm
+      - .xhtm
+      - .xhtml
+  - name: vue
+    display_name: Vue.js component
+    extensions:
+      - .vue
+  - name: data
+    display_name: Data or configuration files
+    extensions:
+      - .json
+      - .yml
+      - .yaml
+      - .raml
+legacy_qltest_extraction: true
+options:
+  trap:
+    title: TRAP options
+    description: Options about how the extractor handles TRAP files
+    type: object
+    visibility: 3
+    properties:
+      cache:
+        title: TRAP cache options
+        description: Options about how the extractor handles its TRAP cache
+        type: object
+        properties:
+          dir:
+            title: TRAP cache directory
+            description: The directory of the TRAP cache to use
+            type: string
+          bound:
+            title: TRAP cache bound
+            description: A soft limit (in MB) on the size of the TRAP cache
+            type: string
+            pattern: "[0-9]+"
+          write:
+            title: TRAP cache writeable
+            description: Whether to write to the TRAP cache as well as reading it
+            type: string
+            pattern: "(true|TRUE|false|FALSE)"
+  skip_types:
+    title: Skip type extraction for TypeScript
+    description: Whether to skip the extraction of types in a TypeScript application
+    type: string
+    pattern: "^(false|true)$"

javascript/resources/tools/autobuild.cmd

@@ -0,0 +1,30 @@
+@echo off
+SETLOCAL EnableDelayedExpansion
+
+set jvm_args=-Xss16m
+
+rem If CODEQL_RAM is set, use half for Java and half for TS.
+if NOT [%CODEQL_RAM%] == [] (
+    set /a "half_ram=CODEQL_RAM/2"
+    set LGTM_TYPESCRIPT_RAM=%half_ram%
+    set jvm_args=!jvm_args! -Xmx!half_ram!m
+)
+
+rem If CODEQL_THREADS is set, propagate via LGTM_THREADS.
+if NOT [%CODEQL_THREADS%] == [] (
+    set LGTM_THREADS=%CODEQL_THREADS%
+)
+
+rem The JS autobuilder expects to find typescript modules under SEMMLE_DIST/tools.
+rem They are included in the pack, but we need to set SEMMLE_DIST appropriately.
+set SEMMLE_DIST=%CODEQL_EXTRACTOR_JAVASCRIPT_ROOT%
+
+rem The JS autobuilder expects LGTM_SRC to be set to the source root.
+set LGTM_SRC=%CD%
+
+type NUL && "%CODEQL_JAVA_HOME%\bin\java.exe" %jvm_args% ^
+    -cp "%CODEQL_EXTRACTOR_JAVASCRIPT_ROOT%\tools\extractor-javascript.jar" ^
+    com.semmle.js.extractor.AutoBuild
+exit /b %ERRORLEVEL%
+
+ENDLOCAL

javascript/resources/tools/autobuild.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+set -eu
+
+jvm_args=-Xss16m
+
+# If CODEQL_RAM is set, use half for Java and half for TS.
+if [ -n "${CODEQL_RAM:-}" ] ; then
+    half_ram="$(( CODEQL_RAM / 2 ))"
+    LGTM_TYPESCRIPT_RAM="$half_ram"
+    export LGTM_TYPESCRIPT_RAM
+    jvm_args="$jvm_args -Xmx${half_ram}m"
+fi
+
+# If CODEQL_THREADS is set, propagate via LGTM_THREADS.
+if [ -n "${CODEQL_THREADS:-}" ] ; then
+    LGTM_THREADS="$CODEQL_THREADS"
+    export LGTM_THREADS
+fi
+
+# The JS autobuilder expects to find typescript modules under SEMMLE_DIST/tools.
+# They are included in the pack, but we need to set SEMMLE_DIST appropriately.
+# We want to word-split $jvm_args, so disable the shellcheck warning.
+# shellcheck disable=SC2086
+env SEMMLE_DIST="$CODEQL_EXTRACTOR_JAVASCRIPT_ROOT" \
+    LGTM_SRC="$(pwd)" \
+    "${CODEQL_JAVA_HOME}/bin/java" $jvm_args \
+    -cp "$CODEQL_EXTRACTOR_JAVASCRIPT_ROOT/tools/extractor-javascript.jar" \
+    com.semmle.js.extractor.AutoBuild

javascript/resources/tools/baseline-config.json

@@ -0,0 +1,8 @@
+{
+    "paths-ignore": [
+        "**/node_modules/**",
+        "**/bower_components/**",
+        "**/*.min.js",
+        "**/*-min.js"
+    ]
+}

javascript/resources/tools/configure-baseline.cmd

@@ -0,0 +1,2 @@
+@echo off
+type "%CODEQL_EXTRACTOR_JAVASCRIPT_ROOT%\tools\baseline-config.json"

javascript/resources/tools/configure-baseline.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+cat "$CODEQL_EXTRACTOR_JAVASCRIPT_ROOT/tools/baseline-config.json"

javascript/resources/tools/index-dir.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+echo "Not implemented." 1>&2
+exit 1