Swift: use git-lfs devcontainer feature

Install script: Windows-compatible cleanup path.

.devcontainer/swift/devcontainer.json

@@ -1,25 +1,28 @@
 {
-	"extensions": [
-		"github.vscode-codeql",
-		"hbenl.vscode-test-explorer",
-		"ms-vscode.test-adapter-converter",
-		"slevesque.vscode-zipexplorer",
-		"ms-vscode.cpptools"
-	],
-	"settings": {
-		"files.watcherExclude": {
-			"**/target/**": true
-		},
-		"codeQL.runningQueries.memory": 2048
-	},
-	"build": {
-		"dockerfile": "Dockerfile",
-	},
-	"runArgs": [
-		"--cap-add=SYS_PTRACE",
-		"--security-opt",
-		"seccomp=unconfined"
-	],
-	"remoteUser": "vscode",
-	"onCreateCommand": ".devcontainer/swift/user.sh"
+  "extensions": [
+    "github.vscode-codeql",
+    "hbenl.vscode-test-explorer",
+    "ms-vscode.test-adapter-converter",
+    "slevesque.vscode-zipexplorer",
+    "ms-vscode.cpptools"
+  ],
+  "features": {
+    "git-lfs": "latest"
+  },
+  "settings": {
+    "files.watcherExclude": {
+      "**/target/**": true
+    },
+    "codeQL.runningQueries.memory": 2048
+  },
+  "build": {
+    "dockerfile": "Dockerfile",
+  },
+  "runArgs": [
+    "--cap-add=SYS_PTRACE",
+    "--security-opt",
+    "seccomp=unconfined"
+  ],
+  "remoteUser": "vscode",
+  "onCreateCommand": ".devcontainer/swift/user.sh"
 }

.devcontainer/swift/root.sh

@@ -3,9 +3,6 @@ set -xe
 BAZELISK_VERSION=v1.12.0
 BAZELISK_DOWNLOAD_SHA=6b0bcb2ea15bca16fffabe6fda75803440375354c085480fe361d2cbf32501db
 
-# install git lfs apt source
-curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
-
 # install gh apt source
 (type -p wget >/dev/null || (sudo apt update && sudo apt-get install wget -y)) \
 && sudo mkdir -p -m 755 /etc/apt/keyrings \
@@ -21,7 +18,6 @@ apt-get -y install --no-install-recommends \
     python3-distutils \
     python3-pip \
     bash-completion \
-    git-lfs \
     gh
 
 # Install Bazel

.devcontainer/swift/user.sh

@@ -1,7 +1,5 @@
 set -xe
 
-git lfs install
-
 # add the workspace to the codeql search path
 mkdir -p /home/vscode/.config/codeql
 echo "--search-path /workspaces/codeql" > /home/vscode/.config/codeql/config

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 1.2.0
+version: 1.2.1-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/src/Likely Bugs/Memory Management/SuspiciousCallToStrncat.ql

@@ -4,7 +4,7 @@
  * @kind problem
  * @problem.severity warning
  * @security-severity 9.3
- * @precision medium
+ * @precision high
  * @id cpp/unsafe-strncat
  * @tags reliability
  *       correctness

cpp/ql/src/change-notes/2024-07-08-unsafe-strncat-query.md

@@ -0,0 +1,4 @@
+---
+category: queryMetadata
+---
+* The precision of `cpp/unsafe-strncat` ("Potentially unsafe call to strncat") has been increased to `high`. As a result, it will be run by default as part of the Code Scanning suite.

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 1.0.3
+version: 1.0.4-dev
 groups:
   - cpp
   - queries

cpp/ql/test/query-tests/Likely Bugs/Memory Management/SuspiciousCallToStrncat/SuspiciousCallToStrncat.expected

@@ -3,3 +3,5 @@
 | test.c:67:3:67:9 | call to strncat | Potentially unsafe call to strncat. |
 | test.c:75:3:75:9 | call to strncat | Potentially unsafe call to strncat. |
 | test.c:76:3:76:9 | call to strncat | Potentially unsafe call to strncat. |
+| test.c:91:3:91:9 | call to strncat | Potentially unsafe call to strncat. |
+| test.c:99:3:99:9 | call to strncat | Potentially unsafe call to strncat. |

cpp/ql/test/query-tests/Likely Bugs/Memory Management/SuspiciousCallToStrncat/test.c

@@ -82,3 +82,20 @@ void strncat_test5(char *s) {
   strncat(buf, s, len - strlen(buf) - 1); // GOOD
   strncat(buf, s, len - strlen(buf)); // GOOD
 }
+
+void strncat_test6() {
+  {
+  char dest[60];
+  dest[0] = '\0';
+  // Will write `dest[0 .. 5]`
+  strncat(dest, "small", sizeof(dest)); // GOOD [FALSE POSITIVE]
+  }
+
+  {
+  char dest[60];
+  memset(dest, 'a', sizeof(dest));
+  dest[54] = '\0';
+  // Will write `dest[54 .. 59]`
+  strncat(dest, "small", sizeof(dest)); // GOOD [FALSE POSITIVE]
+  }
+}

csharp/ql/campaigns/Solorigate/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-all
-version: 1.7.20
+version: 1.7.21-dev
 groups:
   - csharp
   - solorigate

csharp/ql/campaigns/Solorigate/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-queries
-version: 1.7.20
+version: 1.7.21-dev
 groups:
   - csharp
   - solorigate

csharp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-all
-version: 1.0.3
+version: 1.0.4-dev
 groups: csharp
 dbscheme: semmlecode.csharp.dbscheme
 extractor: csharp

csharp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-queries
-version: 1.0.3
+version: 1.0.4-dev
 groups:
   - csharp
   - queries

go/ql/consistency-queries/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql-go-consistency-queries
-version: 1.0.3
+version: 1.0.4-dev
 groups:
   - go
   - queries

go/ql/lib/change-notes/2024-07-08-fix-clear-sanitizer.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* There was a bug which meant that the built-in function `clear` was considered as a sanitizer in some cases when it shouldn't have been. This has now been fixed, which may lead to more alerts.

go/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/go-all
-version: 1.1.2
+version: 1.1.3-dev
 groups: go
 dbscheme: go.dbscheme
 extractor: go

go/ql/lib/semmle/go/dataflow/internal/TaintTrackingUtil.qll

@@ -423,7 +423,7 @@ private class ClearSanitizer extends DefaultTaintSanitizer {
       arg = call.getAnArgument() and
       arg = var.getAUse() and
       arg != this and
-      this.getBasicBlock().(ReachableBasicBlock).dominates(this.getBasicBlock())
+      arg.getBasicBlock().(ReachableBasicBlock).dominates(this.getBasicBlock())
     )
   }
 }

go/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/go-queries
-version: 1.0.3
+version: 1.0.4-dev
 groups:
   - go
   - queries

go/ql/test/library-tests/semmle/go/dataflow/DefaultTaintSanitizer/Builtin.go

@@ -8,9 +8,38 @@ func clearTestBad(sourceReq *http.Request) string {
 	return string(b)
 }
 
+func clearTestBad2(sourceReq *http.Request, x bool) string {
+	b := make([]byte, 8)
+	sourceReq.Body.Read(b)
+	if x {
+		clear(b)
+	}
+	return string(b)
+}
+
+func clearTestBad3(sourceReq *http.Request, x bool) string {
+	b := make([]byte, 8)
+	sourceReq.Body.Read(b)
+	if x {
+		return string(b)
+	}
+	clear(b)
+	return string(b)
+}
+
 func clearTestGood(sourceReq *http.Request) string {
 	b := make([]byte, 8)
 	sourceReq.Body.Read(b)
 	clear(b) // should prevent taint flow
 	return string(b)
 }
+
+func clearTestGood2(sourceReq *http.Request, x bool) string {
+	b := make([]byte, 8)
+	sourceReq.Body.Read(b)
+	clear(b) // should prevent taint flow
+	if x {
+		return string(b)
+	}
+	return ""
+}

go/ql/test/library-tests/semmle/go/dataflow/DefaultTaintSanitizer/DefaultSanitizer.expected

@@ -1,10 +1,22 @@
 edges
 | Builtin.go:6:2:6:2 | definition of b | Builtin.go:8:9:8:17 | type conversion | provenance |  |
 | Builtin.go:7:2:7:15 | selection of Body | Builtin.go:6:2:6:2 | definition of b | provenance | MaD:626 |
+| Builtin.go:12:2:12:2 | definition of b | Builtin.go:17:9:17:17 | type conversion | provenance |  |
+| Builtin.go:13:2:13:15 | selection of Body | Builtin.go:12:2:12:2 | definition of b | provenance | MaD:626 |
+| Builtin.go:21:2:21:2 | definition of b | Builtin.go:24:10:24:18 | type conversion | provenance |  |
+| Builtin.go:22:2:22:15 | selection of Body | Builtin.go:21:2:21:2 | definition of b | provenance | MaD:626 |
 nodes
 | Builtin.go:6:2:6:2 | definition of b | semmle.label | definition of b |
 | Builtin.go:7:2:7:15 | selection of Body | semmle.label | selection of Body |
 | Builtin.go:8:9:8:17 | type conversion | semmle.label | type conversion |
+| Builtin.go:12:2:12:2 | definition of b | semmle.label | definition of b |
+| Builtin.go:13:2:13:15 | selection of Body | semmle.label | selection of Body |
+| Builtin.go:17:9:17:17 | type conversion | semmle.label | type conversion |
+| Builtin.go:21:2:21:2 | definition of b | semmle.label | definition of b |
+| Builtin.go:22:2:22:15 | selection of Body | semmle.label | selection of Body |
+| Builtin.go:24:10:24:18 | type conversion | semmle.label | type conversion |
 subpaths
 #select
 | Builtin.go:8:9:8:17 | type conversion | Builtin.go:7:2:7:15 | selection of Body | Builtin.go:8:9:8:17 | type conversion | Found taint flow |
+| Builtin.go:17:9:17:17 | type conversion | Builtin.go:13:2:13:15 | selection of Body | Builtin.go:17:9:17:17 | type conversion | Found taint flow |
+| Builtin.go:24:10:24:18 | type conversion | Builtin.go:22:2:22:15 | selection of Body | Builtin.go:24:10:24:18 | type conversion | Found taint flow |

java/kotlin-extractor/dev/.gitignore

@@ -1,2 +1 @@
-/.kotlinc_version
-/.kotlinc_installed
+/.kotlinc_*

java/kotlin-extractor/dev/wrapper.py

@@ -29,6 +29,7 @@ import os
 
 DEFAULT_VERSION = "2.0.0"
 
+
 def options():
     parser = argparse.ArgumentParser(add_help=False)
     parser.add_argument("tool")
@@ -38,11 +39,15 @@ def options():
     return parser.parse_known_args()
 
 
-url_template = 'https://github.com/JetBrains/kotlin/releases/download/v{version}/kotlin-compiler-{version}.zip'
+file_template = "kotlin-compiler-{version}.zip"
+url_template = "https://github.com/JetBrains/kotlin/releases/download/v{version}/kotlin-compiler-{version}.zip"
 this_dir = pathlib.Path(__file__).resolve().parent
 version_file = this_dir / ".kotlinc_version"
 install_dir = this_dir / ".kotlinc_installed"
-windows_ripunzip = this_dir.parents[4] / "resources" / "lib" / "windows" / "ripunzip" / "ripunzip.exe"
+zips_dir = this_dir / ".kotlinc_zips"
+windows_ripunzip = (
+    this_dir.parents[4] / "resources" / "lib" / "windows" / "ripunzip" / "ripunzip.exe"
+)
 
 
 class Error(Exception):
@@ -62,16 +67,6 @@ class ZipFilePreservingPermissions(zipfile.ZipFile):
         return targetpath
 
 
-def check_version(version: str):
-    try:
-        with urllib.request.urlopen(url_template.format(version=version)) as response:
-            pass
-    except urllib.error.HTTPError as e:
-        if e.code == 404:
-            raise Error(f"Version {version} not found in github.com/JetBrains/kotlin/releases") from e
-        raise
-
-
 def get_version():
     try:
         return version_file.read_text()
@@ -86,29 +81,39 @@ def install(version: str, quiet: bool):
     else:
         info_out = sys.stderr
         info = lambda *args: print(*args, file=sys.stderr)
+    file = file_template.format(version=version)
     url = url_template.format(version=version)
     if install_dir.exists():
         shutil.rmtree(install_dir)
     install_dir.mkdir()
+    zips_dir.mkdir(exist_ok=True)
+    zip = zips_dir / file
+
+    if not zip.exists():
+        info(f"downloading {url}")
+        tmp_zip = zip.with_suffix(".tmp")
+        with open(tmp_zip, "wb") as out, urllib.request.urlopen(url) as response:
+            shutil.copyfileobj(response, out)
+        tmp_zip.rename(zip)
     ripunzip = shutil.which("ripunzip")
-    if ripunzip is None and platform.system() == "Windows" and windows_ripunzip.exists():
+    if (
+        ripunzip is None
+        and platform.system() == "Windows"
+        and windows_ripunzip.exists()
+    ):
         ripunzip = windows_ripunzip
     if ripunzip:
-        info(f"downloading and extracting {url} using ripunzip")
-        subprocess.run([ripunzip, "unzip-uri", url], stdout=info_out, stderr=info_out, cwd=install_dir,
-                       check=True)
-        return
-    with io.BytesIO() as buffer:
-        info(f"downloading {url}")
-        with urllib.request.urlopen(url) as response:
-            while True:
-                bytes = response.read()
-                if not bytes:
-                    break
-                buffer.write(bytes)
-        buffer.seek(0)
-        info(f"extracting kotlin-compiler-{version}.zip")
-        with ZipFilePreservingPermissions(buffer) as archive:
+        info(f"extracting {zip} using ripunzip")
+        subprocess.run(
+            [ripunzip, "unzip-file", zip],
+            stdout=info_out,
+            stderr=info_out,
+            cwd=install_dir,
+            check=True,
+        )
+    else:
+        info(f"extracting {zip}")
+        with ZipFilePreservingPermissions(zip) as archive:
             archive.extractall(install_dir)
 
 
@@ -130,6 +135,9 @@ def clear():
     if version_file.exists():
         print(f"removing {version_file}", file=sys.stderr)
         version_file.unlink()
+    if zips_dir.exists():
+        print(f"removing {zips_dir}", file=sys.stderr)
+        shutil.rmtree(zips_dir)
 
 
 def main(opts, forwarded_opts):
@@ -140,7 +148,6 @@ def main(opts, forwarded_opts):
     if opts.select == "default":
         selected_version = DEFAULT_VERSION
     elif opts.select is not None:
-        check_version(opts.select)
         selected_version = opts.select
     else:
         selected_version = current_version or DEFAULT_VERSION
@@ -153,7 +160,10 @@ def main(opts, forwarded_opts):
         return
     if opts.version:
         if opts.tool == "kotlinc":
-            print(f"info: kotlinc-jvm {selected_version} (codeql dev wrapper)", file=sys.stderr)
+            print(
+                f"info: kotlinc-jvm {selected_version} (codeql dev wrapper)",
+                file=sys.stderr,
+            )
             return
         forwarded_opts.append("-version")
 

java/ql/automodel/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-automodel-queries
-version: 1.0.3
+version: 1.0.4-dev
 groups:
     - java
     - automodel

java/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-all
-version: 1.1.2
+version: 1.1.3-dev
 groups: java
 dbscheme: config/semmlecode.dbscheme
 extractor: java

java/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-queries
-version: 1.1.0
+version: 1.1.1-dev
 groups:
   - java
   - queries

javascript/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/javascript-all
-version: 1.1.0
+version: 1.1.1-dev
 groups: javascript
 dbscheme: semmlecode.javascript.dbscheme
 extractor: javascript

javascript/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/javascript-queries
-version: 1.0.3
+version: 1.0.4-dev
 groups:
   - javascript
   - queries

misc/bazel/internal/install.py

@@ -11,6 +11,8 @@ import argparse
 import pathlib
 import shutil
 import subprocess
+import platform
+import time
 from python.runfiles import runfiles
 
 runfiles = runfiles.Create()
@@ -41,8 +43,25 @@ else:
     assert destdir.is_absolute(), "Provide `--build-file` to resolve destination directory"
 script = runfiles.Rlocation(opts.pkg_install_script)
 
+_WIN_FILE_IN_USE_ERROR_CODE = 32
+
 if destdir.exists() and opts.cleanup:
-    shutil.rmtree(destdir)
+    if platform.system() == 'Windows':
+        # On Windows we might have virus scanner still looking at the path so
+        # attempt removal a couple of times sleeping between each attempt.
+        for retry_delay in [1, 2, 2]:
+            try:
+                shutil.rmtree(destdir)
+                break
+            except OSError as e:
+                if e.winerror == _WIN_FILE_IN_USE_ERROR_CODE:
+                    time.sleep(retry_delay)
+                else:
+                    raise
+        else:
+            shutil.rmtree(destdir)
+    else:
+        shutil.rmtree(destdir)
 
 destdir.mkdir(parents=True, exist_ok=True)
 subprocess.run([script, "--destdir", destdir], check=True)

misc/suite-helpers/qlpack.yml

@@ -1,4 +1,4 @@
 name: codeql/suite-helpers
-version: 1.0.3
+version: 1.0.4-dev
 groups: shared
 warnOnImplicitThis: true

python/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-all
-version: 1.0.3
+version: 1.0.4-dev
 groups: python
 dbscheme: semmlecode.python.dbscheme
 extractor: python

python/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-queries
-version: 1.0.3
+version: 1.0.4-dev
 groups:
   - python
   - queries

ruby/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/ruby-all
-version: 1.0.3
+version: 1.0.4-dev
 groups: ruby
 extractor: ruby
 dbscheme: ruby.dbscheme

ruby/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/ruby-queries
-version: 1.0.3
+version: 1.0.4-dev
 groups:
   - ruby
   - queries

shared/controlflow/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/controlflow
-version: 1.0.3
+version: 1.0.4-dev
 groups: shared
 library: true
 dependencies:

shared/dataflow/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/dataflow
-version: 1.0.3
+version: 1.0.4-dev
 groups: shared
 library: true
 dependencies:

shared/mad/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/mad
-version: 1.0.3
+version: 1.0.4-dev
 groups: shared
 library: true
 dependencies:

shared/rangeanalysis/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/rangeanalysis
-version: 1.0.3
+version: 1.0.4-dev
 groups: shared
 library: true
 dependencies:

shared/regex/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/regex
-version: 1.0.3
+version: 1.0.4-dev
 groups: shared
 library: true
 dependencies:

shared/ssa/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/ssa
-version: 1.0.3
+version: 1.0.4-dev
 groups: shared
 library: true
 dependencies:

shared/threat-models/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/threat-models
-version: 1.0.3
+version: 1.0.4-dev
 library: true
 groups: shared
 dataExtensions:

shared/tutorial/qlpack.yml

@@ -1,7 +1,7 @@
 name: codeql/tutorial
 description: Library for the CodeQL detective tutorials, helping new users learn to
   write CodeQL queries.
-version: 1.0.3
+version: 1.0.4-dev
 groups: shared
 library: true
 warnOnImplicitThis: true

shared/typeflow/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/typeflow
-version: 1.0.3
+version: 1.0.4-dev
 groups: shared
 library: true
 dependencies:

shared/typetracking/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/typetracking
-version: 1.0.3
+version: 1.0.4-dev
 groups: shared
 library: true
 dependencies:

shared/typos/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/typos
-version: 1.0.3
+version: 1.0.4-dev
 groups: shared
 library: true
 warnOnImplicitThis: true

shared/util/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/util
-version: 1.0.3
+version: 1.0.4-dev
 groups: shared
 library: true
 dependencies: null

shared/xml/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/xml
-version: 1.0.3
+version: 1.0.4-dev
 groups: shared
 library: true
 dependencies:

shared/yaml/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/yaml
-version: 1.0.3
+version: 1.0.4-dev
 groups: shared
 library: true
 warnOnImplicitThis: true

swift/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/swift-all
-version: 1.0.3
+version: 1.0.4-dev
 groups: swift
 extractor: swift
 dbscheme: swift.dbscheme

swift/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/swift-queries
-version: 1.0.3
+version: 1.0.4-dev
 groups:
   - swift
   - queries