Merge branch 'main' into andersfugmann/improve_upper_bound

cpp/ql/examples/qlpack.lock.yml

@@ -0,0 +1,4 @@
+---
+dependencies: {}
+compiled: false
+lockVersion: 1.0.0

cpp/ql/examples/qlpack.yml

@@ -1,3 +1,4 @@
-name: codeql-cpp-examples
-version: 0.0.0
-libraryPathDependencies: codeql/cpp-all
+name: codeql/cpp-examples
+version: 0.0.2
+dependencies:
+  codeql/cpp-all: "*"

cpp/ql/lib/qlpack.yml

@@ -3,3 +3,5 @@ version: 0.0.2
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp
 library: true
+dependencies:
+  codeql/cpp-upgrades: 0.0.2

cpp/ql/src/codeql-suites/cpp-code-scanning.qls

@@ -3,4 +3,4 @@
 - apply: code-scanning-selectors.yml
   from: codeql/suite-helpers
 - apply: codeql-suites/exclude-slow-queries.yml
-  from: codeql-cpp
+  from: codeql/cpp-queries

cpp/ql/src/codeql-suites/cpp-lgtm-full.qls

@@ -3,7 +3,7 @@
 - apply: lgtm-selectors.yml
   from: codeql/suite-helpers
 - apply: codeql-suites/exclude-slow-queries.yml
-  from: codeql-cpp
+  from: codeql/cpp-queries
 # These are only for IDE use.
 - exclude:
     tags contain:

cpp/ql/src/codeql-suites/cpp-security-and-quality.qls

@@ -3,4 +3,4 @@
 - apply: security-and-quality-selectors.yml
   from: codeql/suite-helpers
 - apply: codeql-suites/exclude-slow-queries.yml
-  from: codeql-cpp
+  from: codeql/cpp-queries

cpp/ql/src/codeql-suites/cpp-security-extended.qls

@@ -3,4 +3,4 @@
 - apply: security-extended-selectors.yml
   from: codeql/suite-helpers
 - apply: codeql-suites/exclude-slow-queries.yml
-  from: codeql-cpp
+  from: codeql/cpp-queries

cpp/ql/src/qlpack.yml

@@ -1,8 +1,8 @@
 name: codeql/cpp-queries
 version: 0.0.2
 dependencies:
-    codeql/cpp-all: ^0.0.2
-    codeql/suite-helpers: ^0.0.2
+    codeql/cpp-all: "*"
+    codeql/suite-helpers: "*"
 suites: codeql-suites
 extractor: cpp
 defaultSuiteFile: codeql-suites/cpp-code-scanning.qls

cpp/ql/test/qlpack.lock.yml

@@ -0,0 +1,4 @@
+---
+dependencies: {}
+compiled: false
+lockVersion: 1.0.0

cpp/ql/test/qlpack.yml

@@ -1,5 +1,5 @@
-name: codeql-cpp-tests
-version: 0.0.0
+name: codeql/cpp-tests
+version: 0.0.2
 dependencies:
   codeql/cpp-all: "*"
   codeql/cpp-queries: "*"

cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml

@@ -1,6 +1,6 @@
 # This directory has its own qlpack for reasons detailed in commit 2550788598010fa2117274607c9d58f64f997f34
-name: codeql-cpp-tests-cwe-190-tainted
-version: 0.0.0
+name: codeql/cpp-tests-cwe-190-tainted
+version: 0.0.2
 dependencies:
   codeql/cpp-all: "*"
   codeql/cpp-queries: "*"

cpp/upgrades/qlpack.lock.yml

@@ -0,0 +1,4 @@
+---
+dependencies: {}
+compiled: false
+lockVersion: 1.0.0

cpp/upgrades/qlpack.yml

@@ -1,2 +1,3 @@
-name: codeql-cpp-upgrades
+name: codeql/cpp-upgrades
 upgrades: .
+version: 0.0.2

csharp/ql/examples/qlpack.lock.yml

@@ -0,0 +1,4 @@
+---
+dependencies: {}
+compiled: false
+lockVersion: 1.0.0

csharp/ql/examples/qlpack.yml

@@ -1,4 +1,4 @@
 name: codeql-csharp-examples
-version: 0.0.0
+version: 0.0.2
 dependencies:
-  codeql/csharp-all: ^0.0.1
+  codeql/csharp-all: "*"

csharp/ql/lib/qlpack.lock.yml

@@ -0,0 +1,4 @@
+---
+dependencies: {}
+compiled: false
+lockVersion: 1.0.0

csharp/ql/lib/qlpack.yml

@@ -4,3 +4,5 @@ dbscheme: semmlecode.csharp.dbscheme
 suites: codeql-suites
 extractor: csharp
 library: true
+dependencies:
+  codeql/csharp-upgrades: 0.0.2

csharp/ql/src/qlpack.lock.yml

@@ -0,0 +1,4 @@
+---
+dependencies: {}
+compiled: false
+lockVersion: 1.0.0

csharp/ql/src/qlpack.yml

@@ -3,5 +3,5 @@ version: 0.0.2
 suites: codeql-suites
 extractor: csharp
 dependencies:
-  codeql/csharp-all: ^0.0.2
-  codeql/suite-helpers: ^0.0.2
+  codeql/csharp-all: "*"
+  codeql/suite-helpers: "*"

csharp/ql/test/qlpack.lock.yml

@@ -0,0 +1,4 @@
+---
+dependencies: {}
+compiled: false
+lockVersion: 1.0.0

csharp/ql/test/qlpack.yml

@@ -1,7 +1,7 @@
 name: codeql-csharp-tests
-version: 0.0.0
+version: 0.0.2
 dependencies:
-  codeql/csharp-all: ^0.0.2
-  codeql/csharp-queries: ^0.0.2
+  codeql/csharp-all: "*"
+  codeql/csharp-queries: "*"
 extractor: csharp
 tests: .

csharp/upgrades/qlpack.yml

@@ -1,2 +1,3 @@
-name: codeql-csharp-upgrades
+name: codeql/csharp-upgrades
 upgrades: .
+version: 0.0.2

java/ql/examples/qlpack.lock.yml

@@ -0,0 +1,4 @@
+---
+dependencies: {}
+compiled: false
+lockVersion: 1.0.0

java/ql/examples/qlpack.yml

@@ -1,3 +1,4 @@
 name: codeql-java-examples
-version: 0.0.0
-libraryPathDependencies: codeql/java-all
+version: 0.0.2
+dependencies:
+    codeql/java-all: "*"

java/ql/src/qlpack.yml

@@ -3,5 +3,5 @@ version: 0.0.2
 suites: codeql-suites
 extractor: java
 dependencies:
-    codeql/java-all: ^0.0.2
-    codeql/suite-helpers: ^0.0.2
+    codeql/java-all: "*"
+    codeql/suite-helpers: "*"

java/ql/test/qlpack.yml

@@ -1,7 +1,7 @@
 name: codeql/java-tests
-version: 0.0.0
+version: 0.0.2
 dependencies:
-    codeql/java-all: ^0.0.1
-    codeql/java-queries: ^0.0.1
+    codeql/java-all: "*"
+    codeql/java-queries: "*"
 extractor: java
 tests: .

javascript/ql/examples/qlpack.yml

@@ -1,3 +1,4 @@
 name: codeql-javascript-examples
-version: 0.0.0
-libraryPathDependencies: codeql/javascript-all
+version: 0.0.3
+dependencies:
+  codeql/javascript-all: "*"

javascript/ql/src/qlpack.yml

@@ -1,7 +1,7 @@
 name: codeql/javascript-queries
-version: 0.0.2
+version: 0.0.3
 suites: codeql-suites
 extractor: javascript
 dependencies:
-    codeql/javascript-all: ^0.0.2
-    codeql/suite-helpers: ^0.0.2
+    codeql/javascript-all: "*"
+    codeql/suite-helpers: "*"

javascript/ql/test/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/javascript-tests
-version: 0.0.0
+version: 0.0.3
 dependencies:
   codeql/javascript-all: "*"
   codeql/javascript-queries: "*"

misc/scripts/generate-code-scanning-query-list.py

@@ -3,6 +3,7 @@ import json
 import csv
 import sys
 import os
+import argparse
 
 """
 This script collects CodeQL queries that are part of code scanning query packs
@@ -12,10 +13,19 @@ Errors are printed to stderr. This script requires that 'git' and 'codeql' comma
 are on the PATH. It'll try to automatically set the CodeQL search path correctly,
 as long as you run the script from one of the following locations:
  - anywhere from within a clone of the CodeQL Git repo
- - from the parent directory of a clone of the CodeQL Git repo (assuming 'codeql' 
+ - from the parent directory of a clone of the CodeQL Git repo (assuming 'codeql'
    and 'codeql-go' directories both exist)
 """
 
+parser = argparse.ArgumentParser(__name__)
+parser.add_argument(
+    "--ignore-missing-query-packs",
+    action="store_true",
+    help="Don't fail if a query pack can't be found",
+)
+arguments = parser.parse_args()
+assert hasattr(arguments, "ignore_missing_query_packs")
+
 # Define which languages and query packs to consider
 languages = [ "cpp", "csharp", "go", "java", "javascript", "python"]
 packs = [ "code-scanning", "security-and-quality", "security-extended" ]
@@ -27,14 +37,14 @@ def prefix_repo_nwo(filename):
     This function relies on `git` being available.
 
     For example:
-        /home/alice/git/ql/java/ql/src/MyQuery.ql  
+        /home/alice/git/ql/java/ql/src/MyQuery.ql
     becomes:
         github/codeql/java/ql/src/MyQuery.ql
-     
+
     If we can't detect a known NWO (e.g. github/codeql, github/codeql-go), the
     path will be truncated to the root of the git repo:
         ql/java/ql/src/MyQuery.ql
-    
+
     If the filename is not part of a Git repo, the return value is the
     same as the input value: the whole path.
     """
@@ -45,9 +55,9 @@ def prefix_repo_nwo(filename):
     except:
         # Not a Git repo
         return filename
-    
+
     git_toplevel_dir = git_toplevel_dir_subp.stdout.strip()
-    
+
     # Detect 'github/codeql' and 'github/codeql-go' repositories by checking the remote (it's a bit
     # of a hack but will work in most cases, as long as the remotes have 'codeql' and 'codeql-go'
     # in the URL
@@ -100,7 +110,7 @@ except Exception as e:
 #
 # (and assumes the codeql-go repo is in a similar location)
 codeql_search_path = "./codeql:./codeql-go:."   # will be extended further down
-    
+
 # Extend CodeQL search path by detecting root of the current Git repo (if any). This means that you
 # can run this script from any location within the CodeQL git repository.
 try:
@@ -116,7 +126,7 @@ except:
 # Create CSV writer and write CSV header to stdout
 csvwriter = csv.writer(sys.stdout)
 csvwriter.writerow([
-    "Query filename", "Suite", "Query name", "Query ID", 
+    "Query filename", "Suite", "Query name", "Query ID",
     "Kind", "Severity", "Precision", "Tags"
 ])
 
@@ -129,16 +139,20 @@ for lang in languages:
         except Exception as e:
             # Resolving queries might go wrong if the github/codeql and github/codeql-go repositories are not
             # on the search path.
+            level = "Warning" if arguments.ignore_missing_query_packs else "Error"
             print(
-                "Warning: couldn't find query pack '%s' for language '%s'. Do you have the right repositories in the right places (search path: '%s')?" % (pack, lang, codeql_search_path),
+                "%s: couldn't find query pack '%s' for language '%s'. Do you have the right repositories in the right places (search path: '%s')?" % (level, pack, lang, codeql_search_path),
                 file=sys.stderr
-            ) 
-            continue
+            )
+            if arguments.ignore_missing_query_packs:
+                continue
+            else:
+                sys.exit("You can use '--ignore-missing-query-packs' to ignore this error")
 
         # Investigate metadata for every query by using 'codeql resolve metadata'
         for queryfile in queries_subp.stdout.strip().split("\n"):
             query_metadata_json = subprocess_run(["codeql","resolve","metadata",queryfile]).stdout.strip()
-            
+
             # Turn an absolute path to a query file into an nwo-prefixed path (e.g. github/codeql/java/ql/src/....)
             queryfile_nwo = prefix_repo_nwo(queryfile)
 
@@ -146,7 +160,7 @@ for lang in languages:
 
             # Python's CSV writer will automatically quote fields if necessary
             csvwriter.writerow([
-                queryfile_nwo, pack, 
+                queryfile_nwo, pack,
                 get_query_metadata('name', meta, queryfile_nwo),
                 get_query_metadata('id', meta, queryfile_nwo),
                 get_query_metadata('kind', meta, queryfile_nwo),
@@ -154,4 +168,3 @@ for lang in languages:
                 get_query_metadata('precision', meta, queryfile_nwo),
                 get_query_metadata('tags', meta, queryfile_nwo)
             ])
-            

python/ql/examples/qlpack.yml

@@ -1,3 +1,4 @@
 name: codeql/python-examples
-version: 0.0.0
-libraryPathDependencies: codeql/python-all
+version: 0.0.2
+dependencies:
+    codeql/python-all: "*"

python/ql/lib/qlpack.yml

@@ -4,4 +4,4 @@ dbscheme: semmlecode.python.dbscheme
 extractor: python
 library: true
 dependencies:
-    codeql/python-upgrades: ~0.0.2
+    codeql/python-upgrades: 0.0.2

python/ql/src/qlpack.yml

@@ -1,8 +1,8 @@
 name: codeql/python-queries
 version: 0.0.2
 dependencies:
-    codeql/python-all: ^0.0.2
-    codeql/suite-helpers: ^0.0.2
+    codeql/python-all: "*"
+    codeql/suite-helpers: "*"
 suites: codeql-suites
 extractor: python
 defaultSuiteFile: codeql-suites/python-code-scanning.qls

python/ql/test/qlpack.yml

@@ -1,7 +1,7 @@
 name: codeql/python-tests
-version: 0.0.0
+version: 0.0.2
 dependencies:
-    codeql/python-all: ^0.0.1
-    codeql/python-queries: ^0.0.1
+    codeql/python-all: "*"
+    codeql/python-queries: "*"
 extractor: python
 tests: .