Refactor into separate library and query packs

.codeqlmanifest.json

@@ -1,5 +1,6 @@
 {
     "provide": [
+        "ql/lib/qlpack.yml",
         "ql/src/qlpack.yml",
         "ql/consistency-queries/qlpack.yml",
         "ql/test/qlpack.yml",

.github/workflows/build.yml

@@ -47,17 +47,17 @@ jobs:
         run: cargo build --release
       - name: Generate dbscheme
         if: ${{ matrix.os == 'ubuntu-latest' }}
-        run: target/release/ruby-generator --dbscheme ql/src/ruby.dbscheme --library ql/src/codeql_ruby/ast/internal/TreeSitter.qll
+        run: target/release/ruby-generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql_ruby/ast/internal/TreeSitter.qll
       - uses: actions/upload-artifact@v2
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
           name: ruby.dbscheme
-          path: ql/src/ruby.dbscheme
+          path: ql/lib/ruby.dbscheme
       - uses: actions/upload-artifact@v2
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
           name: TreeSitter.qll
-          path: ql/src/codeql_ruby/ast/internal/TreeSitter.qll
+          path: ql/lib/codeql_ruby/ast/internal/TreeSitter.qll
       - uses: actions/upload-artifact@v2
         with:
           name: extractor-${{ matrix.os }}
@@ -83,7 +83,7 @@ jobs:
       - name: Build Query Pack
         run: |
           codeql/codeql pack create ql/src --output target/packs
-          PACK_FOLDER=$(readlink -f target/packs/github/codeql-ruby/*)
+          PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
           codeql/codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
           (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
       - name: Compile with previous CodeQL versions
@@ -126,7 +126,7 @@ jobs:
           path: osx64
       - run: |
           mkdir -p ruby
-          cp -r codeql-extractor.yml tools ql/src/ruby.dbscheme.stats ruby/
+          cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme.stats ruby/
           mkdir -p ruby/tools/{linux64,osx64,win64}
           cp linux64/ruby-autobuilder ruby/tools/linux64/autobuilder
           cp osx64/ruby-autobuilder ruby/tools/osx64/autobuilder
@@ -204,8 +204,8 @@ jobs:
           echo "| 2 |" > "test.expected"
           echo 'name: sample-tests
           version: 0.0.0
-          libraryPathDependencies:
-          - codeql-ruby
+          dependencies:
+            codeql/ruby-all: 0.0.1
           extractor: ruby
           tests: .
           ' > qlpack.yml

.github/workflows/dataset_measure.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches: [main]
     paths:
-      - ql/src/ruby.dbscheme
+      - ql/lib/ruby.dbscheme
   pull_request:
     branches: [main]
     paths:
-      - ql/src/ruby.dbscheme
+      - ql/lib/ruby.dbscheme
   workflow_dispatch:
 
 jobs:
@@ -73,8 +73,8 @@ jobs:
           path: stats
       - run: |
           python -m pip install --user lxml
-          find stats -name 'stats.xml' | sort | xargs python scripts/merge_stats.py --output ql/src/ruby.dbscheme.stats --normalise ruby_tokeninfo
+          find stats -name 'stats.xml' | sort | xargs python scripts/merge_stats.py --output ql/lib/ruby.dbscheme.stats --normalise ruby_tokeninfo
       - uses: actions/upload-artifact@v2
         with:
           name: ruby.dbscheme.stats
-          path: ql/src/ruby.dbscheme.stats
+          path: ql/lib/ruby.dbscheme.stats

Makefile

@@ -21,8 +21,8 @@ FILES=codeql-extractor.yml\
       tools/autobuild.sh\
       tools/qltest.sh\
       tools/autobuild.cmd\
-      ql/src/ruby.dbscheme.stats\
-      ql/src/ruby.dbscheme
+      ql/lib/ruby.dbscheme.stats\
+      ql/lib/ruby.dbscheme
 
 BIN_FILES=target/release/ruby-extractor$(EXE) target/release/ruby-autobuilder$(EXE)
 
@@ -30,7 +30,7 @@ extractor-common:
 	rm -rf build
 	mkdir build
 	mkdir build/codeql-extractor-ruby
-	cp codeql-extractor.yml ql/src/ruby.dbscheme ql/src/ruby.dbscheme.stats build/codeql-extractor-ruby
+	cp codeql-extractor.yml ql/lib/ruby.dbscheme ql/lib/ruby.dbscheme.stats build/codeql-extractor-ruby
 	cp -r tools build/codeql-extractor-ruby/
 
 .PHONY:	tools
@@ -45,8 +45,8 @@ target/release/%$(EXE):
 
 dbscheme:
 	cargo build --bin ruby-generator
-	cargo run -p ruby-generator -- --dbscheme ql/src/ruby.dbscheme --library ql/src/codeql_ruby/ast/internal/TreeSitter.qll
-	codeql query format -i ql/src/codeql_ruby/ast/internal/TreeSitter.qll
+	cargo run -p ruby-generator -- --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql_ruby/ast/internal/TreeSitter.qll
+	codeql query format -i ql/lib/codeql_ruby/ast/internal/TreeSitter.qll
 
 .PHONY:	extractor
 extractor:	$(FILES) $(BIN_FILES)
@@ -61,7 +61,7 @@ extractor:	$(FILES) $(BIN_FILES)
 	cp tools/autobuild.sh extractor-pack/tools/autobuild.sh
 	cp tools/qltest.sh extractor-pack/tools/qltest.sh
 	cp tools/autobuild.cmd extractor-pack/tools/autobuild.cmd
-	cp ql/src/ruby.dbscheme.stats extractor-pack/ruby.dbscheme.stats
-	cp ql/src/ruby.dbscheme extractor-pack/ruby.dbscheme
+	cp ql/lib/ruby.dbscheme.stats extractor-pack/ruby.dbscheme.stats
+	cp ql/lib/ruby.dbscheme extractor-pack/ruby.dbscheme
 	cp target/release/ruby-extractor$(EXE) extractor-pack/tools/$(CODEQL_PLATFORM)/extractor$(EXE)
 	cp target/release/ruby-autobuilder$(EXE) extractor-pack/tools/$(CODEQL_PLATFORM)/autobuilder$(EXE)

README.md

@@ -12,13 +12,13 @@ cargo build --release
 
 ## Generating the database schema and QL library
 
-The generated `ql/src/ruby.dbscheme` and `ql/src/codeql_ruby/ast/internal/TreeSitter.qll` files are included in the repository, but they can be re-generated as follows:
+The generated `ql/lib/ruby.dbscheme` and `ql/lib/codeql_ruby/ast/internal/TreeSitter.qll` files are included in the repository, but they can be re-generated as follows:
 
 ```bash
 # Run the generator
-cargo run --release -p ruby-generator -- --dbscheme ql/src/ruby.dbscheme --library ql/src/codeql_ruby/ast/internal/TreeSitter.qll
+cargo run --release -p ruby-generator -- --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql_ruby/ast/internal/TreeSitter.qll
 # Then auto-format the QL library
-codeql query format -i ql/src/codeql_ruby/ast/internal/TreeSitter.qll
+codeql query format -i ql/lib/codeql_ruby/ast/internal/TreeSitter.qll
 ```
 
 ## Building a CodeQL database for a Ruby program

doc/prepare-db-upgrade.md

@@ -1,6 +1,6 @@
 # Upgrading the Ruby database schema
 
-The schema (`ql/src/ruby.dbscheme`) is automatically generated from tree-sitter's `node-types.json`. When the tree-sitter grammar changes, the database schema is likely to change as well, and we need to write an upgrade script. This document explains how to do that.
+The schema (`ql/lib/ruby.dbscheme`) is automatically generated from tree-sitter's `node-types.json`. When the tree-sitter grammar changes, the database schema is likely to change as well, and we need to write an upgrade script. This document explains how to do that.
 
 ## Process Overview
 
@@ -81,11 +81,11 @@ Upgrade scripts can be a little bit fiddly, so it's essential that you test them
 
 To create the upgrade directory manually, without using `scripts/prepare-db-upgrade.sh`:
 
-1. Get a hash of the old `.dbscheme` file, from just before your changes. You can do this by checking out the code prior to your changes and running `git hash-object ql/src/ruby.dbscheme`
+1. Get a hash of the old `.dbscheme` file, from just before your changes. You can do this by checking out the code prior to your changes and running `git hash-object ql/lib/ruby.dbscheme`
 
-2. Go back to your branch and create an upgrade directory with that hash as its name, for example: `mkdir ql/src/upgrades/454f1e15151422355049dc4f1f0486a03baeffef`
+2. Go back to your branch and create an upgrade directory with that hash as its name, for example: `mkdir ql/lib/upgrades/454f1e15151422355049dc4f1f0486a03baeffef`
 
 3. Copy the old `.dbscheme` file to that directory, using the name old.dbscheme.
-   `cp ql/src/ruby.dbscheme ql/src/upgrades/454f1e15151422355049dc4f1f0486a03baeffef/old.dbscheme`
+   `cp ql/lib/ruby.dbscheme ql/lib/upgrades/454f1e15151422355049dc4f1f0486a03baeffef/old.dbscheme`
 
 4. Put a copy of your new `.dbscheme` file in that directory and create an `upgrade.properties` file (as described above).

ql/consistency-queries/qlpack.yml

@@ -1,5 +1,5 @@
-name: codeql-ruby-consistency-queries
-version: 0.0.0
-libraryPathDependencies:
-  - codeql-ruby
-extractor: ruby
+name: codeql/ruby-consistency-queries
+version: 0.0.1
+dependencies:
+  codeql/ruby-all: 0.0.1
+

ql/examples/qlpack.yml

@@ -1,3 +1,4 @@
-name: codeql-ruby-examples
-version: 0.0.0
-libraryPathDependencies: codeql-ruby
+name: codeql/ruby-examples
+version: 0.0.1
+dependencies: 
+  codeql/ruby-all: 0.0.1

ql/lib/qlpack.yml

@@ -0,0 +1,6 @@
+name: codeql/ruby-all
+version: 0.0.1
+extractor: ruby
+dbscheme: ruby.dbscheme
+upgrades: upgrades
+

ql/src/codeql-suites/ruby-code-scanning.qls

@@ -1,4 +1,5 @@
 - description: Standard Code Scanning queries for Ruby
-- qlpack: codeql-ruby
+- queries: .
 - apply: code-scanning-selectors.yml
-  from: codeql-suite-helpers
+  from: codeql/suite-helpers
+

ql/src/codeql-suites/ruby-lgtm-full.qls

@@ -1,7 +1,7 @@
 - description: Standard LGTM queries for Ruby, including ones not displayed by default
-- qlpack: codeql-ruby
+- queries: .
 - apply: lgtm-selectors.yml
-  from: codeql-suite-helpers
+  from: codeql/suite-helpers
 # These are only for IDE use.
 - exclude:
     tags contain:

ql/src/codeql-suites/ruby-lgtm.qls

@@ -1,4 +1,4 @@
 - description: Standard LGTM queries for Ruby
 - apply: codeql-suites/ruby-lgtm-full.qls
 - apply: lgtm-displayed-only.yml
-  from: codeql-suite-helpers
+  from: codeql/suite-helpers

ql/src/codeql-suites/ruby-security-and-quality.qls

@@ -1,4 +1,4 @@
 - description: Security-and-quality queries for Ruby
-- qlpack: codeql-ruby
+- queries: .
 - apply: security-and-quality-selectors.yml
-  from: codeql-suite-helpers
+  from: codeql/suite-helpers

ql/src/codeql-suites/ruby-security-extended.qls

@@ -1,4 +1,5 @@
 - description: Security-extended queries for Ruby
-- qlpack: codeql-ruby
+- queries: .
 - apply: security-extended-selectors.yml
-  from: codeql-suite-helpers
+  from: codeql/suite-helpers
+

ql/src/qlpack.yml

@@ -1,6 +1,7 @@
-name: codeql-ruby
-version: 0.0.0
-dbscheme: ruby.dbscheme
+name: codeql/ruby-queries
+version: 0.0.1
 suites: codeql-suites
-extractor: ruby
-upgrades: upgrades
+dependencies:
+  codeql/ruby-all: 0.0.1
+  codeql/suite-helpers: 0.0.1
+

ql/test/qlpack.yml

@@ -1,7 +1,7 @@
-name: codeql-ruby-tests
-version: 0.0.0
-libraryPathDependencies:
-  - codeql-ruby
-  - codeql-ruby-examples
+name: codeql/ruby-tests
+version: 0.0.1
+dependencies:
+  codeql/ruby-queries: 0.0.1
+  codeql/ruby-examples: 0.0.1
 extractor: ruby
 tests: .

scripts/create-extractor-pack.ps1

@@ -1,6 +1,6 @@
 cargo build --release
 
-cargo run --release -p ruby-generator -- --dbscheme ql/src/ruby.dbscheme --library ql/src/codeql_ruby/ast/internal/TreeSitter.qll
+cargo run --release -p ruby-generator -- --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql_ruby/ast/internal/TreeSitter.qll
 codeql query format -i ql\src\codeql_ruby\ast\internal\TreeSitter.qll
 
 rm -Recurse -Force extractor-pack

scripts/create-extractor-pack.sh

@@ -12,12 +12,12 @@ fi
 
 cargo build --release
 
-cargo run --release -p ruby-generator -- --dbscheme ql/src/ruby.dbscheme --library ql/src/codeql_ruby/ast/internal/TreeSitter.qll
-codeql query format -i ql/src/codeql_ruby/ast/internal/TreeSitter.qll
+cargo run --release -p ruby-generator -- --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql_ruby/ast/internal/TreeSitter.qll
+codeql query format -i ql/lib/codeql_ruby/ast/internal/TreeSitter.qll
 
 rm -rf extractor-pack
 mkdir -p extractor-pack
-cp -r codeql-extractor.yml tools ql/src/ruby.dbscheme ql/src/ruby.dbscheme.stats extractor-pack/
+cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme ql/lib/ruby.dbscheme.stats extractor-pack/
 mkdir -p extractor-pack/tools/${platform}
 cp target/release/ruby-extractor extractor-pack/tools/${platform}/extractor
 cp target/release/ruby-autobuilder extractor-pack/tools/${platform}/autobuilder

scripts/identical-files.json

@@ -1,27 +1,27 @@
 {
     "SSA": [
         "codeql/csharp/ql/src/semmle/code/csharp/dataflow/internal/SsaImplCommon.qll",
-        "ql/src/codeql_ruby/dataflow/internal/SsaImplCommon.qll"
+        "ql/lib/codeql_ruby/dataflow/internal/SsaImplCommon.qll"
     ],
     "DataFlow Common": [
         "codeql/csharp/ql/src/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll",
-        "ql/src/codeql_ruby/dataflow/internal/DataFlowImplCommon.qll"
+        "ql/lib/codeql_ruby/dataflow/internal/DataFlowImplCommon.qll"
     ],
     "DataFlow": [
         "codeql/csharp/ql/src/semmle/code/csharp/dataflow/internal/DataFlowImpl.qll",
-        "ql/src/codeql_ruby/dataflow/internal/DataFlowImpl.qll"
+        "ql/lib/codeql_ruby/dataflow/internal/DataFlowImpl.qll"
     ],
     "DataFlow Consistency": [
         "codeql/csharp/ql/src/semmle/code/csharp/dataflow/internal/DataFlowImplConsistency.qll",
-        "ql/src/codeql_ruby/dataflow/internal/DataFlowImplConsistency.qll"
+        "ql/lib/codeql_ruby/dataflow/internal/DataFlowImplConsistency.qll"
     ],
     "TaintTracking": [
         "codeql/csharp/ql/src/semmle/code/csharp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
-        "ql/src/codeql_ruby/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
+        "ql/lib/codeql_ruby/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
     ],
     "TypeTracker": [
         "codeql/python/ql/src/semmle/python/dataflow/new/internal/TypeTracker.qll",
-        "ql/src/codeql_ruby/typetracking/TypeTracker.qll"
+        "ql/lib/codeql_ruby/typetracking/TypeTracker.qll"
     ],
   "Inline Test Expectations": [
     "codeql/python/ql/test/TestUtilities/InlineExpectationsTest.qll",

scripts/prepare-db-upgrade.sh

@@ -61,8 +61,8 @@ if [ $# -gt 0 ]; then
   usage 2 "Unrecognised operand: $1"
 fi
 
-scheme_file="ql/src/ruby.dbscheme"
-upgrade_root="ql/src/upgrades"
+scheme_file="ql/lib/ruby.dbscheme"
+upgrade_root="ql/lib/upgrades"
 
 check_hash_valid()
 {