Rename directories to include language. Also update files

codeql-sqlite-java/AddUser.java

@@ -0,0 +1,45 @@
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+
+public class AddUser {
+    public static Connection connect() {
+        Connection conn = null;
+        try {
+            String url = "jdbc:sqlite:users.sqlite";
+            conn = DriverManager.getConnection(url);
+            System.out.println("Connected...");
+        } catch (SQLException e) {
+            System.out.println(e.getMessage());
+        }
+        return conn;
+    }
+
+    static String get_user_info() {
+        System.out.println("Enter name:");
+        return System.console().readLine();
+    }
+
+    static void write_info(int id, String info) {
+        try (Connection conn = connect()) { 
+            String query = String.format("INSERT INTO users VALUES (%d, '%s')", id, info);
+            conn.createStatement().executeUpdate(query);
+            System.err.printf("Sent: %s", query);
+        } catch (SQLException e) {
+            System.out.println(e.getMessage());
+        }
+    }
+    
+    static int get_new_id() {
+        return (int)(Math.random()*100000);
+    }
+
+    public static void main(String[] args) {
+        String info;
+        int id;
+
+        info = get_user_info();
+        id = get_new_id();
+        write_info(id, info);
+    }
+}

codeql-sqlite-java/Illustrations.ql

@@ -0,0 +1,18 @@
+/**
+ * @name Illustrations
+ * @description Illustrations of some codeql classes.
+ * @kind table
+ */
+
+import java
+import semmle.code.java.dataflow.FlowSources
+import semmle.code.java.security.SqlInjectionQuery
+import QueryInjectionFlow::PathGraph
+
+// Find starting points -- UserInput etc. -- from
+// ../ql/java/ql/src/Security/CWE/CWE-089/SqlTainted.ql
+
+// using QueryInjectionSink shows a sink.  So we're missing a source.
+
+from UserInput ui, QueryInjectionSink qsi
+select ui, qsi

codeql-sqlite-java/README.org

@@ -0,0 +1,221 @@
+* Using sqlite to illustrate models-as-data
+  This description uses / recycles a codeql workshop.  
+** Build the codeql database
+   To get started, build the codeql database (adjust paths to your setup):
+   #+BEGIN_SRC sh
+     # Build the db with source commit id.
+     SRCDIR=$(pwd)
+     DB=$SRCDIR/java-sqlite-$(cd $SRCDIR && git rev-parse --short HEAD).db
+
+     echo $DB
+     test -d "$DB" && rm -fR "$DB"
+     mkdir -p "$DB"
+
+     # Use the correct codeql
+     export PATH="$(cd ../codeql && pwd):$PATH"
+     codeql database create --language=java -s . -j 8 -v $DB --command='./build.sh'
+
+     # Check for AddUser in the db
+     unzip -v $DB/src.zip | grep AddUser
+   #+END_SRC
+   Then add this database directory to your VS Code =DATABASES= tab.
+** Tests using a default query
+   You can run the stdlib query
+   [[../ql/java/ql/src/Security/CWE/CWE-089/SqlTainted.ql]] but will get no results.
+   It does point at classes to inspect -- in particular, the source and sink
+   classes.  Run [[./Illustrations.ql]]; from the command line or vs studio code.
+   Via cli:
+   #+BEGIN_SRC sh 
+     # run query
+     codeql query run                                \
+            -v                                       \
+            --database java-sqlite-e2e555c.db        \
+            --output result.bqrs                     \
+            --threads=12                             \
+            --ram=14000                              \
+            Illustrations.ql
+
+     # format results
+     codeql bqrs decode --format=text result.bqrs | sed -n '/^Result set: #select/,$p'
+   #+END_SRC
+   This shows
+   #+BEGIN_SRC text
+     Result set: #select
+     |  ui  |  qsi  |
+     +------+-------+
+     | args | query |
+   #+END_SRC
+   In the editor, these link to
+   1. =main(ARGS)= and
+   2. =conn.createStatement().executeUpdate(QUERY);= 
+   The second is correct, but =System.console().readLine();= is not found.
+   Thus, =SqlTainted.ql= will not find anything.
+
+** TODO supplement sources via the model editor
+   - [ ] We have no flow
+     + check source, sink
+     + we have a sink
+     + but ActiveThreatModelSource finds no source
+   - [ ] We can supplement in different ways
+** supplement codeql: Write full manual query: already in workshop
+** TODO supplement codeql: Add to FlowSource or a subclass
+
+       Note: this /one area/ that just has to be known.  Browsing source will *not*
+       help you.
+
+       CodeQL reading hint:
+       : class ActiveThreatModelSource extends DataFlow::Node
+       uses
+       : this.(SourceNode).getThreatModel()
+       So following the cast (SourceNode) may be useful:
+       #+BEGIN_SRC java
+         /**
+          ,* A data flow source.
+          ,*/
+         abstract class SourceNode extends DataFlow::Node 
+       #+END_SRC
+       Following the =abstract class= is promising:
+       #+BEGIN_SRC java
+         abstract class RemoteFlowSource extends SourceNode
+       #+END_SRC
+       and others.
+
+       In 
+       [[../ql/java/ql/lib/Customizations.qll]]
+       notice the comments mentioning RemoteFlowSource.
+       Use imports from [[../ql/java/ql/src/Security/CWE/CWE-089/SqlTainted.ql]]
+       but note that there are conflicts.  you will use
+       : private import semmle.code.java.dataflow.FlowSources
+       Follow this to FlowSources, and find the mentioned RemoteFlowSource
+       : abstract class RemoteFlowSource extends SourceNode
+
+       Add the custom source.  The modified [[../ql/java/ql/lib/Customizations.qll]] is
+       #+BEGIN_SRC java
+         import java
+         private import semmle.code.java.dataflow.FlowSources
+
+         class ReadLine extends RemoteFlowSource {
+           ReadLine() {
+             exists(Call read |
+               read.getCallee().getName() = "readLine" and
+               read = this.asExpr()
+             )
+           }
+
+           override string getSourceType() { result = "Console readline" }
+         }
+       #+END_SRC
+
+       Note that the predicate
+       #+BEGIN_SRC java
+         module QueryInjectionFlowConfig implements DataFlow::ConfigSig {
+           predicate isSource(DataFlow::Node src) { src instanceof ActiveThreatModelSource }
+               ...;
+         }
+       #+END_SRC
+       now also returns the readLine() result -- although we extended
+       RemoteFlowSource, not ActiveThreatModelSource
+
+** TODO supplement codeql: Add to models-as-data
+       - schema in codeql: [[../ql/java/ql/lib/semmle/code/java/dataflow/internal/ExternalFlowExtensions.qll]]
+
+       - data sample: [[../.github/codeql/extensions/jedis-db-local-java/models/redis.clients.jedis.model.yml]]
+
+       In the model editor, we see a java.io.*Console.*readline' (using =show already modeled= option)
+       #+BEGIN_SRC sh 
+         1:$ rg -i 'java.io.*Console.*readline' ql/java
+         ql/java/ql/lib/ext/generated/java.io.model.yml
+         16:      - ["java.io", "Console", False, "readLine", "()", "", "Argument[this]", "ReturnValue", "taint", "df-generated"]
+         17:      - ["java.io", "Console", False, "readLine", "(String,Object[])", "", "Argument[0]", "Argument[this]", "taint", "df-generated"]
+         18:      - ["java.io", "Console", False, "readLine", "(String,Object[])", "", "Argument[1].ArrayElement", "Argument[this]", "taint", "df-generated"]
+         19:      - ["java.io", "Console", False, "readLine", "(String,Object[])", "", "Argument[this]", "ReturnValue", "taint", "df-generated"]
+       #+END_SRC
+       note: this file is in the generated/ tree.
+
+       The current readline modeling is in the =summaryModel= section; we need it
+       in a =sourceModel=
+       #+BEGIN_SRC yaml
+         extensions:
+           - addsTo:
+               pack: codeql/java-all
+               extensible: summaryModel
+             data:
+               ...
+               - ["java.io", "Console", False, "readLine", "()", "", "Argument[this]", "ReturnValue", "taint", "df-generated"]
+               - ["java.io", "Console", False, "readLine", "(String,Object[])", "", "Argument[0]", "Argument[this]", "taint", "df-generated"]
+               - ["java.io", "Console", False, "readLine", "(String,Object[])", "", "Argument[1].ArrayElement", "Argument[this]", "taint", "df-generated"]
+               - ["java.io", "Console", False, "readLine", "(String,Object[])", "", "Argument
+       #+END_SRC
+
+       The model editor will not show this because its already modeled.  To
+       illustrate text-based additions, we'll use plain text.
+       Starting from
+       #+BEGIN_SRC yaml
+         extensions:
+           - addsTo:
+               pack: codeql/java-all
+               extensible: summaryModel
+             data:
+               ...
+               - ["java.io", "Console", False, "readLine", "()", "", "Argument[this]", "ReturnValue", "taint", "df-generated"]
+               - ["java.io", "Console", False, "readLine", "(String,Object[])", "", "Argument[0]", "Argument[this]", "taint", "df-generated"]
+               - ["java.io", "Console", False, "readLine", "(String,Object[])", "", "Argument[1].ArrayElement", "Argument[this]", "taint", "df-generated"]
+               - ["java.io", "Console", False, "readLine", "(String,Object[])", "", "Argument
+       #+END_SRC
+       and the field information
+       #+BEGIN_SRC java
+         extensible predicate sourceModel(
+           string package, string type, boolean subtypes, string name, string signature, string ext,
+           string output, string kind, string provenance, QlBuiltins::ExtensionId madId
+         );
+       #+END_SRC
+       Starting from =summaryModel=
+       #+BEGIN_SRC yaml
+         # summaryModel
+         # string package, string type, boolean subtypes, string name, string signature, string ext, string input,     string output, string kind,  string provenance, QlBuiltins::ExtensionId madId
+         - ["java.io",     "Console",   False,            "readLine",  "()",             "",         "Argument[this]", "ReturnValue", "taint",      "df-generated"]
+       #+END_SRC
+
+       we can construct the =sourceModel=
+       #+BEGIN_SRC yaml
+         extensions:
+           - addsTo:
+               pack: codeql/java-all
+               extensible: sourceModel
+             data: 
+               # sourceModel
+               # string package, string type, boolean subtypes, string name, string signature, string ext,                   string output,    string kind,   string provenance, QlBuiltins::ExtensionId madId
+               - ["java.io",     "Console",   False,            "readLine",  "()",             "",                           "ReturnValue",    "remote",      "manual"]
+
+               # # from original
+               # # summaryModel
+               # # string package, string type, boolean subtypes, string name, string signature, string ext, string input,     string output, string kind,  string provenance, QlBuiltins::ExtensionId madId
+               # - ["java.io",     "Console",   False,            "readLine",  "()",             "",         "Argument[this]", "ReturnValue", "taint",      "df-generated"]
+
+       #+END_SRC
+
+       and move this into [[../.github/codeql/extensions/sqlite-db/models/sqlite.model.yml]]
+   
+
+
+       To ensure that these model extensions are applied during query runs, include
+       this setting
+       #+begin_src javascript
+         {
+             ...,
+             "settings": {
+                 ...,
+                 "codeQL.runningQueries.useExtensionPacks": "all"
+             }
+         }
+       #+end_src
+
+       in the workspace configuration file [[../qllab.code-workspace]]
+
+       In some environments (e.g., older VS Code versions), you may also need to
+       replicate this setting in [[../.vscode/settings.json]]; there it simplifies to
+       #+begin_src javascript
+         "codeQL.runningQueries.useExtensionPacks": "all"
+       #+end_src
+
+       Now we can run [[../ql/java/ql/src/Security/CWE/CWE-089/SqlTainted.ql]] again.

codeql-sqlite-java/TaintFlowDebugging.md

@@ -0,0 +1,65 @@
+# Debugging with partial flow
+
+When there is a missing node on a path that you are trying to define there are 2 ways to figure out what the missing node is:
+
+1) Using the predicate `any()` in place of a sink. [QL documentation](https://codeql.github.com/docs/writing-codeql-queries/debugging-data-flow-queries-using-partial-flow/#partial-flow) explains the details of why this is not too performat, but it works fine when an application is small or you have a confident idea of what the missing piece is and really only need a small quick check to help out.
+
+2) [Partial flow](https://codeql.github.com/docs/writing-codeql-queries/debugging-data-flow-queries-using-partial-flow/#debugging-data-flow-queries-using-partial-flow) dataflow configuration is the preferred, more thorough solution for debugging dataflow. This configuration can allow you to see the partial pathes either forward or backward through the application and answer a question of "how far does the current path get?". This is extremely useful for cases where a node is missing in the dataflow graph, and an additional taint step is required to model the full problem.
+
+
+The QL below demonstrates what partial debugging would look like, on [this Java SqlInjection sample](https://github.com/advanced-security/codeql-workshops-staging/tree/master/java/codeql-dataflow-sql-injection).
+
+```
+/**
+ * @name introduction workshop
+ * @description Sample SQL Injection problem
+ * @id test
+ * @kind path-problem
+ * @problem.severity warning
+ */
+
+import java
+
+class ReadLineSource extends Source {
+  ReadLineSource() { this.getMethod().hasQualifiedName("java.io", "Console", "readLine") }
+}
+
+abstract class Source extends MethodCall { }
+
+class Sink extends MethodCall {
+  Sink() { this.getMethod().hasQualifiedName("java.sql", "Statement", "executeUpdate") }
+}
+
+import semmle.code.java.dataflow.TaintTracking
+
+module MyFlowConfiguration implements DataFlow::ConfigSig {
+  predicate isSource(DataFlow::Node source) { source.asExpr() instanceof Source }
+
+  predicate isSink(DataFlow::Node sink) {
+    exists(Sink sink2 | sink.asExpr() = sink2.getArgument(_))
+    //previous debug technique shown , not ideal though
+    //any()
+  }
+
+  //this is the necessary flow step to close the gap
+  //   predicate isAdditionalFlowStep(DataFlow::Node inNode, DataFlow::Node outNode) {
+  //     exists(MethodCall mc |
+  //       outNode.asExpr() = mc and
+  //       mc.getMethod().hasQualifiedName("java.lang", "String", "format") and
+  //       inNode.asExpr() = mc.getAnArgument()
+  //     )
+  //   }
+}
+
+int explorationLimit() { result = 100 }
+
+module MyFlow = DataFlow::Global<MyFlowConfiguration>;
+
+module MyPartialFlow = MyFlow::FlowExplorationFwd<explorationLimit/0>;
+
+import MyPartialFlow::PartialPathGraph
+
+from MyPartialFlow::PartialPathNode start, MyPartialFlow::PartialPathNode end
+where MyPartialFlow::partialFlow(start, end, _)
+select end, start, end, "Sql injection from $@", start, "here"
+```

codeql-sqlite-java/TaintFlowDebugging.ql

@@ -0,0 +1,60 @@
+/**
+ * @name introduction workshop
+ * @description Sample SQL Injection problem
+ * @id test
+ * @kind path-problem
+ * @problem.severity warning
+ */
+
+import java
+import semmle.code.java.dataflow.FlowSources
+import semmle.code.java.dataflow.TaintTracking
+
+class ReadLineSource extends Source {
+  ReadLineSource() { this.getMethod().hasQualifiedName("java.io", "Console", "readLine") }
+}
+
+abstract class Source extends MethodCall { }
+
+class Sink extends MethodCall {
+  Sink() { this.getMethod().hasQualifiedName("java.sql", "Statement", "executeUpdate") }
+}
+
+module MyFlowConfiguration implements DataFlow::ConfigSig {
+  predicate isSource(DataFlow::Node source) {
+    //exists(Source s | source.asExpr() = s)
+    source.asExpr() instanceof Source
+  }
+
+  predicate isSink(DataFlow::Node sink) {
+    exists(Sink sink2 | sink.asExpr() = sink2.getArgument(_))
+    //any()
+  }
+
+  predicate isBarrier(DataFlow::Node node) {
+    exists(MethodCall s |
+      s.getMethod().getName() = "hypotheticalSanitizer" and
+      s.getAnArgument() = node.asExpr()
+    )
+  }
+  //   predicate isAdditionalFlowStep(DataFlow::Node inNode, DataFlow::Node outNode) {
+  //     exists(MethodCall mc |
+  //       outNode.asExpr() = mc and
+  //       mc.getMethod().hasQualifiedName("java.lang", "String", "format") and
+  //       inNode.asExpr() = mc.getAnArgument()
+  //     )
+  //     // exists(MethodCall mc |
+  //     //   mc.getAnArgument() = inNode.asExpr() and
+  //     //   outNode.asExpr() = mc
+  //     // )
+  //   }
+}
+
+//purposely does not find the result
+module MyFlow = DataFlow::Global<MyFlowConfiguration>;
+
+import MyFlow::PathGraph
+
+from MyFlow::PathNode source, MyFlow::PathNode sink
+where MyFlow::flowPath(source, sink)
+select sink, source, sink, "Potential sql injection here "

codeql-sqlite-java/add-user

@@ -0,0 +1,3 @@
+#!/bin/sh
+java -cp ".:sqlite-jdbc-3.36.0.1.jar" AddUser $@
+

codeql-sqlite-java/admin

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+set -e
+
+script=$(basename "$0")
+
+GREEN='\033[0;32m'
+MAGENTA='\033[0;95m'
+NC='\033[0m'
+RED='\033[0;31m'
+YELLOW='\033[0;33m'
+
+help() {
+    echo -e "Usage: ./${script} [options]" \
+         "\n${YELLOW}Options: ${NC}" \
+         "\n\t -h  ${GREEN}Show Help ${NC}" \
+         "\n\t -c  ${MAGENTA}Creates a users table ${NC}" \
+         "\n\t -s  ${MAGENTA}Shows all records in the users table ${NC}" \
+         "\n\t -r  ${RED}Removes users table ${NC}" 
+}
+remove-db () {
+    rm users.sqlite
+}
+
+create-db () {
+    echo '
+    CREATE TABLE users (
+        user_id INTEGER not null,
+        name TEXT NOT NULL
+    );
+    '  | sqlite3 users.sqlite 
+}        
+
+show-db () {
+    echo '
+    SELECT * FROM users;
+    ' | sqlite3 users.sqlite 
+}
+
+if [ $# == 0 ]; then 
+   help
+   exit 0
+fi
+
+while getopts "h?csr" option
+do
+    case "${option}"
+    in
+        h|\?)
+            help
+            exit 0
+        ;;
+        c)  create-db
+        ;;
+        s)  show-db
+        ;;
+        r)  remove-db
+        ;;
+    esac
+done

codeql-sqlite-java/build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+javac -cp "sqlite-jdbc-3.36.0.1.jar" AddUser.java
+

codeql-sqlite-java/codeql-pack.lock.yml

@@ -0,0 +1,4 @@
+---
+lockVersion: 1.0.0
+dependencies: {}
+compiled: false

codeql-sqlite-java/qlpack.yml

@@ -0,0 +1,7 @@
+# Change 'getting-started' to the name of a user or organization that you have write access to.
+name: codeql-sqlite/codeql-extra-queries-java
+version: 0.0.0
+dependencies:
+  # This uses the latest version of the codeql/java-all library.
+  # You may want to change to a more precise semver string.
+  codeql/java-all: "*"

codeql-sqlite-java/sarif-summary.jq

@@ -0,0 +1,60 @@
+# -*- sh -*-
+.runs | .[] | .results | .[] |
+    ( (.ruleId, ": ", 
+       (.message.text | split("\n") | ( .[0], " [", length-1 , " more]")),
+       "\n")
+      , 
+      (if (.codeFlows != null) then
+          (.codeFlows | .[] | 
+               ("    Path\n"
+                ,
+                ( .threadFlows | .[] | .locations | .[] | .location | "        "
+                  ,
+                  ( .physicalLocation | ( .artifactLocation.uri, ":", .region.startLine, ":"))
+                  ,
+                  (.message.text, " ")
+                  , 
+                  "\n"
+                )))
+          else 
+              (.locations | .[] |
+                   ( "        "
+                     ,
+                     (.physicalLocation | ( .artifactLocation.uri, ":", .region.startLine, ":"))
+                   ))
+              ,
+              # .message.text, 
+              "\n"
+              end)
+    )  | tostring 
+
+# This script extracts the following parts of the sarif output:
+# 
+# # problem
+# "runs" : [ {
+#   "results" : [ {
+#     "ruleId" : "cpp/UncheckedErrorCode",
+
+# # path problem
+# "runs" : [ {
+#   "tool" : {
+#     "driver" : {
+#       "rules" : [ {
+#         "properties" : {
+#           "kind" : "path-problem",
+
+# "runs" : [ {
+#   "results" : [ {
+#     "ruleId" : "cpp/DangerousArithmetic",
+#     "ruleIndex" : 6,
+#     "message" : {
+#       "text" : "Potential overflow (conversion: int -> unsigned int)\nPotential overflow (con
+
+# "runs" : [ {
+#   "results" : [ {
+#     "codeFlows" : [ {
+#       "threadFlows" : [ {
+#         "locations" : [ {
+#           "location" : {
+#             "message" : {
+#               "text" : "buff"