Merge pull request #9257 from michaelnebel/java/mad-commons-io-sha

Java: Update commons-io SHA for model regeneration and update models.

.github/workflows/mad_modelDiff.yml

@@ -61,7 +61,7 @@ jobs:
             DATABASE=$2
             cd codeql-$QL_VARIANT
             SHORTNAME=`basename $DATABASE`
-            python java/ql/src/utils/model-generator/GenerateFlowModel.py $DATABASE $MODELS/${SHORTNAME}.qll
+            python java/ql/src/utils/model-generator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE $MODELS/${SHORTNAME}.qll
             mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll
             cd ..
           }

.github/workflows/mad_regenerate-models.yml

@@ -20,7 +20,7 @@ jobs:
         ref: ["placeholder"]
         include:
           - slug: "apache/commons-io"
-            ref: "8985de8fe74f6622a419b37a6eed0dbc484dc128"
+            ref: "13258ce2d07aa0e764bbaa8020af4dcd3a02a620"
         exclude:
           - slug: "placeholder"
             ref: "placeholder"

java/ql/lib/semmle/code/java/dataflow/ExternalFlow.qll

@@ -92,6 +92,7 @@ private module Frameworks {
   private import semmle.code.java.frameworks.apache.IO
   private import semmle.code.java.frameworks.apache.Lang
   private import semmle.code.java.frameworks.Flexjson
+  private import semmle.code.java.frameworks.generated
   private import semmle.code.java.frameworks.guava.Guava
   private import semmle.code.java.frameworks.jackson.JacksonSerializability
   private import semmle.code.java.frameworks.javaee.jsf.JSFRenderer

java/ql/lib/semmle/code/java/frameworks/apache/IO.qll

@@ -1,16 +1,21 @@
 /** Custom definitions related to the Apache Commons IO library. */
 
 import java
-import IOGenerated
 private import semmle.code.java.dataflow.ExternalFlow
 
-// TODO: manual models that were not generated yet
 private class ApacheCommonsIOCustomSummaryCsv extends SummaryModelCsv {
+  /**
+   * Models that are not yet auto generated or where the generated summaries will
+   * be ignored.
+   * Note that if a callable has any handwritten summary, all generated summaries
+   * will be ignored for that callable.
+   */
   override predicate row(string row) {
     row =
       [
         "org.apache.commons.io;IOUtils;false;toBufferedInputStream;;;Argument[0];ReturnValue;taint",
-        "org.apache.commons.io;IOUtils;true;writeLines;(Collection,String,Writer);;Argument[0];Argument[2];taint",
+        "org.apache.commons.io;IOUtils;true;writeLines;(Collection,String,Writer);;Argument[0].Element;Argument[2];taint",
+        "org.apache.commons.io;IOUtils;true;writeLines;(Collection,String,Writer);;Argument[1];Argument[2];taint",
         "org.apache.commons.io;IOUtils;true;toByteArray;(Reader);;Argument[0];ReturnValue;taint",
         "org.apache.commons.io;IOUtils;true;toByteArray;(Reader,String);;Argument[0];ReturnValue;taint",
       ]

java/ql/lib/semmle/code/java/frameworks/generated.qll

@@ -0,0 +1,9 @@
+/**
+ * A module importing all generated Models as Data models.
+ */
+
+import java
+
+private module GeneratedFrameworks {
+  private import apache.IOGenerated
+}

java/ql/src/utils/model-generator/RegenerateModels.py

@@ -39,8 +39,9 @@ def regenerateModel(lgtmSlug, extractedDb):
     modelFile = defaultModelPath + "/" + lgtmSlugToModelFile[lgtmSlug]
     codeQlRoot = findGitRoot()
     targetModel = codeQlRoot + "/" + modelFile
-    subprocess.check_call([codeQlRoot + "/java/ql/src/utils/model-generator/GenerateFlowModel.py", extractedDb,
-                           targetModel])
+    subprocess.check_call([codeQlRoot + "/java/ql/src/utils/model-generator/GenerateFlowModel.py",
+                           "--with-summaries", "--with-sinks",
+                           extractedDb, targetModel])
     print("Regenerated " + targetModel)
     shutil.rmtree(tmpDir)
 

java/ql/src/utils/model-generator/internal/CaptureModelsSpecific.qll

@@ -232,4 +232,8 @@ string asInputArgument(DataFlow::Node source) {
  * Holds if `kind` is a relevant sink kind for creating sink models.
  */
 bindingset[kind]
-predicate isRelevantSinkKind(string kind) { not kind = "logging" }
+predicate isRelevantSinkKind(string kind) {
+  not kind = "logging" and
+  not kind.matches("regex-use%") and
+  not kind = "write-file"
+}