Merge branch 'main' into kaeluka/add-provenance-to-metadata

java/ql/lib/CHANGELOG.md

@@ -1,3 +1,17 @@
+## 0.7.2
+
+### New Features
+
+* A `Diagnostic.getCompilationInfo()` predicate has been added.
+
+### Minor Analysis Improvements
+
+* Fixed a typo in the `StdlibRandomSource` class in `RandomDataSource.qll`, which caused the class to improperly model calls to the `nextBytes` method. Queries relying on `StdlibRandomSource` may see an increase in results.
+* Improved the precision of virtual dispatch of `java.io.InputStream` methods. Now, calls to these methods will not dispatch to arbitrary implementations of `InputStream` if there is a high-confidence alternative (like a models-as-data summary).
+* Added more dataflow steps for `java.io.InputStream`s that wrap other `java.io.InputStream`s.
+* Added models for the Struts 2 framework.
+* Improved the modeling of Struts 2 sources of untrusted data by tainting the whole object graph of the objects unmarshaled from an HTTP request.
+
 ## 0.7.1
 
 ### New Features

java/ql/lib/change-notes/2023-07-12-add-models-for-struts2-framework.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* Added models for the Struts 2 framework.
-

java/ql/lib/change-notes/2023-07-12-improve-sources-for-the-struts2-framework.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* Improved the modeling of Struts 2 sources of untrusted data by tainting the whole object graph of the objects unmarshaled from an HTTP request.
-

java/ql/lib/change-notes/2023-07-14-getCompilationInfo.md

@@ -1,4 +0,0 @@
----
-category: feature
----
-* A `Diagnostic.getCompilationInfo()` predicate has been added.

java/ql/lib/change-notes/2023-07-19-inputstream-dispatch.md

@@ -1,5 +0,0 @@
----
-category: minorAnalysis
----
-* Improved the precision of virtual dispatch of `java.io.InputStream` methods. Now, calls to these methods will not dispatch to arbitrary implementations of `InputStream` if there is a high-confidence alternative (like a models-as-data summary).
-  

java/ql/lib/change-notes/2023-07-19-inputstream-wrapper-steps.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* Added more dataflow steps for `java.io.InputStream`s that wrap other `java.io.InputStream`s.

java/ql/lib/change-notes/2023-08-04-mad-withoutelement.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Add support for `WithElement` and `WithoutElement` for MaD access paths.

java/ql/lib/change-notes/2023-08-07-randomdatasource-typo-fix.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* Fixed a typo in the `StdlibRandomSource` class in `RandomDataSource.qll`, which caused the class to improperly model calls to the `nextBytes` method. Queries relying on `StdlibRandomSource` may see an increase in results.

java/ql/lib/change-notes/released/0.7.2.md

@@ -0,0 +1,13 @@
+## 0.7.2
+
+### New Features
+
+* A `Diagnostic.getCompilationInfo()` predicate has been added.
+
+### Minor Analysis Improvements
+
+* Fixed a typo in the `StdlibRandomSource` class in `RandomDataSource.qll`, which caused the class to improperly model calls to the `nextBytes` method. Queries relying on `StdlibRandomSource` may see an increase in results.
+* Improved the precision of virtual dispatch of `java.io.InputStream` methods. Now, calls to these methods will not dispatch to arbitrary implementations of `InputStream` if there is a high-confidence alternative (like a models-as-data summary).
+* Added more dataflow steps for `java.io.InputStream`s that wrap other `java.io.InputStream`s.
+* Added models for the Struts 2 framework.
+* Improved the modeling of Struts 2 sources of untrusted data by tainting the whole object graph of the objects unmarshaled from an HTTP request.

java/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.7.1
+lastReleaseVersion: 0.7.2

java/ql/lib/ext/java.util.model.yml

@@ -140,7 +140,8 @@ extensions:
       - ["java.util", "LinkedHashSet", False, "LinkedHashSet", "(Collection)", "", "Argument[0].Element", "Argument[this].Element", "value", "manual"]
       - ["java.util", "LinkedList", False, "LinkedList", "(Collection)", "", "Argument[0].Element", "Argument[this].Element", "value", "manual"]
       - ["java.util", "List", True, "add", "(int,Object)", "", "Argument[1]", "Argument[this].Element", "value", "manual"]
-      - ["java.util", "List", True, "addAll", "(int,Collection)", "", "Argument[1].Element", "Argument[this].Element", "value", "manual"]
+      - ["java.util", "List", True, "addAll", "(int,Collection)", "", "Argument[1].WithElement", "Argument[this]", "value", "manual"]
+      - ["java.util", "List", True, "clear", "()", "", "Argument[this].WithoutElement", "Argument[this]", "value", "manual"]
       - ["java.util", "List", False, "copyOf", "(Collection)", "", "Argument[0].Element", "ReturnValue.Element", "value", "manual"]
       - ["java.util", "List", True, "get", "(int)", "", "Argument[this].Element", "ReturnValue", "value", "manual"]
       - ["java.util", "List", True, "listIterator", "", "", "Argument[this].Element", "ReturnValue.Element", "value", "manual"]
@@ -313,6 +314,7 @@ extensions:
       - ["java.util", "Scanner", True, "useLocale", "", "", "Argument[this]", "ReturnValue", "value", "manual"]
       - ["java.util", "Scanner", True, "useRadix", "", "", "Argument[this]", "ReturnValue", "value", "manual"]
       - ["java.util", "Set", False, "copyOf", "(Collection)", "", "Argument[0].Element", "ReturnValue.Element", "value", "manual"]
+      - ["java.util", "Set", False, "clear", "()", "", "Argument[this].WithoutElement", "Argument[this]", "value", "manual"]
       - ["java.util", "Set", False, "of", "(Object)", "", "Argument[0]", "ReturnValue.Element", "value", "manual"]
       - ["java.util", "Set", False, "of", "(Object,Object)", "", "Argument[0..1]", "ReturnValue.Element", "value", "manual"]
       - ["java.util", "Set", False, "of", "(Object,Object,Object)", "", "Argument[0..2]", "ReturnValue.Element", "value", "manual"]
@@ -424,10 +426,8 @@ extensions:
       # When `WithoutElement` is implemented, these should be changed to summary models of the form `Argument[this].WithoutElement -> Argument[this]`.
       - ["java.util", "Collection", "removeIf", "(Predicate)", "summary", "manual"]
       - ["java.util", "Iterator", "remove", "()", "summary", "manual"]
-      - ["java.util", "List", "clear", "()", "summary", "manual"]
       - ["java.util", "List", "remove", "(Object)", "summary", "manual"]
       - ["java.util", "Map", "clear", "()", "summary", "manual"]
-      - ["java.util", "Set", "clear", "()", "summary", "manual"]
       - ["java.util", "Set", "remove", "(Object)", "summary", "manual"]
       - ["java.util", "Set", "removeAll", "(Collection)", "summary", "manual"]
 

java/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-all
-version: 0.7.2-dev
+version: 0.7.3-dev
 groups: java
 dbscheme: config/semmlecode.dbscheme
 extractor: java

java/ql/lib/semmle/code/java/dataflow/ExternalFlow.qll

@@ -35,8 +35,9 @@
  *    or method, or a parameter.
  * 7. The `input` column specifies how data enters the element selected by the
  *    first 6 columns, and the `output` column specifies how data leaves the
- *    element selected by the first 6 columns. An `input` can be either "",
- *    "Argument[n]", "Argument[n1..n2]", "ReturnValue":
+ *    element selected by the first 6 columns. An `input` can be a dot separated
+ *    path consisting of either "", "Argument[n]", "Argument[n1..n2]",
+ *    "ReturnValue", "Element", "WithoutElement", or "WithElement":
  *    - "": Selects a write to the selected element in case this is a field.
  *    - "Argument[n]": Selects an argument in a call to the selected element.
  *      The arguments are zero-indexed, and `this` specifies the qualifier.
@@ -44,9 +45,15 @@
  *      the given range. The range is inclusive at both ends.
  *    - "ReturnValue": Selects a value being returned by the selected element.
  *      This requires that the selected element is a method with a body.
+ *    - "Element": Selects the collection elements of the selected element.
+ *    - "WithoutElement": Selects the selected element but without
+ *       its collection elements.
+ *    - "WithElement": Selects the collection elements of the selected element, but
+ *       points to the selected element.
  *
- *    An `output` can be either "", "Argument[n]", "Argument[n1..n2]", "Parameter",
- *    "Parameter[n]", "Parameter[n1..n2]", or "ReturnValue":
+ *    An `output` can be can be a dot separated path consisting of either "",
+ *    "Argument[n]", "Argument[n1..n2]", "Parameter", "Parameter[n]",
+ *    "Parameter[n1..n2]", "ReturnValue", or "Element":
  *    - "": Selects a read of a selected field, or a selected parameter.
  *    - "Argument[n]": Selects the post-update value of an argument in a call to the
  *      selected element. That is, the value of the argument after the call returns.
@@ -61,6 +68,7 @@
  *    - "Parameter[n1..n2]": Similar to "Parameter[n]" but selects any parameter
  *      in the given range. The range is inclusive at both ends.
  *    - "ReturnValue": Selects the return value of a call to the selected element.
+ *    - "Element": Selects the collection elements of the selected element.
  * 8. The `kind` column is a tag that can be referenced from QL to determine to
  *    which classes the interpreted elements should be added. For example, for
  *    sources "remote" indicates a default remote flow source, and for summaries

java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImplSpecific.qll

@@ -170,6 +170,10 @@ predicate neutralSummaryElement(SummarizedCallableBase c, string provenance) {
 bindingset[c]
 SummaryComponent interpretComponentSpecific(AccessPathToken c) {
   exists(Content content | parseContent(c, content) and result = SummaryComponent::content(content))
+  or
+  c = "WithoutElement" and result = SummaryComponent::withoutContent(any(CollectionContent cc))
+  or
+  c = "WithElement" and result = SummaryComponent::withContent(any(CollectionContent cc))
 }
 
 /** Gets the summary component for specification component `c`, if any. */
@@ -196,6 +200,10 @@ private string getContentSpecific(Content c) {
 /** Gets the textual representation of the content in the format used for MaD models. */
 string getMadRepresentationSpecific(SummaryComponent sc) {
   exists(Content c | sc = TContentSummaryComponent(c) and result = getContentSpecific(c))
+  or
+  sc = TWithoutContentSummaryComponent(_) and result = "WithoutElement"
+  or
+  sc = TWithContentSummaryComponent(_) and result = "WithElement"
 }
 
 bindingset[pos]

java/ql/src/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.7.2
+
+### Minor Analysis Improvements
+
+* The sanitizer in `java/potentially-weak-cryptographic-algorithm` has been improved, so the query may yield additional results.
+
 ## 0.7.1
 
 ### Minor Analysis Improvements

java/ql/src/change-notes/released/0.7.2.md

@@ -1,4 +1,5 @@
----
-category: minorAnalysis
----
+## 0.7.2
+
+### Minor Analysis Improvements
+
 * The sanitizer in `java/potentially-weak-cryptographic-algorithm` has been improved, so the query may yield additional results.

java/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.7.1
+lastReleaseVersion: 0.7.2

java/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-queries
-version: 0.7.2-dev
+version: 0.7.3-dev
 groups: 
   - java
   - queries

java/ql/test/query-tests/Metrics/GeneratedVsManualCoverage/TopJdkApisTest/TopJdkApisTest.expected

@@ -13,7 +13,7 @@
 | java.time | 0 | 0 | 0 | 17 | 17 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |
 | java.time.chrono | 0 | 0 | 0 | 1 | 1 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |
 | java.time.format | 0 | 0 | 0 | 2 | 2 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |
-| java.util | 0 | 0 | 84 | 68 | 152 | 0.5526315789473685 | 0.0 | 0.5526315789473685 | 0.0 | NaN | 0.4473684210526316 |
+| java.util | 0 | 0 | 86 | 66 | 152 | 0.5657894736842105 | 0.0 | 0.5657894736842105 | 0.0 | NaN | 0.4342105263157895 |
 | java.util.concurrent | 0 | 0 | 9 | 9 | 18 | 0.5 | 0.0 | 0.5 | 0.0 | NaN | 0.5 |
 | java.util.concurrent.atomic | 0 | 0 | 2 | 11 | 13 | 0.15384615384615385 | 0.0 | 0.15384615384615385 | 0.0 | NaN | 0.8461538461538461 |
 | java.util.concurrent.locks | 0 | 0 | 0 | 2 | 2 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |