Merge branch 'main' into python/captured-variables-for-typetracking

This commit is contained in:
yoff
2023-04-25 21:32:18 +02:00
committed by GitHub
20 changed files with 168 additions and 96 deletions

View File

@@ -340,6 +340,47 @@ class Callable extends StmtParent, Member, @callable {
}
}
/**
* Holds if the given type is public and, if it is a nested type, that all of
* its enclosing types are public as well.
*/
private predicate veryPublic(RefType t) {
t.isPublic() and
(
not t instanceof NestedType or
veryPublic(t.(NestedType).getEnclosingType())
)
}
/** A callable that is the same as its source declaration. */
class SrcCallable extends Callable {
SrcCallable() { this.isSourceDeclaration() }
/**
* Holds if this callable is effectively public in the sense that it can be
* called from outside the codebase. This means either a `public` callable on
* a sufficiently public type or a `protected` callable on a sufficiently
* public non-`final` type.
*/
predicate isEffectivelyPublic() {
exists(RefType t | t = this.getDeclaringType() |
this.isPublic() and veryPublic(t)
or
this.isProtected() and not t.isFinal() and veryPublic(t)
)
or
exists(SrcRefType tsub, Method m |
veryPublic(tsub) and
tsub.hasMethod(m, _) and
m.getSourceDeclaration() = this
|
this.isPublic()
or
this.isProtected() and not tsub.isFinal()
)
}
}
/** Gets the erasure of `t1` if it is a raw type, or `t1` itself otherwise. */
private Type eraseRaw(Type t1) {
if t1 instanceof RawType then result = t1.getErasure() else result = t1

View File

@@ -0,0 +1,77 @@
/** Provides classes and predicates for exclusions related to MaD models. */
import java
/** Holds if the given package `p` is a test package. */
pragma[nomagic]
private predicate isTestPackage(Package p) {
p.getName()
.matches([
"org.junit%", "junit.%", "org.mockito%", "org.assertj%",
"com.github.tomakehurst.wiremock%", "org.hamcrest%", "org.springframework.test.%",
"org.springframework.mock.%", "org.springframework.boot.test.%", "reactor.test%",
"org.xmlunit%", "org.testcontainers.%", "org.opentest4j%", "org.mockserver%",
"org.powermock%", "org.skyscreamer.jsonassert%", "org.rnorth.visibleassertions",
"org.openqa.selenium%", "com.gargoylesoftware.htmlunit%", "org.jboss.arquillian.testng%",
"org.testng%"
])
}
/**
* A test library.
*/
class TestLibrary extends RefType {
TestLibrary() { isTestPackage(this.getPackage()) }
}
/** Holds if the given file is a test file. */
private predicate isInTestFile(File file) {
file.getAbsolutePath().matches(["%/test/%", "%/guava-tests/%", "%/guava-testlib/%"]) and
not file.getAbsolutePath().matches("%/ql/test/%") // allows our test cases to work
}
/** Holds if the given compilation unit's package is a JDK internal. */
private predicate isJdkInternal(CompilationUnit cu) {
cu.getPackage().getName().matches("org.graalvm%") or
cu.getPackage().getName().matches("com.sun%") or
cu.getPackage().getName().matches("sun%") or
cu.getPackage().getName().matches("jdk%") or
cu.getPackage().getName().matches("java2d%") or
cu.getPackage().getName().matches("build.tools%") or
cu.getPackage().getName().matches("propertiesparser%") or
cu.getPackage().getName().matches("org.jcp%") or
cu.getPackage().getName().matches("org.w3c%") or
cu.getPackage().getName().matches("org.ietf.jgss%") or
cu.getPackage().getName().matches("org.xml.sax%") or
cu.getPackage().getName().matches("com.oracle%") or
cu.getPackage().getName().matches("org.omg%") or
cu.getPackage().getName().matches("org.relaxng%") or
cu.getPackage().getName() = "compileproperties" or
cu.getPackage().getName() = "transparentruler" or
cu.getPackage().getName() = "genstubs" or
cu.getPackage().getName() = "netscape.javascript" or
cu.getPackage().getName() = ""
}
/** Holds if the given callable is not worth modeling. */
predicate isUninterestingForModels(Callable c) {
isInTestFile(c.getCompilationUnit().getFile()) or
isJdkInternal(c.getCompilationUnit()) or
c instanceof MainMethod or
c instanceof StaticInitializer or
exists(FunctionalExpr funcExpr | c = funcExpr.asMethod()) or
c.getDeclaringType() instanceof TestLibrary or
c.(Constructor).isParameterless()
}
/**
* A class that represents all callables for which we might be
* interested in having a MaD model.
*/
class ModelApi extends SrcCallable {
ModelApi() {
this.fromSource() and
this.isEffectivelyPublic() and
not isUninterestingForModels(this)
}
}

View File

@@ -1,9 +1,9 @@
private import semmle.code.java.dataflow.FlowSummary
private import utils.modelgenerator.internal.CaptureModels
private import semmle.code.java.dataflow.internal.ModelExclusions
private import TopJdkApis
/**
* Returns the number of `DataFlowTargetApi`s with Summary MaD models
* Returns the number of `ModelApi`s with Summary MaD models
* for a given package and provenance.
*/
bindingset[package, apiSubset]
@@ -13,7 +13,7 @@ private int getNumMadModeledApis(string package, string provenance, string apiSu
count(SummarizedCallable sc |
callableSubset(sc.asCallable(), apiSubset) and
package = sc.asCallable().getCompilationUnit().getPackage().getName() and
sc.asCallable() instanceof DataFlowTargetApi and
sc.asCallable() instanceof ModelApi and
(
// "auto-only"
not sc.hasManualModel() and
@@ -34,12 +34,12 @@ private int getNumMadModeledApis(string package, string provenance, string apiSu
)
}
/** Returns the total number of `DataFlowTargetApi`s for a given package. */
/** Returns the total number of `ModelApi`s for a given package. */
private int getNumApis(string package, string apiSubset) {
result =
strictcount(DataFlowTargetApi dataFlowTargApi |
callableSubset(dataFlowTargApi, apiSubset) and
package = dataFlowTargApi.getCompilationUnit().getPackage().getName()
strictcount(ModelApi api |
callableSubset(api, apiSubset) and
package = api.getCompilationUnit().getPackage().getName()
)
}
@@ -70,7 +70,7 @@ predicate modelCoverageGenVsMan(
// calculate the total generated and total manual numbers
generated = generatedOnly + both and
manual = manualOnly + both and
// count the total number of `DataFlowTargetApi`s for each package
// count the total number of `ModelApi`s for each package
all = getNumApis(package, apiSubset) and
non = all - (generatedOnly + both + manualOnly) and
// Proportion of coverage

View File

@@ -8,27 +8,7 @@ private import semmle.code.java.dataflow.FlowSummary
private import semmle.code.java.dataflow.internal.DataFlowPrivate
private import semmle.code.java.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
private import semmle.code.java.dataflow.TaintTracking
pragma[nomagic]
private predicate isTestPackage(Package p) {
p.getName()
.matches([
"org.junit%", "junit.%", "org.mockito%", "org.assertj%",
"com.github.tomakehurst.wiremock%", "org.hamcrest%", "org.springframework.test.%",
"org.springframework.mock.%", "org.springframework.boot.test.%", "reactor.test%",
"org.xmlunit%", "org.testcontainers.%", "org.opentest4j%", "org.mockserver%",
"org.powermock%", "org.skyscreamer.jsonassert%", "org.rnorth.visibleassertions",
"org.openqa.selenium%", "com.gargoylesoftware.htmlunit%", "org.jboss.arquillian.testng%",
"org.testng%"
])
}
/**
* A test library.
*/
private class TestLibrary extends RefType {
TestLibrary() { isTestPackage(this.getPackage()) }
}
private import semmle.code.java.dataflow.internal.ModelExclusions
private string containerAsJar(Container container) {
if container instanceof JarFile then result = container.getBaseName() else result = "rt.jar"

View File

@@ -5,6 +5,7 @@
private import java as J
private import semmle.code.java.dataflow.internal.DataFlowPrivate
private import semmle.code.java.dataflow.internal.ContainerFlow as ContainerFlow
private import semmle.code.java.dataflow.internal.ModelExclusions
private import semmle.code.java.dataflow.DataFlow as Df
private import semmle.code.java.dataflow.SSA as Ssa
private import semmle.code.java.dataflow.TaintTracking as Tt
@@ -26,33 +27,6 @@ private J::Method superImpl(J::Method m) {
not m instanceof J::ToStringMethod
}
private predicate isInTestFile(J::File file) {
file.getAbsolutePath().matches(["%/test/%", "%/guava-tests/%", "%/guava-testlib/%"]) and
not file.getAbsolutePath().matches("%/ql/test/%") // allows our test cases to work
}
private predicate isJdkInternal(J::CompilationUnit cu) {
cu.getPackage().getName().matches("org.graalvm%") or
cu.getPackage().getName().matches("com.sun%") or
cu.getPackage().getName().matches("sun%") or
cu.getPackage().getName().matches("jdk%") or
cu.getPackage().getName().matches("java2d%") or
cu.getPackage().getName().matches("build.tools%") or
cu.getPackage().getName().matches("propertiesparser%") or
cu.getPackage().getName().matches("org.jcp%") or
cu.getPackage().getName().matches("org.w3c%") or
cu.getPackage().getName().matches("org.ietf.jgss%") or
cu.getPackage().getName().matches("org.xml.sax%") or
cu.getPackage().getName().matches("com.oracle%") or
cu.getPackage().getName().matches("org.omg%") or
cu.getPackage().getName().matches("org.relaxng%") or
cu.getPackage().getName() = "compileproperties" or
cu.getPackage().getName() = "transparentruler" or
cu.getPackage().getName() = "genstubs" or
cu.getPackage().getName() = "netscape.javascript" or
cu.getPackage().getName() = ""
}
private predicate isInfrequentlyUsed(J::CompilationUnit cu) {
cu.getPackage().getName().matches("javax.swing%") or
cu.getPackage().getName().matches("java.awt%")
@@ -62,13 +36,8 @@ private predicate isInfrequentlyUsed(J::CompilationUnit cu) {
* Holds if it is relevant to generate models for `api`.
*/
private predicate isRelevantForModels(J::Callable api) {
not isInTestFile(api.getCompilationUnit().getFile()) and
not isJdkInternal(api.getCompilationUnit()) and
not isInfrequentlyUsed(api.getCompilationUnit()) and
not api instanceof J::MainMethod and
not api instanceof J::StaticInitializer and
not exists(J::FunctionalExpr funcExpr | api = funcExpr.asMethod()) and
not api.(J::Constructor).isParameterless()
not isUninterestingForModels(api) and
not isInfrequentlyUsed(api.getCompilationUnit())
}
/**

View File

@@ -7,8 +7,8 @@ abstract class AbstractStringBuilder implements Appendable, CharSequence
{
public char charAt(int p0){ return '0'; } // manual summary
public int length(){ return 0; } // manual neutral
public void setCharAt(int p0, char p1){} // manual neutral, Note: not currently counted by query due to exclusions in `TargetApiSpecific`
public void setLength(int p0){} // manual neutral, Note: not currently counted by query due to exclusions in `TargetApiSpecific`
public void setCharAt(int p0, char p1){} // manual neutral
public void setLength(int p0){} // manual neutral
public AbstractStringBuilder append(CharSequence p0){ return null; }

View File

@@ -7,7 +7,7 @@ import java.io.Serializable;
abstract public class Enum<E extends Enum<E>> implements Comparable<E>, Serializable
{
protected Enum() {}
protected Enum(String p0, int p1){} // manual neutral, Note: this will not be counted in query results since `protected` not `public`
protected Enum(String p0, int p1){} // manual neutral
public String toString(){ return null; } // manual neutral
public final String name(){ return null; } // manual neutral
public final boolean equals(Object p0){ return false; } // manual neutral

View File

@@ -12,7 +12,4 @@ public class StringBuffer extends AbstractStringBuilder implements Serializable
public StringBuffer append(char p0){ return null; } // manual summary
public StringBuffer append(CharSequence s, int start, int end) { return null; }
public void setCharAt(int p0, char p1){}
public void setLength(int p0){}
}

View File

@@ -19,7 +19,4 @@ public class StringBuilder extends AbstractStringBuilder implements Serializable
public StringBuilder(int p0){} // manual summary
public StringBuilder append(CharSequence s, int start, int end) { return null; }
public void setCharAt(int p0, char p1){}
public void setLength(int p0){}
}

View File

@@ -1,21 +1,23 @@
| java.io | 0 | 0 | 22 | 9 | 31 | 0.7096774193548387 | 0.0 | 0.7096774193548387 | 0.0 | NaN | 0.2903225806451613 |
| java.lang | 0 | 0 | 60 | 89 | 149 | 0.40268456375838924 | 0.0 | 0.40268456375838924 | 0.0 | NaN | 0.5973154362416108 |
| java.awt | 0 | 0 | 2 | 1 | 3 | 0.6666666666666666 | 0.0 | 0.6666666666666666 | 0.0 | NaN | 0.3333333333333333 |
| java.io | 0 | 0 | 22 | 15 | 37 | 0.5945945945945946 | 0.0 | 0.5945945945945946 | 0.0 | NaN | 0.40540540540540543 |
| java.lang | 0 | 0 | 62 | 94 | 156 | 0.3974358974358974 | 0.0 | 0.3974358974358974 | 0.0 | NaN | 0.6025641025641025 |
| java.lang.invoke | 0 | 0 | 0 | 1 | 1 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |
| java.lang.reflect | 0 | 0 | 0 | 4 | 4 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |
| java.math | 0 | 0 | 0 | 16 | 16 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |
| java.net | 0 | 0 | 5 | 0 | 5 | 1.0 | 0.0 | 1.0 | 0.0 | NaN | 0.0 |
| java.nio | 0 | 0 | 2 | 3 | 5 | 0.4 | 0.0 | 0.4 | 0.0 | NaN | 0.6 |
| java.nio.charset | 0 | 0 | 0 | 1 | 1 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |
| java.nio.file | 0 | 0 | 1 | 1 | 2 | 0.5 | 0.0 | 0.5 | 0.0 | NaN | 0.5 |
| java.sql | 0 | 0 | 0 | 2 | 2 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |
| java.nio.file | 0 | 0 | 7 | 1 | 8 | 0.875 | 0.0 | 0.875 | 0.0 | NaN | 0.125 |
| java.sql | 0 | 0 | 2 | 14 | 16 | 0.125 | 0.0 | 0.125 | 0.0 | NaN | 0.875 |
| java.text | 0 | 0 | 0 | 5 | 5 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |
| java.time | 0 | 0 | 0 | 17 | 17 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |
| java.time.chrono | 0 | 0 | 0 | 1 | 1 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |
| java.time.format | 0 | 0 | 0 | 2 | 2 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |
| java.util | 0 | 0 | 56 | 45 | 101 | 0.5544554455445545 | 0.0 | 0.5544554455445545 | 0.0 | NaN | 0.44554455445544555 |
| java.util.concurrent | 0 | 0 | 6 | 7 | 13 | 0.46153846153846156 | 0.0 | 0.46153846153846156 | 0.0 | NaN | 0.5384615384615384 |
| java.util | 0 | 0 | 84 | 68 | 152 | 0.5526315789473685 | 0.0 | 0.5526315789473685 | 0.0 | NaN | 0.4473684210526316 |
| java.util.concurrent | 0 | 0 | 9 | 9 | 18 | 0.5 | 0.0 | 0.5 | 0.0 | NaN | 0.5 |
| java.util.concurrent.atomic | 0 | 0 | 2 | 11 | 13 | 0.15384615384615385 | 0.0 | 0.15384615384615385 | 0.0 | NaN | 0.8461538461538461 |
| java.util.function | 0 | 0 | 0 | 1 | 1 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |
| java.util.concurrent.locks | 0 | 0 | 0 | 2 | 2 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |
| java.util.function | 0 | 0 | 0 | 6 | 6 | 0.0 | 0.0 | 0.0 | NaN | NaN | 1.0 |
| java.util.logging | 0 | 0 | 1 | 1 | 2 | 0.5 | 0.0 | 0.5 | 0.0 | NaN | 0.5 |
| java.util.regex | 0 | 0 | 3 | 1 | 4 | 0.75 | 0.0 | 0.75 | 0.0 | NaN | 0.25 |
| java.util.stream | 0 | 0 | 4 | 5 | 9 | 0.4444444444444444 | 0.0 | 0.4444444444444444 | 0.0 | NaN | 0.5555555555555556 |
| java.util.stream | 0 | 0 | 18 | 8 | 26 | 0.6923076923076923 | 0.0 | 0.6923076923076923 | 0.0 | NaN | 0.3076923076923077 |

View File

@@ -43,3 +43,13 @@ def codeql_workspace(repository_name = "codeql"):
"https://github.com/bazelbuild/bazel-skylib/releases/download/1.4.1/bazel-skylib-1.4.1.tar.gz",
],
)
maybe(
repo_rule = http_archive,
name = "absl",
sha256 = "cec2e5bf780532bd0ac672eb8d43c0f8bbe84ca5df8718320184034b7f59a398",
urls = [
"https://github.com/abseil/abseil-cpp/archive/d2c5297a3c3948de765100cb7e5cccca1210d23c.tar.gz",
],
strip_prefix = "abseil-cpp-d2c5297a3c3948de765100cb7e5cccca1210d23c",
)

View File

@@ -1200,6 +1200,7 @@ predicate normalCallArg(CallNode call, Node arg, ArgumentPosition apos) {
* time the bound method is used, such that the `clear()` call would essentially be
* translated into `l.clear()`, and we can still have use-use flow.
*/
pragma[assume_small_delta]
cached
predicate getCallArg(CallNode call, Function target, CallType type, Node arg, ArgumentPosition apos) {
Stages::DataFlow::ref() and

View File

@@ -16,6 +16,7 @@ swift_cc_binary(
"//swift/extractor/remapping",
"//swift/extractor/translators",
"//swift/third_party/swift-llvm-support",
"@absl//absl/strings",
],
)

View File

@@ -9,6 +9,8 @@
#include <swift/FrontendTool/FrontendTool.h>
#include <swift/Basic/InitializeSwiftModules.h>
#include "absl/strings/str_join.h"
#include "swift/extractor/SwiftExtractor.h"
#include "swift/extractor/infra/TargetDomains.h"
#include "swift/extractor/remapping/SwiftFileInterception.h"
@@ -184,18 +186,13 @@ codeql::SwiftExtractorConfiguration configure(int argc, char** argv) {
return configuration;
}
// TODO: use `absl::StrJoin` or `boost::algorithm::join`
static auto argDump(int argc, char** argv) {
std::string ret;
for (auto arg = argv + 1; arg < argv + argc; ++arg) {
ret += *arg;
ret += ' ';
if (argc < 2) {
return ""s;
}
ret.pop_back();
return ret;
return absl::StrJoin(argv + 1, argv + argc, " ");
}
// TODO: use `absl::StrJoin` or `boost::algorithm::join`
static auto envDump(char** envp) {
std::string ret;
for (auto env = envp; *env; ++env) {

View File

@@ -50,5 +50,6 @@ swift_cc_library(
deps = [
"//swift/extractor/infra/file",
"//swift/extractor/infra/log",
"@absl//absl/numeric:bits",
],
)

View File

@@ -5,6 +5,7 @@
#include <iostream>
#include <string>
#include <vector>
#include "absl/numeric/bits.h"
#include <binlog/binlog.hpp>
#include <cmath>
#include <charconv>
@@ -53,10 +54,9 @@ class UntypedTrapLabel {
private:
size_t strSize() const {
if (id_ == undefined) return 17; // #ffffffffffffffff
if (id_ == 0) return 2; // #0
// TODO: use absl::bit_width or C+20 std::bit_width instead of this ugly formula
return /* # */ 1 + /* hex digits */ static_cast<size_t>(ceil(log2(id_ + 1) / 4));
if (id_ == 0) return 2; // #0
// Number of hex digits is ceil(bit_width(id) / 4), but C++ integer division can only do floor.
return /* # */ 1 + /* hex digits */ 1 + (absl::bit_width(id_) - 1) / 4;
}
};

View File

@@ -6,7 +6,6 @@
import swift
import codeql.swift.dataflow.DataFlow
import codeql.swift.dataflow.TaintTracking
import codeql.swift.dataflow.FlowSources
import codeql.swift.security.InsecureTLSExtensions
/**

View File

@@ -13,7 +13,7 @@ private import codeql.swift.security.PredicateInjectionExtensions
* A taint-tracking configuration for predicate injection vulnerabilities.
*/
module PredicateInjectionConfig implements DataFlow::ConfigSig {
predicate isSource(DataFlow::Node source) { source instanceof RemoteFlowSource }
predicate isSource(DataFlow::Node source) { source instanceof FlowSource }
predicate isSink(DataFlow::Node sink) { sink instanceof PredicateInjectionSink }

View File

@@ -5,7 +5,7 @@
* @kind path-problem
* @problem.severity warning
* @security-severity 7.5
* @precision medium
* @precision high
* @id swift/cleartext-storage-database
* @tags security
* external/cwe/cwe-312

View File

@@ -4,7 +4,7 @@
* @kind path-problem
* @problem.severity warning
* @security-severity 7.5
* @precision medium
* @precision high
* @id swift/cleartext-storage-preferences
* @tags security
* external/cwe/cwe-312