Merge pull request #5342 from shati-patel/docs-delete-unused-script

Docs: Remove unused script and workflow

.github/workflows/check-change-note.yml

@@ -0,0 +1,21 @@
+on:
+  pull_request_target:
+    types: [labeled, unlabeled, opened, synchronize, reopened, ready_for_review]
+    paths:
+      - "*/ql/src/**/*.ql"
+      - "*/ql/src/**/*.qll"
+      - "!**/experimental/**"
+
+jobs:
+  check-change-note:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fail if no change note found. To fix, either add one, or add the `no-change-note-required` label.
+        if: |
+          github.event.pull_request.draft == false &&
+          !contains(github.event.pull_request.labels.*.name, 'no-change-note-required')
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate |
+          jq 'any(.[].filename ; test("/change-notes/.*[.]md$"))' --exit-status

.github/workflows/codeql-analysis.yml

@@ -9,6 +9,8 @@ on:
     branches:
       - main
       - 'rc/*'
+    paths:
+      - 'csharp/**'
   schedule:
     - cron: '0 9 * * 1'
 

.github/workflows/generate-query-help-docs.yml

@@ -1,60 +0,0 @@
-name: Generate CodeQL query help documentation using Sphinx
-
-on:
-  workflow_dispatch:
-      inputs:
-        description:
-          description: A description of the purpose of this job. For human consumption.
-          required: false
-  push:
-    branches:
-     - 'lgtm.com'
-  pull_request:
-    paths:
-      - '.github/workflows/generate-query-help-docs.yml'
-      - 'docs/codeql/query-help/**'
-      
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Clone github/codeql
-      uses: actions/checkout@v2
-      with:
-        path: codeql
-    - name: Clone github/codeql-go
-      uses: actions/checkout@v2
-      with:
-        repository: 'github/codeql-go'
-        path: codeql-go
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      uses: dsaltares/fetch-gh-release-asset@aa37ae5c44d3c9820bc12fe675e8670ecd93bd1c
-      with:
-        repo: "github/codeql-cli-binaries"
-        version: "latest"
-        file: "codeql-linux64.zip"
-        token: ${{ secrets.GITHUB_TOKEN }}
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
-    - name: Set up query help docs folder  
-      run: |
-        cp -r codeql/docs/codeql/** .
-    - name: Query help to markdown
-      run: |
-        PATH="$PATH:codeql-cli/codeql" python codeql/docs/codeql/query-help-markdown.py
-    - name: Run Sphinx for query help
-      uses: ammaraskar/sphinx-action@8b4f60114d7fd1faeba1a712269168508d4750d2 # v0.4
-      with:
-        docs-folder: "query-help/"
-        pre-build-command: "python -m pip install --upgrade recommonmark && python -m pip install --upgrade sphinx-markdown-tables"
-        build-command: "sphinx-build -b dirhtml . _build" 
-    - name: Upload HTML artifacts
-      uses: actions/upload-artifact@v2
-      with:
-        name: query-help-html
-        path: query-help/_build
-    

CODEOWNERS

@@ -4,17 +4,9 @@
 /javascript/ @github/codeql-javascript
 /python/ @github/codeql-python
 
-# Assign query help for docs review
-/cpp/**/*.qhelp @hubwriter
-/csharp/**/*.qhelp @jf205
-/java/**/*.qhelp @felicitymay
-/javascript/**/*.qhelp @mchammer01
-/python/**/*.qhelp @felicitymay
-/docs/language/ @shati-patel @jf205
-
-# Exclude help for experimental queries from docs review
-/cpp/**/experimental/**/*.qhelp @github/codeql-c-analysis
-/csharp/**/experimental/**/*.qhelp @github/codeql-csharp
-/java/**/experimental/**/*.qhelp @github/codeql-java
-/javascript/**/experimental/**/*.qhelp @github/codeql-javascript
-/python/**/experimental/**/*.qhelp @github/codeql-python
+# Make @xcorail (GitHub Security Lab) a code owner for experimental queries so he gets pinged when we promote a query out of experimental
+/cpp/**/experimental/**/* @github/codeql-c-analysis @xcorail
+/csharp/**/experimental/**/* @github/codeql-csharp @xcorail
+/java/**/experimental/**/* @github/codeql-java @xcorail
+/javascript/**/experimental/**/* @github/codeql-javascript @xcorail
+/python/**/experimental/**/* @github/codeql-python @xcorail

config/identical-files.json

@@ -425,5 +425,10 @@
     "java/ql/src/IDEContextual.qll",
     "javascript/ql/src/IDEContextual.qll",
     "python/ql/src/analysis/IDEContextual.qll"
+  ],
+  "SSA C#": [
+    "csharp/ql/src/semmle/code/csharp/dataflow/internal/SsaImplCommon.qll",
+    "csharp/ql/src/semmle/code/csharp/controlflow/internal/pressa/SsaImplCommon.qll",
+    "csharp/ql/src/semmle/code/csharp/dataflow/internal/basessa/SsaImplCommon.qll"
   ]
-}
+}

cpp/change-notes/2021-02-24-memset-may-be-deleted.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* A new query (`cpp/memset-may-be-deleted`) is added to the default query suite. The query finds calls to `memset` that may be removed by the compiler. This behavior can make information-leak vulnerabilities easier to exploit. This query was originally [submitted as an experimental query by @ihsinme](https://github.com/github/codeql/pull/4953).

cpp/ql/src/Security/CWE/CWE-014/MemsetMayBeDeleted-bad.c

@@ -0,0 +1,3 @@
+char password[MAX_PASSWORD_LENGTH];
+// read and verify password
+memset(password, 0, MAX_PASSWORD_LENGTH);

cpp/ql/src/Security/CWE/CWE-014/MemsetMayBeDeleted-good.c

@@ -0,0 +1,3 @@
+char password[MAX_PASSWORD_LENGTH];
+// read and verify password
+memset_s(password, MAX_PASSWORD_LENGTH, 0, MAX_PASSWORD_LENGTH);

cpp/ql/src/Security/CWE/CWE-014/MemsetMayBeDeleted.qhelp

@@ -0,0 +1,45 @@
+<!DOCTYPE qhelp PUBLIC
+  "-//Semmle//qhelp//EN"
+  "qhelp.dtd">
+<qhelp>
+<overview>
+<p>Calling <code>memset</code> or <code>bzero</code> on a buffer to clear its contents may get optimized
+away by the compiler if the buffer is not subsequently used. This is not desirable behavior if the buffer
+contains sensitive data that could somehow be retrieved by an attacker.</p>
+
+</overview>
+<recommendation>
+
+<p>Use alternative platform-supplied functions that will not get optimized away. Examples of such
+functions include <code>memset_s</code>, <code>SecureZeroMemory</code>, and <code>bzero_explicit</code>.
+Alternatively, passing the <code>-fno-builtin-memset</code> option to the GCC/Clang compiler usually
+also prevents the optimization. Finally, you can use the public-domain <code>secure_memzero</code> function
+(see references below). This function, however, is not guaranteed to work on all platforms and compilers.</p>
+
+</recommendation>
+<example>
+<p>The following program fragment uses <code>memset</code> to erase sensitive information after it is no
+longer needed:</p>
+<sample src="MemsetMayBeDeleted-bad.c" />
+<p>Because of dead store elimination, the call to <code>memset</code> may be removed by the compiler
+(since the buffer is not subsequently used), resulting in potentially sensitive data remaining in memory.
+</p>
+
+<p>The best solution to this problem is to use the <code>memset_s</code> function instead of
+<code>memset</code>:</p>
+<sample src="MemsetMayBeDeleted-good.c" />
+
+</example>
+<references>
+
+<li>
+CERT C Coding Standard:
+<a href="https://wiki.sei.cmu.edu/confluence/display/c/MSC06-C.+Beware+of+compiler+optimizations">MSC06-C. Beware of compiler optimizations</a>.
+</li>
+<li>
+USENIX: The Advanced Computing Systems Association:
+<a href="https://www.usenix.org/system/files/conference/usenixsecurity17/sec17-yang.pdf">Dead Store Elimination (Still) Considered Harmfuls</a>
+</li>
+
+</references>
+</qhelp>

cpp/ql/src/Security/CWE/CWE-014/MemsetMayBeDeleted.ql

@@ -0,0 +1,81 @@
+/**
+ * @name Call to `memset` may be deleted
+ * @description Using the `memset` function to clear private data in a variable that has no subsequent use
+ *              can make information-leak vulnerabilities easier to exploit because the compiler can remove the call.
+ * @kind problem
+ * @id cpp/memset-may-be-deleted
+ * @problem.severity warning
+ * @precision high
+ * @tags security
+ *       external/cwe/cwe-14
+ */
+
+import cpp
+import semmle.code.cpp.dataflow.EscapesTree
+import semmle.code.cpp.commons.Exclusions
+import semmle.code.cpp.models.interfaces.Alias
+
+class MemsetFunction extends Function {
+  MemsetFunction() {
+    this.hasGlobalOrStdOrBslName("memset")
+    or
+    this.hasGlobalOrStdName("wmemset")
+    or
+    this.hasGlobalName(["bzero", "__builtin_memset"])
+  }
+}
+
+predicate isNonEscapingArgument(Expr escaped) {
+  exists(Call call, AliasFunction aliasFunction, int i |
+    aliasFunction = call.getTarget() and
+    call.getArgument(i) = escaped.getUnconverted() and
+    (
+      aliasFunction.parameterNeverEscapes(i)
+      or
+      aliasFunction.parameterEscapesOnlyViaReturn(i) and
+      (call instanceof ExprInVoidContext or call.getConversion*() instanceof BoolConversion)
+    )
+  )
+}
+
+pragma[noinline]
+predicate callToMemsetWithRelevantVariable(
+  LocalVariable v, VariableAccess acc, FunctionCall call, MemsetFunction memset
+) {
+  not v.isStatic() and
+  // Reference-typed variables get special treatment in `variableAddressEscapesTree` so we leave them
+  // out of this query.
+  not v.getUnspecifiedType() instanceof ReferenceType and
+  call.getTarget() = memset and
+  acc = v.getAnAccess() and
+  // `v` escapes as the argument to `memset`
+  variableAddressEscapesTree(acc, call.getArgument(0).getFullyConverted())
+}
+
+pragma[noinline]
+predicate relevantVariable(LocalVariable v, FunctionCall call, MemsetFunction memset) {
+  exists(VariableAccess acc, VariableAccess anotherAcc |
+    callToMemsetWithRelevantVariable(v, acc, call, memset) and
+    // `v` is not only just used in the call to `memset`.
+    anotherAcc = v.getAnAccess() and
+    acc != anotherAcc and
+    not anotherAcc.isUnevaluated()
+  )
+}
+
+from FunctionCall call, LocalVariable v, MemsetFunction memset
+where
+  relevantVariable(v, call, memset) and
+  not isFromMacroDefinition(call) and
+  // `v` doesn't escape anywhere else.
+  forall(Expr escape | variableAddressEscapesTree(v.getAnAccess(), escape) |
+    isNonEscapingArgument(escape)
+  ) and
+  // There is no later use of `v`.
+  not v.getAnAccess() = call.getASuccessor*() and
+  // Not using the `-fno-builtin-memset` flag
+  exists(Compilation c |
+    c.getAFileCompiled() = call.getFile() and
+    not c.getAnArgument() = "-fno-builtin-memset"
+  )
+select call, "Call to " + memset.getName() + " may be deleted by the compiler."

cpp/ql/src/Security/CWE/CWE-676/DangerousFunctionOverflow.ql

@@ -8,6 +8,7 @@
  * @tags reliability
  *       security
  *       external/cwe/cwe-242
+ *       external/cwe/cwe-676
  */
 
 import cpp

cpp/ql/src/experimental/Security/CWE/CWE-14/CompilerRemovalOfCodeToClearBuffers.c

@@ -1,35 +0,0 @@
-// BAD: the memset call will probably be removed.
-void getPassword(void) {
-  char pwd[64];
-  if (GetPassword(pwd, sizeof(pwd))) {
-    /* Checking of password, secure operations, etc. */
-  }
-  memset(pwd, 0, sizeof(pwd));
-}
-// GOOD: in this case the memset will not be removed.
-void getPassword(void) {
-  char pwd[64];
- 
-  if (retrievePassword(pwd, sizeof(pwd))) {
-     /* Checking of password, secure operations, etc. */
-  }
-  memset_s(pwd, 0, sizeof(pwd));
-}
-// GOOD: in this case the memset will not be removed.
-void getPassword(void) {
-  char pwd[64];
-  if (retrievePassword(pwd, sizeof(pwd))) {
-    /* Checking of password, secure operations, etc. */
-  }
-  SecureZeroMemory(pwd, sizeof(pwd));
-}
-// GOOD: in this case the memset will not be removed.
-void getPassword(void) {
-  char pwd[64];
-  if (retrievePassword(pwd, sizeof(pwd))) {
-    /* Checking of password, secure operations, etc. */
-  }
-#pragma optimize("", off)
-  memset(pwd, 0, sizeof(pwd));
-#pragma optimize("", on)
-}

cpp/ql/src/experimental/Security/CWE/CWE-14/CompilerRemovalOfCodeToClearBuffers.qhelp

@@ -1,31 +0,0 @@
-<!DOCTYPE qhelp PUBLIC
-  "-//Semmle//qhelp//EN"
-  "qhelp.dtd">
-<qhelp>
-<overview>
-<p>Compiler optimization will exclude the cleaning of private information.
-Using the <code>memset</code> function to clear private data in a variable that has no subsequent use is potentially dangerous, since the compiler can remove the call.
-For some compilers, optimization is also possible when using calls to free memory after the <code>memset</code> function.</p>
-
-<p>It is possible to miss detection of vulnerabilities if used to clear fields of structures or parts of a buffer.</p>
-
-</overview>
-<recommendation>
-
-<p>We recommend to use the <code>RtlSecureZeroMemory</code> or <code>memset_s</code> functions, or compilation flags that exclude optimization of <code>memset</code> calls (e.g. -fno-builtin-memset).</p>
-
-</recommendation>
-<example>
-<p>The following example demonstrates an erroneous and corrected use of the <code>memset</code> function.</p>
-<sample src="CompilerRemovalOfCodeToClearBuffers.c" />
-
-</example>
-<references>
-
-<li>
-  CERT C Coding Standard:
-  <a href="https://wiki.sei.cmu.edu/confluence/display/c/MSC06-C.+Beware+of+compiler+optimizations">MSC06-C. Beware of compiler optimizations</a>.
-</li>
-
-</references>
-</qhelp>

cpp/ql/src/experimental/Security/CWE/CWE-14/CompilerRemovalOfCodeToClearBuffers.ql

@@ -1,127 +0,0 @@
-/**
- * @name Compiler Removal Of Code To Clear Buffers
- * @description Using <code>memset</code> the function to clear private data in a variable that has no subsequent use
- *              is potentially dangerous because the compiler can remove the call.
- * @kind problem
- * @id cpp/compiler-removal-of-code-to-clear-buffers
- * @problem.severity warning
- * @precision medium
- * @tags security
- *       external/cwe/cwe-14
- */
-
-import cpp
-import semmle.code.cpp.dataflow.DataFlow
-import semmle.code.cpp.dataflow.StackAddress
-
-/**
- * A call to `memset` of the form `memset(ptr, value, num)`, for some local variable `ptr`.
- */
-class CompilerRemovaMemset extends FunctionCall {
-  CompilerRemovaMemset() {
-    this.getTarget().hasGlobalOrStdName("memset") and
-    exists(DataFlow::Node source, DataFlow::Node sink, LocalVariable isv, Expr exp |
-      DataFlow::localFlow(source, sink) and
-      this.getArgument(0) = isv.getAnAccess() and
-      (
-        source.asExpr() = exp
-        or
-        // handle the case where exp is defined by an address being passed into some function.
-        source.asDefiningArgument() = exp
-      ) and
-      exp.getLocation().getEndLine() < this.getArgument(0).getLocation().getStartLine() and
-      sink.asExpr() = this.getArgument(0)
-    )
-  }
-
-  predicate isExistsAllocForThisVariable() {
-    exists(AllocationExpr alloc, Variable v |
-      alloc = v.getAnAssignedValue() and
-      this.getArgument(0) = v.getAnAccess() and
-      alloc.getASuccessor+() = this
-    )
-    or
-    not stackPointerFlowsToUse(this.getArgument(0), _, _, _)
-  }
-
-  predicate isExistsFreeForThisVariable() {
-    exists(DeallocationExpr free, Variable v |
-      this.getArgument(0) = v.getAnAccess() and
-      free.getFreedExpr() = v.getAnAccess() and
-      this.getASuccessor+() = free
-    )
-  }
-
-  predicate isExistsCallWithThisVariableExcludingDeallocationCalls() {
-    exists(FunctionCall fc, Variable v |
-      not fc instanceof DeallocationExpr and
-      this.getArgument(0) = v.getAnAccess() and
-      fc.getAnArgument() = v.getAnAccess() and
-      this.getASuccessor+() = fc
-    )
-  }
-
-  predicate isVariableUseAfterMemsetExcludingCalls() {
-    exists(DataFlow::Node source, DataFlow::Node sink, LocalVariable isv, Expr exp |
-      DataFlow::localFlow(source, sink) and
-      this.getArgument(0) = isv.getAnAccess() and
-      source.asExpr() = isv.getAnAccess() and
-      exp.getLocation().getStartLine() > this.getArgument(2).getLocation().getEndLine() and
-      not exp.getParent() instanceof FunctionCall and
-      sink.asExpr() = exp
-    )
-  }
-
-  predicate isVariableUseBoundWithArgumentFunction() {
-    exists(DataFlow::Node source, DataFlow::Node sink, LocalVariable isv, Parameter p, Expr exp |
-      DataFlow::localFlow(source, sink) and
-      this.getArgument(0) = isv.getAnAccess() and
-      this.getEnclosingFunction().getAParameter() = p and
-      exp.getAChild*() = p.getAnAccess() and
-      source.asExpr() = exp and
-      sink.asExpr() = isv.getAnAccess()
-    )
-  }
-
-  predicate isVariableUseBoundWithGlobalVariable() {
-    exists(
-      DataFlow::Node source, DataFlow::Node sink, LocalVariable isv, GlobalVariable gv, Expr exp
-    |
-      DataFlow::localFlow(source, sink) and
-      this.getArgument(0) = isv.getAnAccess() and
-      exp.getAChild*() = gv.getAnAccess() and
-      source.asExpr() = exp and
-      sink.asExpr() = isv.getAnAccess()
-    )
-  }
-
-  predicate isExistsCompilationFlagsBlockingRemoval() {
-    exists(Compilation c |
-      c.getAFileCompiled() = this.getFile() and
-      c.getAnArgument() = "-fno-builtin-memset"
-    )
-  }
-
-  predicate isUseVCCompilation() {
-    exists(Compilation c |
-      c.getAFileCompiled() = this.getFile() and
-      (
-        c.getArgument(2).matches("%gcc%") or
-        c.getArgument(2).matches("%g++%") or
-        c.getArgument(2).matches("%clang%") or
-        c.getArgument(2) = "--force-recompute"
-      )
-    )
-  }
-}
-
-from CompilerRemovaMemset fc
-where
-  not (fc.isExistsAllocForThisVariable() and not fc.isExistsFreeForThisVariable()) and
-  not (fc.isExistsFreeForThisVariable() and not fc.isUseVCCompilation()) and
-  not fc.isVariableUseAfterMemsetExcludingCalls() and
-  not fc.isExistsCallWithThisVariableExcludingDeallocationCalls() and
-  not fc.isVariableUseBoundWithArgumentFunction() and
-  not fc.isVariableUseBoundWithGlobalVariable() and
-  not fc.isExistsCompilationFlagsBlockingRemoval()
-select fc.getArgument(0), "This variable will not be cleared."

cpp/ql/src/semmle/code/cpp/Function.qll

@@ -467,7 +467,7 @@ class Function extends Declaration, ControlFlowNode, AccessHolder, @function {
       // ... and likewise for destructors.
       this.(Destructor).getADestruction().mayBeGloballyImpure()
     else
-      // Unless it's a function that we know is side-effect-free, it may
+      // Unless it's a function that we know is side-effect free, it may
       // have side-effects.
       not this.hasGlobalOrStdName([
           "strcmp", "wcscmp", "_mbscmp", "strlen", "wcslen", "_mbslen", "_mbslen_l", "_mbstrlen",

cpp/ql/src/semmle/code/cpp/dataflow/internal/AddressFlow.qll

@@ -131,7 +131,22 @@ private predicate lvalueToUpdate(Expr lvalue, Expr outer, ControlFlowNode node)
     exists(Call call | node = call |
       outer = call.getQualifier().getFullyConverted() and
       outer.getUnspecifiedType() instanceof Class and
-      not call.getTarget().hasSpecifier("const")
+      not (
+        call.getTarget().hasSpecifier("const") and
+        // Given the following program:
+        // ```
+        // struct C {
+        //   void* data_;
+        //   void* data() const { return data; }
+        // };
+        // C c;
+        // memcpy(c.data(), source, 16)
+        // ```
+        // the data pointed to by `c.data_` is potentially modified by the call to `memcpy` even though
+        // `C::data` has a const specifier. So we further place the restriction that the type returned
+        // by `call` should not be of the form `const T*` (for some deeply const type `T`).
+        call.getType().isDeeplyConstBelow()
+      )
     )
     or
     assignmentTo(outer, node)
@@ -170,7 +185,11 @@ private predicate pointerToUpdate(Expr pointer, Expr outer, ControlFlowNode node
       or
       outer = call.getQualifier().getFullyConverted() and
       outer.getUnspecifiedType() instanceof PointerType and
-      not call.getTarget().hasSpecifier("const")
+      not (
+        call.getTarget().hasSpecifier("const") and
+        // See the `lvalueToUpdate` case for an explanation of this conjunct.
+        call.getType().isDeeplyConstBelow()
+      )
     )
     or
     exists(PointerFieldAccess fa |

cpp/ql/src/semmle/code/cpp/models/Models.qll

@@ -30,3 +30,6 @@ private import implementations.SmartPointer
 private import implementations.Sscanf
 private import implementations.Send
 private import implementations.Recv
+private import implementations.Accept
+private import implementations.Poll
+private import implementations.Select

cpp/ql/src/semmle/code/cpp/models/implementations/Accept.qll

@@ -0,0 +1,56 @@
+/**
+ * Provides implementation classes modeling `accept` and various similar
+ * functions. See `semmle.code.cpp.models.Models` for usage information.
+ */
+
+import semmle.code.cpp.Function
+import semmle.code.cpp.models.interfaces.ArrayFunction
+import semmle.code.cpp.models.interfaces.Taint
+import semmle.code.cpp.models.interfaces.Alias
+import semmle.code.cpp.models.interfaces.SideEffect
+
+/**
+ * The function `accept` and its assorted variants
+ */
+private class Accept extends ArrayFunction, AliasFunction, TaintFunction, SideEffectFunction {
+  Accept() { this.hasGlobalName(["accept", "accept4", "WSAAccept"]) }
+
+  override predicate hasArrayWithVariableSize(int bufParam, int countParam) {
+    bufParam = 1 and countParam = 2
+  }
+
+  override predicate hasArrayInput(int bufParam) { bufParam = 1 }
+
+  override predicate hasArrayOutput(int bufParam) { bufParam = 1 }
+
+  override predicate parameterNeverEscapes(int index) { exists(this.getParameter(index)) }
+
+  override predicate parameterEscapesOnlyViaReturn(int index) { none() }
+
+  override predicate parameterIsAlwaysReturned(int index) { none() }
+
+  override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) {
+    (input.isParameter(0) or input.isParameterDeref(1)) and
+    (output.isReturnValue() or output.isParameterDeref(1))
+  }
+
+  override predicate hasSpecificWriteSideEffect(ParameterIndex i, boolean buffer, boolean mustWrite) {
+    i = 1 and buffer = true and mustWrite = false
+    or
+    i = 2 and buffer = false and mustWrite = false
+  }
+
+  override predicate hasSpecificReadSideEffect(ParameterIndex i, boolean buffer) {
+    i = 0 and buffer = true
+    or
+    i = 1 and buffer = false
+  }
+
+  override ParameterIndex getParameterSizeIndex(ParameterIndex i) { i = 1 and result = 2 }
+
+  // NOTE: We implement thse two predicates as none because we can't model the low-level changes made to
+  // the structure pointed to by the file-descriptor argument.
+  override predicate hasOnlySpecificReadSideEffects() { none() }
+
+  override predicate hasOnlySpecificWriteSideEffects() { none() }
+}

cpp/ql/src/semmle/code/cpp/models/implementations/Poll.qll

@@ -0,0 +1,44 @@
+/**
+ * Provides implementation classes modeling `poll` and various similar
+ * functions. See `semmle.code.cpp.models.Models` for usage information.
+ */
+
+import semmle.code.cpp.Function
+import semmle.code.cpp.models.interfaces.ArrayFunction
+import semmle.code.cpp.models.interfaces.Alias
+import semmle.code.cpp.models.interfaces.SideEffect
+
+/**
+ * The function `poll` and its assorted variants
+ */
+private class Poll extends ArrayFunction, AliasFunction, SideEffectFunction {
+  Poll() { this.hasGlobalName(["poll", "ppoll", "WSAPoll"]) }
+
+  override predicate hasArrayWithVariableSize(int bufParam, int countParam) {
+    bufParam = 0 and countParam = 1
+  }
+
+  override predicate hasArrayInput(int bufParam) { bufParam = 0 }
+
+  override predicate hasArrayOutput(int bufParam) { bufParam = 0 }
+
+  override predicate parameterNeverEscapes(int index) { exists(this.getParameter(index)) }
+
+  override predicate parameterEscapesOnlyViaReturn(int index) { none() }
+
+  override predicate parameterIsAlwaysReturned(int index) { none() }
+
+  override predicate hasSpecificWriteSideEffect(ParameterIndex i, boolean buffer, boolean mustWrite) {
+    i = 0 and buffer = true and mustWrite = false
+  }
+
+  override predicate hasSpecificReadSideEffect(ParameterIndex i, boolean buffer) {
+    i = 0 and buffer = true
+    or
+    this.hasGlobalName("ppoll") and i = [2, 3] and buffer = false
+  }
+
+  override predicate hasOnlySpecificReadSideEffects() { any() }
+
+  override predicate hasOnlySpecificWriteSideEffects() { any() }
+}

cpp/ql/src/semmle/code/cpp/models/implementations/Pure.qll

@@ -3,7 +3,10 @@ import semmle.code.cpp.models.interfaces.Taint
 import semmle.code.cpp.models.interfaces.Alias
 import semmle.code.cpp.models.interfaces.SideEffect
 
-/** Pure string functions. */
+/**
+ * A function that operates on strings and is pure. That is, its evaluation is
+ * guaranteed to be side-effect free.
+ */
 private class PureStrFunction extends AliasFunction, ArrayFunction, TaintFunction,
   SideEffectFunction {
   PureStrFunction() {
@@ -89,7 +92,9 @@ private string strcmp() {
     ]
 }
 
-/** String standard `strlen` function, and related functions for computing string lengths. */
+/**
+ * A function such as `strlen` that returns the length of the given string.
+ */
 private class StrLenFunction extends AliasFunction, ArrayFunction, SideEffectFunction {
   StrLenFunction() {
     hasGlobalOrStdOrBslName(["strlen", "strnlen", "wcslen"])
@@ -123,7 +128,10 @@ private class StrLenFunction extends AliasFunction, ArrayFunction, SideEffectFun
   }
 }
 
-/** Pure functions. */
+/**
+ * A function that is pure, that is, its evaluation is guaranteed to be
+ * side-effect free. Excludes functions modeled by `PureStrFunction` and `PureMemFunction`.
+ */
 private class PureFunction extends TaintFunction, SideEffectFunction {
   PureFunction() { hasGlobalOrStdOrBslName(["abs", "labs"]) }
 
@@ -140,7 +148,10 @@ private class PureFunction extends TaintFunction, SideEffectFunction {
   override predicate hasOnlySpecificWriteSideEffects() { any() }
 }
 
-/** Pure raw-memory functions. */
+/**
+ * A function that operates on memory buffers and is pure. That is, its
+ * evaluation is guaranteed to be side-effect free.
+ */
 private class PureMemFunction extends AliasFunction, ArrayFunction, TaintFunction,
   SideEffectFunction {
   PureMemFunction() {

cpp/ql/src/semmle/code/cpp/models/implementations/Select.qll

@@ -0,0 +1,40 @@
+/**
+ * Provides implementation classes modeling `select` and various similar
+ * functions. See `semmle.code.cpp.models.Models` for usage information.
+ */
+
+import semmle.code.cpp.Function
+import semmle.code.cpp.models.interfaces.ArrayFunction
+import semmle.code.cpp.models.interfaces.Alias
+import semmle.code.cpp.models.interfaces.SideEffect
+
+/**
+ * The function `select` and its assorted variants
+ */
+private class Select extends ArrayFunction, AliasFunction, SideEffectFunction {
+  Select() { this.hasGlobalName(["select", "pselect"]) }
+
+  override predicate hasArrayWithUnknownSize(int bufParam) { bufParam = [1 .. 3] }
+
+  override predicate hasArrayInput(int bufParam) { bufParam = [1 .. 3] }
+
+  override predicate hasArrayOutput(int bufParam) { bufParam = [1 .. 3] }
+
+  override predicate parameterNeverEscapes(int index) { exists(this.getParameter(index)) }
+
+  override predicate parameterEscapesOnlyViaReturn(int index) { none() }
+
+  override predicate parameterIsAlwaysReturned(int index) { none() }
+
+  override predicate hasSpecificWriteSideEffect(ParameterIndex i, boolean buffer, boolean mustWrite) {
+    i = [1 .. 3] and buffer = true and mustWrite = false
+  }
+
+  override predicate hasSpecificReadSideEffect(ParameterIndex i, boolean buffer) {
+    i = [1 .. 5] and buffer = true
+  }
+
+  override predicate hasOnlySpecificReadSideEffects() { any() }
+
+  override predicate hasOnlySpecificWriteSideEffects() { any() }
+}

cpp/ql/test/experimental/query-tests/Security/CWE/CWE-14/semmle/tests/CompilerRemovalOfCodeToClearBuffers.expected

@@ -1,3 +0,0 @@
-| test.c:13:9:13:13 | buff1 | This variable will not be cleared. |
-| test.c:35:9:35:13 | buff1 | This variable will not be cleared. |
-| test.c:43:9:43:13 | buff1 | This variable will not be cleared. |

cpp/ql/test/experimental/query-tests/Security/CWE/CWE-14/semmle/tests/CompilerRemovalOfCodeToClearBuffers.qlref

@@ -1 +0,0 @@
-experimental/Security/CWE/CWE-14/CompilerRemovalOfCodeToClearBuffers.ql

cpp/ql/test/experimental/query-tests/Security/CWE/CWE-14/semmle/tests/test.c

@@ -1,201 +0,0 @@
-struct buffers
-{
-    unsigned char buff1[50];
-    unsigned char *buff2;
-} globalBuff1,*globalBuff2;
-
-unsigned char * globalBuff;
-void badFunc0_0(){
-	unsigned char buff1[12];
-	int i;
-	for(i=0;i<12;i++)
-		buff1[i]=13;
-	memset(buff1,12,12);
-}
-void nobadFunc0_0(){
-	unsigned char buff1[12];
-	memset(buff1,12,12);
-}
-void nobadFunc0_1(){
-	unsigned char buff1[12];
-	int i;
-	memset(buff1,12,12);
-	for(i=0;i<12;i++)
-		buff1[i]=13;
-	free(buff1);
-}
-void nobadFunc1_0(){
-	unsigned char * buff1;
-	buff1 = (unsigned char *) malloc(12);
-	memset(buff1,12,12);
-}
-void badFunc1_0(){
-	unsigned char * buff1;
-	buff1 = (unsigned char *) malloc(12);
-	memset(buff1,12,12);
-	free(buff1);
-}
-void badFunc1_1(){
-	unsigned char buff1[12];
-	int i;
-	for(i=0;i<12;i++)
-		buff1[i]=13;	
-	memset(buff1,12,12);
-	free(buff1);
-}
-void nobadFunc2_0_0(){
-	unsigned char buff1[12];
-	int i;
-	for(i=0;i<12;i++)
-		buff1[i]=13;
-	memset(buff1,12,12);
-	printf(buff1);
-}
-
-void nobadFunc2_0_1(){
-	unsigned char buff1[12];
-	int i;
-	for(i=0;i<12;i++)
-		buff1[i]=13;
-	memset(buff1,12,12);
-	printf(buff1+3);
-}
-
-void nobadFunc2_0_2(){
-	unsigned char buff1[12];
-	int i;
-	for(i=0;i<12;i++)
-		buff1[i]=13;
-	memset(buff1,12,12);
-	printf(*buff1);
-}
-
-void nobadFunc2_0_3(){
-	unsigned char buff1[12];
-	int i;
-	for(i=0;i<12;i++)
-		buff1[i]=13;
-	memset(buff1,12,12);
-	printf(*(buff1+3));
-}
-unsigned char * nobadFunc2_0_4(){
-	unsigned char buff1[12];
-	int i;
-	for(i=0;i<12;i++)
-		buff1[i]=13;
-	memset(buff1,12,12);
-	return buff1;
-}
-
-unsigned char * nobadFunc2_0_5(){
-	unsigned char buff1[12];
-	int i;
-	for(i=0;i<12;i++)
-		buff1[i]=13;
-	memset(buff1,12,12);
-	return buff1+3;
-}
-unsigned char nobadFunc2_0_6(){
-	unsigned char buff1[12];
-	int i;
-	for(i=0;i<12;i++)
-		buff1[i]=13;
-	memset(buff1,12,12);
-	return *buff1;
-}
-
-unsigned char nobadFunc2_0_7(){
-	unsigned char buff1[12];
-	int i;
-	for(i=0;i<12;i++)
-		buff1[i]=13;
-	memset(buff1,12,12);
-	return *(buff1+3);
-}
-void nobadFunc2_1_0(){
-	unsigned char buff1[12];
-	int i;
-	for(i=0;i<12;i++)
-		buff1[i]=13;
-	memset(buff1,12,12);
-	if(*buff1==0)
-		printf("123123");
-}
-void nobadFunc2_1_1(){
-	unsigned char buff1[12];
-	int i;
-	for(i=0;i<12;i++)
-		buff1[i]=13;
-	memset(buff1,12,12);
-	if(*(buff1+3)==0)
-		printf("123123");
-}
-void nobadFunc2_1_2(){
-	unsigned char buff1[12];
-	int i;
-	for(i=0;i<12;i++)
-		buff1[i]=13;
-	memset(buff1,12,12);
-	buff1[2]=5;
-}
-void nobadFunc3_0(unsigned char * buffAll){
-	unsigned char * buff1 = buffAll;
-	memset(buff1,12,12);
-}
-void nobadFunc3_1(unsigned char * buffAll){
-	unsigned char * buff1 = buffAll+3;
-	memset(buff1,12,12);
-}
-void nobadFunc3_2(struct buffers buffAll){
-	unsigned char * buff1 = buffAll.buff1;
-	memset(buff1,12,12);
-}
-void nobadFunc3_3(struct buffers buffAll){
-	unsigned char * buff1 = buffAll.buff2;
-	memset(buff1,12,12);
-}
-void nobadFunc3_4(struct buffers buffAll){
-	unsigned char * buff1 = buffAll.buff2+3;
-	memset(buff1,12,12);
-}
-void nobadFunc3_5(struct buffers * buffAll){
-	unsigned char * buff1 = buffAll->buff1;
-	memset(buff1,12,12);
-}
-void nobadFunc3_6(struct buffers *buffAll){
-	unsigned char * buff1 = buffAll->buff2;
-	memset(buff1,12,12);
-}
-void nobadFunc4(){
-	unsigned char * buff1 = globalBuff;
-	memset(buff1,12,12);
-}
-void nobadFunc4_0(){
-	unsigned char * buff1 = globalBuff;
-	memset(buff1,12,12);
-}
-void nobadFunc4_1(){
-	unsigned char * buff1 = globalBuff+3;
-	memset(buff1,12,12);
-}
-void nobadFunc4_2(){
-	unsigned char * buff1 = globalBuff1.buff1;
-	memset(buff1,12,12);
-}
-void nobadFunc4_3(){
-	unsigned char * buff1 = globalBuff1.buff2;
-	memset(buff1,12,12);
-}
-void nobadFunc4_4(){
-	unsigned char * buff1 = globalBuff1.buff2+3;
-	memset(buff1,12,12);
-}
-void nobadFunc4_5(){
-	unsigned char * buff1 = globalBuff2->buff1;
-	memset(buff1,12,12);
-}
-void nobadFunc4_6(){
-	unsigned char * buff1 = globalBuff2->buff2;
-	memset(buff1,12,12);
-}
-

cpp/ql/test/library-tests/dataflow/fields/dataflow-consistency.expected

@@ -36,9 +36,6 @@ argHasPostUpdate
 | arrays.cpp:10:8:10:15 | * ... | ArgumentNode is missing PostUpdateNode. |
 | arrays.cpp:16:8:16:13 | access to array | ArgumentNode is missing PostUpdateNode. |
 | arrays.cpp:17:8:17:13 | access to array | ArgumentNode is missing PostUpdateNode. |
-| by_reference.cpp:51:8:51:8 | s | ArgumentNode is missing PostUpdateNode. |
-| by_reference.cpp:57:8:57:8 | s | ArgumentNode is missing PostUpdateNode. |
-| by_reference.cpp:63:8:63:8 | s | ArgumentNode is missing PostUpdateNode. |
 postWithInFlow
 | A.cpp:25:13:25:13 | c [post update] | PostUpdateNode should not be the target of local flow. |
 | A.cpp:27:28:27:28 | c [post update] | PostUpdateNode should not be the target of local flow. |

cpp/ql/test/library-tests/dataflow/fields/partial-definition-diff.expected

@@ -227,14 +227,18 @@
 | by_reference.cpp:20:23:20:27 | value | AST only |
 | by_reference.cpp:24:19:24:22 | this | AST only |
 | by_reference.cpp:24:25:24:29 | value | AST only |
+| by_reference.cpp:40:12:40:15 | this | AST only |
 | by_reference.cpp:50:3:50:3 | s | AST only |
 | by_reference.cpp:50:17:50:26 | call to user_input | AST only |
+| by_reference.cpp:51:8:51:8 | s | AST only |
 | by_reference.cpp:51:10:51:20 | call to getDirectly | AST only |
 | by_reference.cpp:56:3:56:3 | s | AST only |
 | by_reference.cpp:56:19:56:28 | call to user_input | AST only |
+| by_reference.cpp:57:8:57:8 | s | AST only |
 | by_reference.cpp:57:10:57:22 | call to getIndirectly | AST only |
 | by_reference.cpp:62:3:62:3 | s | AST only |
 | by_reference.cpp:62:25:62:34 | call to user_input | AST only |
+| by_reference.cpp:63:8:63:8 | s | AST only |
 | by_reference.cpp:63:10:63:28 | call to getThroughNonMember | AST only |
 | by_reference.cpp:68:17:68:18 | & ... | AST only |
 | by_reference.cpp:68:21:68:30 | call to user_input | AST only |

cpp/ql/test/library-tests/dataflow/fields/partial-definition.expected

@@ -266,14 +266,18 @@
 | by_reference.cpp:20:23:20:27 | value |
 | by_reference.cpp:24:19:24:22 | this |
 | by_reference.cpp:24:25:24:29 | value |
+| by_reference.cpp:40:12:40:15 | this |
 | by_reference.cpp:50:3:50:3 | s |
 | by_reference.cpp:50:17:50:26 | call to user_input |
+| by_reference.cpp:51:8:51:8 | s |
 | by_reference.cpp:51:10:51:20 | call to getDirectly |
 | by_reference.cpp:56:3:56:3 | s |
 | by_reference.cpp:56:19:56:28 | call to user_input |
+| by_reference.cpp:57:8:57:8 | s |
 | by_reference.cpp:57:10:57:22 | call to getIndirectly |
 | by_reference.cpp:62:3:62:3 | s |
 | by_reference.cpp:62:25:62:34 | call to user_input |
+| by_reference.cpp:63:8:63:8 | s |
 | by_reference.cpp:63:10:63:28 | call to getThroughNonMember |
 | by_reference.cpp:68:17:68:18 | & ... |
 | by_reference.cpp:68:21:68:30 | call to user_input |

cpp/ql/test/library-tests/dataflow/taint-tests/bsd.cpp

@@ -0,0 +1,24 @@
+void sink(...);
+int source();
+
+// --- accept ---
+
+struct sockaddr {
+	unsigned char length;
+	int sa_family;
+	char* sa_data;
+};
+
+int accept(int, const sockaddr*, int*);
+
+void sink(sockaddr);
+
+void test_accept() {
+  int s = source();
+  sockaddr addr;
+  int size = sizeof(sockaddr);
+  int a = accept(s, &addr, &size);
+
+  sink(a); // $ ast=17:11 SPURIOUS: ast=18:12 MISSING: ir
+  sink(addr); // $ ast MISSING: ir
+}

cpp/ql/test/library-tests/dataflow/taint-tests/localTaint.expected

@@ -135,6 +135,17 @@
 | arrayassignment.cpp:145:12:145:12 | 5 | arrayassignment.cpp:145:7:145:13 | access to array | TAINT |
 | arrayassignment.cpp:146:7:146:10 | arr3 | arrayassignment.cpp:146:7:146:13 | access to array |  |
 | arrayassignment.cpp:146:12:146:12 | 5 | arrayassignment.cpp:146:7:146:13 | access to array | TAINT |
+| bsd.cpp:17:11:17:16 | call to source | bsd.cpp:20:18:20:18 | s |  |
+| bsd.cpp:18:12:18:15 | addr | bsd.cpp:20:22:20:25 | addr |  |
+| bsd.cpp:18:12:18:15 | addr | bsd.cpp:23:8:23:11 | addr |  |
+| bsd.cpp:19:14:19:29 | sizeof(sockaddr) | bsd.cpp:20:29:20:32 | size |  |
+| bsd.cpp:20:11:20:16 | call to accept | bsd.cpp:22:8:22:8 | a |  |
+| bsd.cpp:20:18:20:18 | s | bsd.cpp:20:11:20:16 | call to accept | TAINT |
+| bsd.cpp:20:21:20:25 | & ... | bsd.cpp:20:11:20:16 | call to accept | TAINT |
+| bsd.cpp:20:22:20:25 | addr | bsd.cpp:20:11:20:16 | call to accept | TAINT |
+| bsd.cpp:20:22:20:25 | addr | bsd.cpp:20:21:20:25 | & ... |  |
+| bsd.cpp:20:28:20:32 | ref arg & ... | bsd.cpp:20:29:20:32 | size [inner post update] |  |
+| bsd.cpp:20:29:20:32 | size | bsd.cpp:20:28:20:32 | & ... |  |
 | constructor_delegation.cpp:8:2:8:8 | this | constructor_delegation.cpp:8:20:8:24 | constructor init of field x [pre-this] |  |
 | constructor_delegation.cpp:8:14:8:15 | _x | constructor_delegation.cpp:8:22:8:23 | _x |  |
 | constructor_delegation.cpp:8:22:8:23 | _x | constructor_delegation.cpp:8:20:8:24 | constructor init of field x | TAINT |
@@ -796,8 +807,23 @@
 | map.cpp:146:40:146:41 | ref arg i1 | map.cpp:150:8:150:9 | i1 |  |
 | map.cpp:148:8:148:8 | call to operator* | map.cpp:148:8:148:10 | call to pair | TAINT |
 | map.cpp:148:9:148:10 | i1 | map.cpp:148:8:148:8 | call to operator* | TAINT |
+| map.cpp:148:9:148:10 | ref arg i1 | map.cpp:146:24:146:25 | i1 |  |
+| map.cpp:148:9:148:10 | ref arg i1 | map.cpp:146:40:146:41 | i1 |  |
+| map.cpp:148:9:148:10 | ref arg i1 | map.cpp:148:9:148:10 | i1 |  |
+| map.cpp:148:9:148:10 | ref arg i1 | map.cpp:149:8:149:9 | i1 |  |
+| map.cpp:148:9:148:10 | ref arg i1 | map.cpp:150:8:150:9 | i1 |  |
 | map.cpp:149:8:149:9 | i1 | map.cpp:149:10:149:10 | call to operator-> | TAINT |
+| map.cpp:149:8:149:9 | ref arg i1 | map.cpp:146:24:146:25 | i1 |  |
+| map.cpp:149:8:149:9 | ref arg i1 | map.cpp:146:40:146:41 | i1 |  |
+| map.cpp:149:8:149:9 | ref arg i1 | map.cpp:148:9:148:10 | i1 |  |
+| map.cpp:149:8:149:9 | ref arg i1 | map.cpp:149:8:149:9 | i1 |  |
+| map.cpp:149:8:149:9 | ref arg i1 | map.cpp:150:8:150:9 | i1 |  |
 | map.cpp:150:8:150:9 | i1 | map.cpp:150:10:150:10 | call to operator-> | TAINT |
+| map.cpp:150:8:150:9 | ref arg i1 | map.cpp:146:24:146:25 | i1 |  |
+| map.cpp:150:8:150:9 | ref arg i1 | map.cpp:146:40:146:41 | i1 |  |
+| map.cpp:150:8:150:9 | ref arg i1 | map.cpp:148:9:148:10 | i1 |  |
+| map.cpp:150:8:150:9 | ref arg i1 | map.cpp:149:8:149:9 | i1 |  |
+| map.cpp:150:8:150:9 | ref arg i1 | map.cpp:150:8:150:9 | i1 |  |
 | map.cpp:152:12:152:13 | m2 | map.cpp:152:15:152:19 | call to begin | TAINT |
 | map.cpp:152:12:152:13 | ref arg m2 | map.cpp:152:30:152:31 | m2 |  |
 | map.cpp:152:12:152:13 | ref arg m2 | map.cpp:182:7:182:8 | m2 |  |
@@ -830,8 +856,23 @@
 | map.cpp:152:40:152:41 | ref arg i2 | map.cpp:156:8:156:9 | i2 |  |
 | map.cpp:154:8:154:8 | call to operator* | map.cpp:154:8:154:10 | call to pair | TAINT |
 | map.cpp:154:9:154:10 | i2 | map.cpp:154:8:154:8 | call to operator* | TAINT |
+| map.cpp:154:9:154:10 | ref arg i2 | map.cpp:152:24:152:25 | i2 |  |
+| map.cpp:154:9:154:10 | ref arg i2 | map.cpp:152:40:152:41 | i2 |  |
+| map.cpp:154:9:154:10 | ref arg i2 | map.cpp:154:9:154:10 | i2 |  |
+| map.cpp:154:9:154:10 | ref arg i2 | map.cpp:155:8:155:9 | i2 |  |
+| map.cpp:154:9:154:10 | ref arg i2 | map.cpp:156:8:156:9 | i2 |  |
 | map.cpp:155:8:155:9 | i2 | map.cpp:155:10:155:10 | call to operator-> | TAINT |
+| map.cpp:155:8:155:9 | ref arg i2 | map.cpp:152:24:152:25 | i2 |  |
+| map.cpp:155:8:155:9 | ref arg i2 | map.cpp:152:40:152:41 | i2 |  |
+| map.cpp:155:8:155:9 | ref arg i2 | map.cpp:154:9:154:10 | i2 |  |
+| map.cpp:155:8:155:9 | ref arg i2 | map.cpp:155:8:155:9 | i2 |  |
+| map.cpp:155:8:155:9 | ref arg i2 | map.cpp:156:8:156:9 | i2 |  |
 | map.cpp:156:8:156:9 | i2 | map.cpp:156:10:156:10 | call to operator-> | TAINT |
+| map.cpp:156:8:156:9 | ref arg i2 | map.cpp:152:24:152:25 | i2 |  |
+| map.cpp:156:8:156:9 | ref arg i2 | map.cpp:152:40:152:41 | i2 |  |
+| map.cpp:156:8:156:9 | ref arg i2 | map.cpp:154:9:154:10 | i2 |  |
+| map.cpp:156:8:156:9 | ref arg i2 | map.cpp:155:8:155:9 | i2 |  |
+| map.cpp:156:8:156:9 | ref arg i2 | map.cpp:156:8:156:9 | i2 |  |
 | map.cpp:158:12:158:13 | m3 | map.cpp:158:15:158:19 | call to begin | TAINT |
 | map.cpp:158:12:158:13 | ref arg m3 | map.cpp:158:30:158:31 | m3 |  |
 | map.cpp:158:12:158:13 | ref arg m3 | map.cpp:252:1:252:1 | m3 |  |
@@ -852,8 +893,23 @@
 | map.cpp:158:40:158:41 | ref arg i3 | map.cpp:162:8:162:9 | i3 |  |
 | map.cpp:160:8:160:8 | call to operator* | map.cpp:160:8:160:10 | call to pair | TAINT |
 | map.cpp:160:9:160:10 | i3 | map.cpp:160:8:160:8 | call to operator* | TAINT |
+| map.cpp:160:9:160:10 | ref arg i3 | map.cpp:158:24:158:25 | i3 |  |
+| map.cpp:160:9:160:10 | ref arg i3 | map.cpp:158:40:158:41 | i3 |  |
+| map.cpp:160:9:160:10 | ref arg i3 | map.cpp:160:9:160:10 | i3 |  |
+| map.cpp:160:9:160:10 | ref arg i3 | map.cpp:161:8:161:9 | i3 |  |
+| map.cpp:160:9:160:10 | ref arg i3 | map.cpp:162:8:162:9 | i3 |  |
 | map.cpp:161:8:161:9 | i3 | map.cpp:161:10:161:10 | call to operator-> | TAINT |
+| map.cpp:161:8:161:9 | ref arg i3 | map.cpp:158:24:158:25 | i3 |  |
+| map.cpp:161:8:161:9 | ref arg i3 | map.cpp:158:40:158:41 | i3 |  |
+| map.cpp:161:8:161:9 | ref arg i3 | map.cpp:160:9:160:10 | i3 |  |
+| map.cpp:161:8:161:9 | ref arg i3 | map.cpp:161:8:161:9 | i3 |  |
+| map.cpp:161:8:161:9 | ref arg i3 | map.cpp:162:8:162:9 | i3 |  |
 | map.cpp:162:8:162:9 | i3 | map.cpp:162:10:162:10 | call to operator-> | TAINT |
+| map.cpp:162:8:162:9 | ref arg i3 | map.cpp:158:24:158:25 | i3 |  |
+| map.cpp:162:8:162:9 | ref arg i3 | map.cpp:158:40:158:41 | i3 |  |
+| map.cpp:162:8:162:9 | ref arg i3 | map.cpp:160:9:160:10 | i3 |  |
+| map.cpp:162:8:162:9 | ref arg i3 | map.cpp:161:8:161:9 | i3 |  |
+| map.cpp:162:8:162:9 | ref arg i3 | map.cpp:162:8:162:9 | i3 |  |
 | map.cpp:166:27:166:29 | call to map | map.cpp:167:7:167:9 | m10 |  |
 | map.cpp:166:27:166:29 | call to map | map.cpp:171:7:171:9 | m10 |  |
 | map.cpp:166:27:166:29 | call to map | map.cpp:252:1:252:1 | m10 |  |
@@ -1460,8 +1516,23 @@
 | map.cpp:298:40:298:41 | ref arg i1 | map.cpp:302:8:302:9 | i1 |  |
 | map.cpp:300:8:300:8 | call to operator* | map.cpp:300:8:300:10 | call to pair | TAINT |
 | map.cpp:300:9:300:10 | i1 | map.cpp:300:8:300:8 | call to operator* | TAINT |
+| map.cpp:300:9:300:10 | ref arg i1 | map.cpp:298:24:298:25 | i1 |  |
+| map.cpp:300:9:300:10 | ref arg i1 | map.cpp:298:40:298:41 | i1 |  |
+| map.cpp:300:9:300:10 | ref arg i1 | map.cpp:300:9:300:10 | i1 |  |
+| map.cpp:300:9:300:10 | ref arg i1 | map.cpp:301:8:301:9 | i1 |  |
+| map.cpp:300:9:300:10 | ref arg i1 | map.cpp:302:8:302:9 | i1 |  |
 | map.cpp:301:8:301:9 | i1 | map.cpp:301:10:301:10 | call to operator-> | TAINT |
+| map.cpp:301:8:301:9 | ref arg i1 | map.cpp:298:24:298:25 | i1 |  |
+| map.cpp:301:8:301:9 | ref arg i1 | map.cpp:298:40:298:41 | i1 |  |
+| map.cpp:301:8:301:9 | ref arg i1 | map.cpp:300:9:300:10 | i1 |  |
+| map.cpp:301:8:301:9 | ref arg i1 | map.cpp:301:8:301:9 | i1 |  |
+| map.cpp:301:8:301:9 | ref arg i1 | map.cpp:302:8:302:9 | i1 |  |
 | map.cpp:302:8:302:9 | i1 | map.cpp:302:10:302:10 | call to operator-> | TAINT |
+| map.cpp:302:8:302:9 | ref arg i1 | map.cpp:298:24:298:25 | i1 |  |
+| map.cpp:302:8:302:9 | ref arg i1 | map.cpp:298:40:298:41 | i1 |  |
+| map.cpp:302:8:302:9 | ref arg i1 | map.cpp:300:9:300:10 | i1 |  |
+| map.cpp:302:8:302:9 | ref arg i1 | map.cpp:301:8:301:9 | i1 |  |
+| map.cpp:302:8:302:9 | ref arg i1 | map.cpp:302:8:302:9 | i1 |  |
 | map.cpp:304:12:304:13 | m2 | map.cpp:304:15:304:19 | call to begin | TAINT |
 | map.cpp:304:12:304:13 | ref arg m2 | map.cpp:304:30:304:31 | m2 |  |
 | map.cpp:304:12:304:13 | ref arg m2 | map.cpp:334:7:334:8 | m2 |  |
@@ -1488,8 +1559,23 @@
 | map.cpp:304:40:304:41 | ref arg i2 | map.cpp:308:8:308:9 | i2 |  |
 | map.cpp:306:8:306:8 | call to operator* | map.cpp:306:8:306:10 | call to pair | TAINT |
 | map.cpp:306:9:306:10 | i2 | map.cpp:306:8:306:8 | call to operator* | TAINT |
+| map.cpp:306:9:306:10 | ref arg i2 | map.cpp:304:24:304:25 | i2 |  |
+| map.cpp:306:9:306:10 | ref arg i2 | map.cpp:304:40:304:41 | i2 |  |
+| map.cpp:306:9:306:10 | ref arg i2 | map.cpp:306:9:306:10 | i2 |  |
+| map.cpp:306:9:306:10 | ref arg i2 | map.cpp:307:8:307:9 | i2 |  |
+| map.cpp:306:9:306:10 | ref arg i2 | map.cpp:308:8:308:9 | i2 |  |
 | map.cpp:307:8:307:9 | i2 | map.cpp:307:10:307:10 | call to operator-> | TAINT |
+| map.cpp:307:8:307:9 | ref arg i2 | map.cpp:304:24:304:25 | i2 |  |
+| map.cpp:307:8:307:9 | ref arg i2 | map.cpp:304:40:304:41 | i2 |  |
+| map.cpp:307:8:307:9 | ref arg i2 | map.cpp:306:9:306:10 | i2 |  |
+| map.cpp:307:8:307:9 | ref arg i2 | map.cpp:307:8:307:9 | i2 |  |
+| map.cpp:307:8:307:9 | ref arg i2 | map.cpp:308:8:308:9 | i2 |  |
 | map.cpp:308:8:308:9 | i2 | map.cpp:308:10:308:10 | call to operator-> | TAINT |
+| map.cpp:308:8:308:9 | ref arg i2 | map.cpp:304:24:304:25 | i2 |  |
+| map.cpp:308:8:308:9 | ref arg i2 | map.cpp:304:40:304:41 | i2 |  |
+| map.cpp:308:8:308:9 | ref arg i2 | map.cpp:306:9:306:10 | i2 |  |
+| map.cpp:308:8:308:9 | ref arg i2 | map.cpp:307:8:307:9 | i2 |  |
+| map.cpp:308:8:308:9 | ref arg i2 | map.cpp:308:8:308:9 | i2 |  |
 | map.cpp:310:12:310:13 | m3 | map.cpp:310:15:310:19 | call to begin | TAINT |
 | map.cpp:310:12:310:13 | ref arg m3 | map.cpp:310:30:310:31 | m3 |  |
 | map.cpp:310:12:310:13 | ref arg m3 | map.cpp:438:1:438:1 | m3 |  |
@@ -1510,8 +1596,23 @@
 | map.cpp:310:40:310:41 | ref arg i3 | map.cpp:314:8:314:9 | i3 |  |
 | map.cpp:312:8:312:8 | call to operator* | map.cpp:312:8:312:10 | call to pair | TAINT |
 | map.cpp:312:9:312:10 | i3 | map.cpp:312:8:312:8 | call to operator* | TAINT |
+| map.cpp:312:9:312:10 | ref arg i3 | map.cpp:310:24:310:25 | i3 |  |
+| map.cpp:312:9:312:10 | ref arg i3 | map.cpp:310:40:310:41 | i3 |  |
+| map.cpp:312:9:312:10 | ref arg i3 | map.cpp:312:9:312:10 | i3 |  |
+| map.cpp:312:9:312:10 | ref arg i3 | map.cpp:313:8:313:9 | i3 |  |
+| map.cpp:312:9:312:10 | ref arg i3 | map.cpp:314:8:314:9 | i3 |  |
 | map.cpp:313:8:313:9 | i3 | map.cpp:313:10:313:10 | call to operator-> | TAINT |
+| map.cpp:313:8:313:9 | ref arg i3 | map.cpp:310:24:310:25 | i3 |  |
+| map.cpp:313:8:313:9 | ref arg i3 | map.cpp:310:40:310:41 | i3 |  |
+| map.cpp:313:8:313:9 | ref arg i3 | map.cpp:312:9:312:10 | i3 |  |
+| map.cpp:313:8:313:9 | ref arg i3 | map.cpp:313:8:313:9 | i3 |  |
+| map.cpp:313:8:313:9 | ref arg i3 | map.cpp:314:8:314:9 | i3 |  |
 | map.cpp:314:8:314:9 | i3 | map.cpp:314:10:314:10 | call to operator-> | TAINT |
+| map.cpp:314:8:314:9 | ref arg i3 | map.cpp:310:24:310:25 | i3 |  |
+| map.cpp:314:8:314:9 | ref arg i3 | map.cpp:310:40:310:41 | i3 |  |
+| map.cpp:314:8:314:9 | ref arg i3 | map.cpp:312:9:312:10 | i3 |  |
+| map.cpp:314:8:314:9 | ref arg i3 | map.cpp:313:8:313:9 | i3 |  |
+| map.cpp:314:8:314:9 | ref arg i3 | map.cpp:314:8:314:9 | i3 |  |
 | map.cpp:318:37:318:39 | call to unordered_map | map.cpp:319:7:319:9 | m10 |  |
 | map.cpp:318:37:318:39 | call to unordered_map | map.cpp:323:7:323:9 | m10 |  |
 | map.cpp:318:37:318:39 | call to unordered_map | map.cpp:438:1:438:1 | m10 |  |
@@ -2347,6 +2448,9 @@
 | set.cpp:55:40:55:41 | ref arg i1 | set.cpp:55:40:55:41 | i1 |  |
 | set.cpp:55:40:55:41 | ref arg i1 | set.cpp:57:9:57:10 | i1 |  |
 | set.cpp:57:9:57:10 | i1 | set.cpp:57:8:57:8 | call to operator* | TAINT |
+| set.cpp:57:9:57:10 | ref arg i1 | set.cpp:55:24:55:25 | i1 |  |
+| set.cpp:57:9:57:10 | ref arg i1 | set.cpp:55:40:55:41 | i1 |  |
+| set.cpp:57:9:57:10 | ref arg i1 | set.cpp:57:9:57:10 | i1 |  |
 | set.cpp:59:12:59:13 | ref arg s2 | set.cpp:59:30:59:31 | s2 |  |
 | set.cpp:59:12:59:13 | ref arg s2 | set.cpp:126:1:126:1 | s2 |  |
 | set.cpp:59:12:59:13 | s2 | set.cpp:59:15:59:19 | call to begin | TAINT |
@@ -2362,6 +2466,9 @@
 | set.cpp:59:40:59:41 | ref arg i2 | set.cpp:59:40:59:41 | i2 |  |
 | set.cpp:59:40:59:41 | ref arg i2 | set.cpp:61:9:61:10 | i2 |  |
 | set.cpp:61:9:61:10 | i2 | set.cpp:61:8:61:8 | call to operator* | TAINT |
+| set.cpp:61:9:61:10 | ref arg i2 | set.cpp:59:24:59:25 | i2 |  |
+| set.cpp:61:9:61:10 | ref arg i2 | set.cpp:59:40:59:41 | i2 |  |
+| set.cpp:61:9:61:10 | ref arg i2 | set.cpp:61:9:61:10 | i2 |  |
 | set.cpp:65:19:65:21 | call to set | set.cpp:66:2:66:4 | s11 |  |
 | set.cpp:65:19:65:21 | call to set | set.cpp:67:2:67:4 | s11 |  |
 | set.cpp:65:19:65:21 | call to set | set.cpp:68:2:68:4 | s11 |  |
@@ -2845,6 +2952,9 @@
 | set.cpp:169:40:169:41 | ref arg i1 | set.cpp:169:40:169:41 | i1 |  |
 | set.cpp:169:40:169:41 | ref arg i1 | set.cpp:171:9:171:10 | i1 |  |
 | set.cpp:171:9:171:10 | i1 | set.cpp:171:8:171:8 | call to operator* | TAINT |
+| set.cpp:171:9:171:10 | ref arg i1 | set.cpp:169:24:169:25 | i1 |  |
+| set.cpp:171:9:171:10 | ref arg i1 | set.cpp:169:40:169:41 | i1 |  |
+| set.cpp:171:9:171:10 | ref arg i1 | set.cpp:171:9:171:10 | i1 |  |
 | set.cpp:173:12:173:13 | ref arg s2 | set.cpp:173:30:173:31 | s2 |  |
 | set.cpp:173:12:173:13 | ref arg s2 | set.cpp:238:1:238:1 | s2 |  |
 | set.cpp:173:12:173:13 | s2 | set.cpp:173:15:173:19 | call to begin | TAINT |
@@ -2860,6 +2970,9 @@
 | set.cpp:173:40:173:41 | ref arg i2 | set.cpp:173:40:173:41 | i2 |  |
 | set.cpp:173:40:173:41 | ref arg i2 | set.cpp:175:9:175:10 | i2 |  |
 | set.cpp:175:9:175:10 | i2 | set.cpp:175:8:175:8 | call to operator* | TAINT |
+| set.cpp:175:9:175:10 | ref arg i2 | set.cpp:173:24:173:25 | i2 |  |
+| set.cpp:175:9:175:10 | ref arg i2 | set.cpp:173:40:173:41 | i2 |  |
+| set.cpp:175:9:175:10 | ref arg i2 | set.cpp:175:9:175:10 | i2 |  |
 | set.cpp:179:29:179:31 | call to unordered_set | set.cpp:180:2:180:4 | s11 |  |
 | set.cpp:179:29:179:31 | call to unordered_set | set.cpp:181:2:181:4 | s11 |  |
 | set.cpp:179:29:179:31 | call to unordered_set | set.cpp:182:2:182:4 | s11 |  |
@@ -3111,21 +3224,27 @@
 | smart_pointer.cpp:11:30:11:50 | call to make_shared | smart_pointer.cpp:13:10:13:10 | p |  |
 | smart_pointer.cpp:11:52:11:57 | call to source | smart_pointer.cpp:11:30:11:50 | call to make_shared | TAINT |
 | smart_pointer.cpp:12:11:12:11 | p | smart_pointer.cpp:12:10:12:10 | call to operator* | TAINT |
+| smart_pointer.cpp:12:11:12:11 | ref arg p | smart_pointer.cpp:13:10:13:10 | p |  |
 | smart_pointer.cpp:17:32:17:54 | call to make_shared | smart_pointer.cpp:18:11:18:11 | p |  |
 | smart_pointer.cpp:17:32:17:54 | call to make_shared | smart_pointer.cpp:19:10:19:10 | p |  |
 | smart_pointer.cpp:18:11:18:11 | p | smart_pointer.cpp:18:10:18:10 | call to operator* | TAINT |
+| smart_pointer.cpp:18:11:18:11 | ref arg p | smart_pointer.cpp:19:10:19:10 | p |  |
 | smart_pointer.cpp:23:30:23:50 | call to make_unique | smart_pointer.cpp:24:11:24:11 | p |  |
 | smart_pointer.cpp:23:30:23:50 | call to make_unique | smart_pointer.cpp:25:10:25:10 | p |  |
 | smart_pointer.cpp:23:52:23:57 | call to source | smart_pointer.cpp:23:30:23:50 | call to make_unique | TAINT |
 | smart_pointer.cpp:24:11:24:11 | p | smart_pointer.cpp:24:10:24:10 | call to operator* | TAINT |
+| smart_pointer.cpp:24:11:24:11 | ref arg p | smart_pointer.cpp:25:10:25:10 | p |  |
 | smart_pointer.cpp:29:32:29:54 | call to make_unique | smart_pointer.cpp:30:11:30:11 | p |  |
 | smart_pointer.cpp:29:32:29:54 | call to make_unique | smart_pointer.cpp:31:10:31:10 | p |  |
 | smart_pointer.cpp:30:11:30:11 | p | smart_pointer.cpp:30:10:30:10 | call to operator* | TAINT |
+| smart_pointer.cpp:30:11:30:11 | ref arg p | smart_pointer.cpp:31:10:31:10 | p |  |
 | smart_pointer.cpp:35:30:35:50 | call to make_shared | smart_pointer.cpp:37:6:37:6 | p |  |
 | smart_pointer.cpp:35:30:35:50 | call to make_shared | smart_pointer.cpp:38:10:38:10 | p |  |
 | smart_pointer.cpp:35:30:35:50 | call to make_shared | smart_pointer.cpp:39:11:39:11 | p |  |
 | smart_pointer.cpp:37:5:37:17 | ... = ... | smart_pointer.cpp:37:5:37:5 | call to operator* [post update] |  |
 | smart_pointer.cpp:37:6:37:6 | p | smart_pointer.cpp:37:5:37:5 | call to operator* | TAINT |
+| smart_pointer.cpp:37:6:37:6 | ref arg p | smart_pointer.cpp:38:10:38:10 | p |  |
+| smart_pointer.cpp:37:6:37:6 | ref arg p | smart_pointer.cpp:39:11:39:11 | p |  |
 | smart_pointer.cpp:37:10:37:15 | call to source | smart_pointer.cpp:37:5:37:17 | ... = ... |  |
 | smart_pointer.cpp:38:10:38:10 | ref arg p | smart_pointer.cpp:39:11:39:11 | p |  |
 | smart_pointer.cpp:39:11:39:11 | p | smart_pointer.cpp:39:10:39:10 | call to operator* | TAINT |
@@ -3134,6 +3253,8 @@
 | smart_pointer.cpp:43:29:43:51 | call to unique_ptr | smart_pointer.cpp:47:11:47:11 | p |  |
 | smart_pointer.cpp:45:5:45:17 | ... = ... | smart_pointer.cpp:45:5:45:5 | call to operator* [post update] |  |
 | smart_pointer.cpp:45:6:45:6 | p | smart_pointer.cpp:45:5:45:5 | call to operator* | TAINT |
+| smart_pointer.cpp:45:6:45:6 | ref arg p | smart_pointer.cpp:46:10:46:10 | p |  |
+| smart_pointer.cpp:45:6:45:6 | ref arg p | smart_pointer.cpp:47:11:47:11 | p |  |
 | smart_pointer.cpp:45:10:45:15 | call to source | smart_pointer.cpp:45:5:45:17 | ... = ... |  |
 | smart_pointer.cpp:46:10:46:10 | ref arg p | smart_pointer.cpp:47:11:47:11 | p |  |
 | smart_pointer.cpp:47:11:47:11 | p | smart_pointer.cpp:47:10:47:10 | call to operator* | TAINT |
@@ -3147,6 +3268,7 @@
 | smart_pointer.cpp:65:28:65:46 | call to make_unique | smart_pointer.cpp:67:10:67:10 | p |  |
 | smart_pointer.cpp:65:48:65:53 | call to source | smart_pointer.cpp:65:28:65:46 | call to make_unique | TAINT |
 | smart_pointer.cpp:65:58:65:58 | 0 | smart_pointer.cpp:65:28:65:46 | call to make_unique | TAINT |
+| smart_pointer.cpp:66:10:66:10 | ref arg p | smart_pointer.cpp:67:10:67:10 | p |  |
 | standalone_iterators.cpp:39:45:39:51 | source1 | standalone_iterators.cpp:39:45:39:51 | source1 |  |
 | standalone_iterators.cpp:39:45:39:51 | source1 | standalone_iterators.cpp:40:11:40:17 | source1 |  |
 | standalone_iterators.cpp:39:45:39:51 | source1 | standalone_iterators.cpp:41:12:41:18 | source1 |  |
@@ -3458,6 +3580,9 @@
 | string.cpp:121:15:121:15 | ref arg (__begin) | string.cpp:121:15:121:15 | (__begin) |  |
 | string.cpp:121:15:121:15 | ref arg (__begin) | string.cpp:121:15:121:15 | (__begin) |  |
 | string.cpp:121:15:121:15 | ref arg (__begin) | string.cpp:121:15:121:15 | (__begin) |  |
+| string.cpp:121:15:121:15 | ref arg (__begin) | string.cpp:121:15:121:15 | (__begin) |  |
+| string.cpp:121:15:121:15 | ref arg (__begin) | string.cpp:121:15:121:15 | (__begin) |  |
+| string.cpp:121:15:121:15 | ref arg (__begin) | string.cpp:121:15:121:15 | (__begin) |  |
 | string.cpp:121:15:121:15 | ref arg (__range) | string.cpp:121:15:121:15 | (__range) |  |
 | string.cpp:121:15:121:15 | s | string.cpp:121:15:121:15 | (__range) |  |
 | string.cpp:121:15:121:15 | s | string.cpp:121:15:121:15 | (__range) |  |
@@ -3476,6 +3601,9 @@
 | string.cpp:125:61:125:62 | ref arg it | string.cpp:125:61:125:62 | it |  |
 | string.cpp:125:61:125:62 | ref arg it | string.cpp:126:9:126:10 | it |  |
 | string.cpp:126:9:126:10 | it | string.cpp:126:8:126:8 | call to operator* | TAINT |
+| string.cpp:126:9:126:10 | ref arg it | string.cpp:125:44:125:45 | it |  |
+| string.cpp:126:9:126:10 | ref arg it | string.cpp:125:61:125:62 | it |  |
+| string.cpp:126:9:126:10 | ref arg it | string.cpp:126:9:126:10 | it |  |
 | string.cpp:129:16:129:16 | (__begin) | string.cpp:129:16:129:16 | call to operator* | TAINT |
 | string.cpp:129:16:129:16 | (__begin) | string.cpp:129:16:129:16 | call to operator++ |  |
 | string.cpp:129:16:129:16 | (__end) | string.cpp:129:16:129:16 | call to iterator |  |
@@ -3489,6 +3617,9 @@
 | string.cpp:129:16:129:16 | ref arg (__begin) | string.cpp:129:16:129:16 | (__begin) |  |
 | string.cpp:129:16:129:16 | ref arg (__begin) | string.cpp:129:16:129:16 | (__begin) |  |
 | string.cpp:129:16:129:16 | ref arg (__begin) | string.cpp:129:16:129:16 | (__begin) |  |
+| string.cpp:129:16:129:16 | ref arg (__begin) | string.cpp:129:16:129:16 | (__begin) |  |
+| string.cpp:129:16:129:16 | ref arg (__begin) | string.cpp:129:16:129:16 | (__begin) |  |
+| string.cpp:129:16:129:16 | ref arg (__begin) | string.cpp:129:16:129:16 | (__begin) |  |
 | string.cpp:129:16:129:16 | ref arg (__range) | string.cpp:129:16:129:16 | (__range) |  |
 | string.cpp:129:16:129:16 | s | string.cpp:129:16:129:16 | (__range) |  |
 | string.cpp:129:16:129:16 | s | string.cpp:129:16:129:16 | (__range) |  |
@@ -3857,11 +3988,13 @@
 | string.cpp:376:31:376:35 | call to begin | string.cpp:378:9:378:13 | iter1 |  |
 | string.cpp:376:31:376:35 | call to begin | string.cpp:379:8:379:12 | iter1 |  |
 | string.cpp:378:9:378:13 | iter1 | string.cpp:378:8:378:8 | call to operator* | TAINT |
+| string.cpp:378:9:378:13 | ref arg iter1 | string.cpp:379:8:379:12 | iter1 |  |
 | string.cpp:379:8:379:12 | iter1 | string.cpp:379:13:379:13 | call to operator[] | TAINT |
 | string.cpp:380:28:380:29 | s2 | string.cpp:380:31:380:35 | call to begin | TAINT |
 | string.cpp:380:31:380:35 | call to begin | string.cpp:382:9:382:13 | iter2 |  |
 | string.cpp:380:31:380:35 | call to begin | string.cpp:383:8:383:12 | iter2 |  |
 | string.cpp:382:9:382:13 | iter2 | string.cpp:382:8:382:8 | call to operator* | TAINT |
+| string.cpp:382:9:382:13 | ref arg iter2 | string.cpp:383:8:383:12 | iter2 |  |
 | string.cpp:383:8:383:12 | iter2 | string.cpp:383:13:383:13 | call to operator[] | TAINT |
 | string.cpp:388:18:388:24 | hello | string.cpp:388:18:388:25 | call to basic_string | TAINT |
 | string.cpp:388:18:388:25 | call to basic_string | string.cpp:391:25:391:26 | s1 |  |
@@ -3905,10 +4038,12 @@
 | string.cpp:398:8:398:9 | i2 | string.cpp:398:3:398:9 | ... = ... |  |
 | string.cpp:398:8:398:9 | i2 | string.cpp:399:12:399:13 | i3 |  |
 | string.cpp:399:10:399:10 | call to operator++ | string.cpp:399:8:399:8 | call to operator* | TAINT |
+| string.cpp:399:10:399:10 | ref arg call to operator++ | string.cpp:399:12:399:13 | ref arg i3 |  |
 | string.cpp:399:12:399:13 | i3 | string.cpp:399:10:399:10 | call to operator++ |  |
 | string.cpp:400:8:400:9 | i2 | string.cpp:400:3:400:9 | ... = ... |  |
 | string.cpp:400:8:400:9 | i2 | string.cpp:401:12:401:13 | i4 |  |
 | string.cpp:401:10:401:10 | call to operator-- | string.cpp:401:8:401:8 | call to operator* | TAINT |
+| string.cpp:401:10:401:10 | ref arg call to operator-- | string.cpp:401:12:401:13 | ref arg i4 |  |
 | string.cpp:401:12:401:13 | i4 | string.cpp:401:10:401:10 | call to operator-- |  |
 | string.cpp:402:8:402:9 | i2 | string.cpp:402:3:402:9 | ... = ... |  |
 | string.cpp:402:8:402:9 | i2 | string.cpp:403:3:403:4 | i5 |  |
@@ -3926,11 +4061,13 @@
 | string.cpp:408:8:408:9 | i2 | string.cpp:409:10:409:11 | i7 |  |
 | string.cpp:409:10:409:11 | i7 | string.cpp:409:12:409:12 | call to operator+= |  |
 | string.cpp:409:12:409:12 | call to operator+= | string.cpp:409:8:409:8 | call to operator* | TAINT |
+| string.cpp:409:12:409:12 | ref arg call to operator+= | string.cpp:409:10:409:11 | ref arg i7 | TAINT |
 | string.cpp:409:14:409:14 | 1 | string.cpp:409:10:409:11 | ref arg i7 | TAINT |
 | string.cpp:410:8:410:9 | i2 | string.cpp:410:3:410:9 | ... = ... |  |
 | string.cpp:410:8:410:9 | i2 | string.cpp:411:10:411:11 | i8 |  |
 | string.cpp:411:10:411:11 | i8 | string.cpp:411:12:411:12 | call to operator-= |  |
 | string.cpp:411:12:411:12 | call to operator-= | string.cpp:411:8:411:8 | call to operator* | TAINT |
+| string.cpp:411:12:411:12 | ref arg call to operator-= | string.cpp:411:10:411:11 | ref arg i8 | TAINT |
 | string.cpp:411:14:411:14 | 1 | string.cpp:411:10:411:11 | ref arg i8 | TAINT |
 | string.cpp:413:8:413:9 | s2 | string.cpp:413:11:413:13 | call to end | TAINT |
 | string.cpp:413:11:413:13 | call to end | string.cpp:413:3:413:15 | ... = ... |  |
@@ -6104,6 +6241,7 @@
 | taint.cpp:655:35:655:40 | source | taint.cpp:657:20:657:25 | source |  |
 | taint.cpp:656:27:656:27 | c | taint.cpp:657:10:657:10 | c |  |
 | taint.cpp:656:27:656:27 | c | taint.cpp:658:8:658:8 | c |  |
+| taint.cpp:657:10:657:10 | ref arg c | taint.cpp:658:8:658:8 | c |  |
 | taint.cpp:657:12:657:15 | call to data | taint.cpp:657:3:657:8 | call to memcpy |  |
 | taint.cpp:657:20:657:25 | source | taint.cpp:657:3:657:8 | call to memcpy | TAINT |
 | taint.cpp:657:20:657:25 | source | taint.cpp:657:12:657:15 | ref arg call to data | TAINT |
@@ -6128,6 +6266,9 @@
 | vector.cpp:19:14:19:14 | ref arg (__begin) | vector.cpp:19:14:19:14 | (__begin) |  |
 | vector.cpp:19:14:19:14 | ref arg (__begin) | vector.cpp:19:14:19:14 | (__begin) |  |
 | vector.cpp:19:14:19:14 | ref arg (__begin) | vector.cpp:19:14:19:14 | (__begin) |  |
+| vector.cpp:19:14:19:14 | ref arg (__begin) | vector.cpp:19:14:19:14 | (__begin) |  |
+| vector.cpp:19:14:19:14 | ref arg (__begin) | vector.cpp:19:14:19:14 | (__begin) |  |
+| vector.cpp:19:14:19:14 | ref arg (__begin) | vector.cpp:19:14:19:14 | (__begin) |  |
 | vector.cpp:19:14:19:14 | ref arg (__range) | vector.cpp:19:14:19:14 | (__range) |  |
 | vector.cpp:19:14:19:14 | v | vector.cpp:19:14:19:14 | (__range) |  |
 | vector.cpp:19:14:19:14 | v | vector.cpp:19:14:19:14 | (__range) |  |
@@ -6148,6 +6289,9 @@
 | vector.cpp:23:66:23:67 | ref arg it | vector.cpp:23:66:23:67 | it |  |
 | vector.cpp:23:66:23:67 | ref arg it | vector.cpp:24:9:24:10 | it |  |
 | vector.cpp:24:9:24:10 | it | vector.cpp:24:8:24:8 | call to operator* | TAINT |
+| vector.cpp:24:9:24:10 | ref arg it | vector.cpp:23:49:23:50 | it |  |
+| vector.cpp:24:9:24:10 | ref arg it | vector.cpp:23:66:23:67 | it |  |
+| vector.cpp:24:9:24:10 | ref arg it | vector.cpp:24:9:24:10 | it |  |
 | vector.cpp:27:15:27:15 | (__begin) | vector.cpp:27:15:27:15 | call to operator* | TAINT |
 | vector.cpp:27:15:27:15 | (__begin) | vector.cpp:27:15:27:15 | call to operator++ |  |
 | vector.cpp:27:15:27:15 | (__end) | vector.cpp:27:15:27:15 | call to iterator |  |
@@ -6161,6 +6305,9 @@
 | vector.cpp:27:15:27:15 | ref arg (__begin) | vector.cpp:27:15:27:15 | (__begin) |  |
 | vector.cpp:27:15:27:15 | ref arg (__begin) | vector.cpp:27:15:27:15 | (__begin) |  |
 | vector.cpp:27:15:27:15 | ref arg (__begin) | vector.cpp:27:15:27:15 | (__begin) |  |
+| vector.cpp:27:15:27:15 | ref arg (__begin) | vector.cpp:27:15:27:15 | (__begin) |  |
+| vector.cpp:27:15:27:15 | ref arg (__begin) | vector.cpp:27:15:27:15 | (__begin) |  |
+| vector.cpp:27:15:27:15 | ref arg (__begin) | vector.cpp:27:15:27:15 | (__begin) |  |
 | vector.cpp:27:15:27:15 | ref arg (__range) | vector.cpp:27:15:27:15 | (__range) |  |
 | vector.cpp:27:15:27:15 | v | vector.cpp:27:15:27:15 | (__range) |  |
 | vector.cpp:27:15:27:15 | v | vector.cpp:27:15:27:15 | (__range) |  |
@@ -7084,16 +7231,20 @@
 | vector.cpp:329:62:329:65 | iter | vector.cpp:329:62:329:65 | iter |  |
 | vector.cpp:329:62:329:65 | iter | vector.cpp:330:3:330:6 | iter |  |
 | vector.cpp:330:2:330:2 | call to operator* [post update] | vector.cpp:329:62:329:65 | iter |  |
+| vector.cpp:330:2:330:2 | call to operator* [post update] | vector.cpp:330:3:330:6 | ref arg iter | TAINT |
 | vector.cpp:330:2:330:17 | ... = ... | vector.cpp:330:2:330:2 | call to operator* [post update] |  |
 | vector.cpp:330:3:330:6 | iter | vector.cpp:330:2:330:2 | call to operator* | TAINT |
+| vector.cpp:330:3:330:6 | ref arg iter | vector.cpp:329:62:329:65 | iter |  |
 | vector.cpp:330:10:330:15 | call to source | vector.cpp:330:2:330:2 | call to operator* [post update] | TAINT |
 | vector.cpp:330:10:330:15 | call to source | vector.cpp:330:2:330:17 | ... = ... |  |
 | vector.cpp:333:64:333:67 | iter | vector.cpp:333:64:333:67 | iter |  |
 | vector.cpp:333:64:333:67 | iter | vector.cpp:334:3:334:6 | iter |  |
 | vector.cpp:333:74:333:74 | i | vector.cpp:334:10:334:10 | i |  |
 | vector.cpp:334:2:334:2 | call to operator* [post update] | vector.cpp:333:64:333:67 | iter |  |
+| vector.cpp:334:2:334:2 | call to operator* [post update] | vector.cpp:334:3:334:6 | ref arg iter | TAINT |
 | vector.cpp:334:2:334:10 | ... = ... | vector.cpp:334:2:334:2 | call to operator* [post update] |  |
 | vector.cpp:334:3:334:6 | iter | vector.cpp:334:2:334:2 | call to operator* | TAINT |
+| vector.cpp:334:3:334:6 | ref arg iter | vector.cpp:333:64:333:67 | iter |  |
 | vector.cpp:334:10:334:10 | i | vector.cpp:334:2:334:2 | call to operator* [post update] | TAINT |
 | vector.cpp:334:10:334:10 | i | vector.cpp:334:2:334:10 | ... = ... |  |
 | vector.cpp:337:38:337:38 | b | vector.cpp:372:5:372:5 | b |  |
@@ -7151,6 +7302,7 @@
 | vector.cpp:340:34:340:35 | ref arg v1 | vector.cpp:415:1:415:1 | v1 |  |
 | vector.cpp:340:34:340:35 | v1 | vector.cpp:340:37:340:41 | call to begin | TAINT |
 | vector.cpp:340:37:340:41 | call to begin | vector.cpp:341:3:341:4 | i1 |  |
+| vector.cpp:341:2:341:2 | call to operator* [post update] | vector.cpp:341:3:341:4 | ref arg i1 | TAINT |
 | vector.cpp:341:2:341:2 | call to operator* [post update] | vector.cpp:342:7:342:8 | v1 |  |
 | vector.cpp:341:2:341:2 | call to operator* [post update] | vector.cpp:415:1:415:1 | v1 |  |
 | vector.cpp:341:2:341:15 | ... = ... | vector.cpp:341:2:341:2 | call to operator* [post update] |  |
@@ -7174,10 +7326,14 @@
 | vector.cpp:344:68:344:69 | ref arg it | vector.cpp:344:68:344:69 | it |  |
 | vector.cpp:344:68:344:69 | ref arg it | vector.cpp:345:4:345:5 | it |  |
 | vector.cpp:345:3:345:3 | call to operator* [post update] | vector.cpp:344:56:344:57 | v2 |  |
+| vector.cpp:345:3:345:3 | call to operator* [post update] | vector.cpp:345:4:345:5 | ref arg it | TAINT |
 | vector.cpp:345:3:345:3 | call to operator* [post update] | vector.cpp:347:7:347:8 | v2 |  |
 | vector.cpp:345:3:345:3 | call to operator* [post update] | vector.cpp:415:1:415:1 | v2 |  |
 | vector.cpp:345:3:345:16 | ... = ... | vector.cpp:345:3:345:3 | call to operator* [post update] |  |
 | vector.cpp:345:4:345:5 | it | vector.cpp:345:3:345:3 | call to operator* | TAINT |
+| vector.cpp:345:4:345:5 | ref arg it | vector.cpp:344:50:344:51 | it |  |
+| vector.cpp:345:4:345:5 | ref arg it | vector.cpp:344:68:344:69 | it |  |
+| vector.cpp:345:4:345:5 | ref arg it | vector.cpp:345:4:345:5 | it |  |
 | vector.cpp:345:9:345:14 | call to source | vector.cpp:345:3:345:3 | call to operator* [post update] | TAINT |
 | vector.cpp:345:9:345:14 | call to source | vector.cpp:345:3:345:16 | ... = ... |  |
 | vector.cpp:347:7:347:8 | ref arg v2 | vector.cpp:415:1:415:1 | v2 |  |
@@ -7193,6 +7349,9 @@
 | vector.cpp:349:15:349:15 | ref arg (__begin) | vector.cpp:349:15:349:15 | (__begin) |  |
 | vector.cpp:349:15:349:15 | ref arg (__begin) | vector.cpp:349:15:349:15 | (__begin) |  |
 | vector.cpp:349:15:349:15 | ref arg (__begin) | vector.cpp:349:15:349:15 | (__begin) |  |
+| vector.cpp:349:15:349:15 | ref arg (__begin) | vector.cpp:349:15:349:15 | (__begin) |  |
+| vector.cpp:349:15:349:15 | ref arg (__begin) | vector.cpp:349:15:349:15 | (__begin) |  |
+| vector.cpp:349:15:349:15 | ref arg (__begin) | vector.cpp:349:15:349:15 | (__begin) |  |
 | vector.cpp:349:15:349:15 | ref arg (__range) | vector.cpp:349:15:349:15 | (__range) |  |
 | vector.cpp:349:15:349:16 | v3 | vector.cpp:349:15:349:15 | (__range) |  |
 | vector.cpp:349:15:349:16 | v3 | vector.cpp:349:15:349:15 | (__range) |  |
@@ -7229,15 +7388,18 @@
 | vector.cpp:359:34:359:35 | v5 | vector.cpp:359:37:359:41 | call to begin | TAINT |
 | vector.cpp:359:37:359:41 | call to begin | vector.cpp:360:3:360:4 | i5 |  |
 | vector.cpp:359:37:359:41 | call to begin | vector.cpp:362:3:362:4 | i5 |  |
+| vector.cpp:360:2:360:2 | call to operator* [post update] | vector.cpp:360:3:360:4 | ref arg i5 | TAINT |
 | vector.cpp:360:2:360:2 | call to operator* [post update] | vector.cpp:361:7:361:8 | v5 |  |
 | vector.cpp:360:2:360:2 | call to operator* [post update] | vector.cpp:363:7:363:8 | v5 |  |
 | vector.cpp:360:2:360:2 | call to operator* [post update] | vector.cpp:415:1:415:1 | v5 |  |
 | vector.cpp:360:2:360:15 | ... = ... | vector.cpp:360:2:360:2 | call to operator* [post update] |  |
 | vector.cpp:360:3:360:4 | i5 | vector.cpp:360:2:360:2 | call to operator* | TAINT |
+| vector.cpp:360:3:360:4 | ref arg i5 | vector.cpp:362:3:362:4 | i5 |  |
 | vector.cpp:360:8:360:13 | call to source | vector.cpp:360:2:360:2 | call to operator* [post update] | TAINT |
 | vector.cpp:360:8:360:13 | call to source | vector.cpp:360:2:360:15 | ... = ... |  |
 | vector.cpp:361:7:361:8 | ref arg v5 | vector.cpp:363:7:363:8 | v5 |  |
 | vector.cpp:361:7:361:8 | ref arg v5 | vector.cpp:415:1:415:1 | v5 |  |
+| vector.cpp:362:2:362:2 | call to operator* [post update] | vector.cpp:362:3:362:4 | ref arg i5 | TAINT |
 | vector.cpp:362:2:362:2 | call to operator* [post update] | vector.cpp:363:7:363:8 | v5 |  |
 | vector.cpp:362:2:362:2 | call to operator* [post update] | vector.cpp:415:1:415:1 | v5 |  |
 | vector.cpp:362:2:362:8 | ... = ... | vector.cpp:362:2:362:2 | call to operator* [post update] |  |
@@ -7251,6 +7413,7 @@
 | vector.cpp:365:34:365:35 | ref arg v6 | vector.cpp:415:1:415:1 | v6 |  |
 | vector.cpp:365:34:365:35 | v6 | vector.cpp:365:37:365:41 | call to begin | TAINT |
 | vector.cpp:365:37:365:41 | call to begin | vector.cpp:366:3:366:4 | i6 |  |
+| vector.cpp:366:2:366:2 | call to operator* [post update] | vector.cpp:366:3:366:4 | ref arg i6 | TAINT |
 | vector.cpp:366:2:366:2 | call to operator* [post update] | vector.cpp:367:7:367:8 | v6 |  |
 | vector.cpp:366:2:366:2 | call to operator* [post update] | vector.cpp:368:2:368:3 | v6 |  |
 | vector.cpp:366:2:366:2 | call to operator* [post update] | vector.cpp:369:7:369:8 | v6 |  |
@@ -7274,6 +7437,7 @@
 | vector.cpp:371:34:371:35 | v7 | vector.cpp:371:37:371:41 | call to begin | TAINT |
 | vector.cpp:371:37:371:41 | call to begin | vector.cpp:373:4:373:5 | i7 |  |
 | vector.cpp:371:37:371:41 | call to begin | vector.cpp:376:4:376:5 | i7 |  |
+| vector.cpp:373:3:373:3 | call to operator* [post update] | vector.cpp:373:4:373:5 | ref arg i7 | TAINT |
 | vector.cpp:373:3:373:3 | call to operator* [post update] | vector.cpp:374:8:374:9 | v7 |  |
 | vector.cpp:373:3:373:3 | call to operator* [post update] | vector.cpp:379:7:379:8 | v7 |  |
 | vector.cpp:373:3:373:3 | call to operator* [post update] | vector.cpp:415:1:415:1 | v7 |  |
@@ -7283,6 +7447,7 @@
 | vector.cpp:373:9:373:14 | call to source | vector.cpp:373:3:373:16 | ... = ... |  |
 | vector.cpp:374:8:374:9 | ref arg v7 | vector.cpp:379:7:379:8 | v7 |  |
 | vector.cpp:374:8:374:9 | ref arg v7 | vector.cpp:415:1:415:1 | v7 |  |
+| vector.cpp:376:3:376:3 | call to operator* [post update] | vector.cpp:376:4:376:5 | ref arg i7 | TAINT |
 | vector.cpp:376:3:376:3 | call to operator* [post update] | vector.cpp:377:8:377:9 | v7 |  |
 | vector.cpp:376:3:376:3 | call to operator* [post update] | vector.cpp:379:7:379:8 | v7 |  |
 | vector.cpp:376:3:376:3 | call to operator* [post update] | vector.cpp:415:1:415:1 | v7 |  |
@@ -7299,15 +7464,18 @@
 | vector.cpp:381:34:381:35 | v8 | vector.cpp:381:37:381:41 | call to begin | TAINT |
 | vector.cpp:381:37:381:41 | call to begin | vector.cpp:382:3:382:4 | i8 |  |
 | vector.cpp:381:37:381:41 | call to begin | vector.cpp:384:3:384:4 | i8 |  |
+| vector.cpp:382:2:382:2 | call to operator* [post update] | vector.cpp:382:3:382:4 | ref arg i8 | TAINT |
 | vector.cpp:382:2:382:2 | call to operator* [post update] | vector.cpp:383:7:383:8 | v8 |  |
 | vector.cpp:382:2:382:2 | call to operator* [post update] | vector.cpp:385:7:385:8 | v8 |  |
 | vector.cpp:382:2:382:2 | call to operator* [post update] | vector.cpp:415:1:415:1 | v8 |  |
 | vector.cpp:382:2:382:15 | ... = ... | vector.cpp:382:2:382:2 | call to operator* [post update] |  |
 | vector.cpp:382:3:382:4 | i8 | vector.cpp:382:2:382:2 | call to operator* | TAINT |
+| vector.cpp:382:3:382:4 | ref arg i8 | vector.cpp:384:3:384:4 | i8 |  |
 | vector.cpp:382:8:382:13 | call to source | vector.cpp:382:2:382:2 | call to operator* [post update] | TAINT |
 | vector.cpp:382:8:382:13 | call to source | vector.cpp:382:2:382:15 | ... = ... |  |
 | vector.cpp:383:7:383:8 | ref arg v8 | vector.cpp:385:7:385:8 | v8 |  |
 | vector.cpp:383:7:383:8 | ref arg v8 | vector.cpp:415:1:415:1 | v8 |  |
+| vector.cpp:384:2:384:2 | call to operator* [post update] | vector.cpp:384:3:384:4 | ref arg i8 | TAINT |
 | vector.cpp:384:2:384:2 | call to operator* [post update] | vector.cpp:385:7:385:8 | v8 |  |
 | vector.cpp:384:2:384:2 | call to operator* [post update] | vector.cpp:415:1:415:1 | v8 |  |
 | vector.cpp:384:2:384:8 | ... = ... | vector.cpp:384:2:384:2 | call to operator* [post update] |  |
@@ -7320,10 +7488,12 @@
 | vector.cpp:387:34:387:35 | v9 | vector.cpp:387:37:387:41 | call to begin | TAINT |
 | vector.cpp:387:37:387:41 | call to begin | vector.cpp:389:3:389:4 | i9 |  |
 | vector.cpp:387:37:387:41 | call to begin | vector.cpp:390:31:390:32 | i9 |  |
+| vector.cpp:389:2:389:2 | call to operator* [post update] | vector.cpp:389:3:389:4 | ref arg i9 | TAINT |
 | vector.cpp:389:2:389:2 | call to operator* [post update] | vector.cpp:392:7:392:8 | v9 |  |
 | vector.cpp:389:2:389:2 | call to operator* [post update] | vector.cpp:415:1:415:1 | v9 |  |
 | vector.cpp:389:2:389:15 | ... = ... | vector.cpp:389:2:389:2 | call to operator* [post update] |  |
 | vector.cpp:389:3:389:4 | i9 | vector.cpp:389:2:389:2 | call to operator* | TAINT |
+| vector.cpp:389:3:389:4 | ref arg i9 | vector.cpp:390:31:390:32 | i9 |  |
 | vector.cpp:389:8:389:13 | call to source | vector.cpp:389:2:389:2 | call to operator* [post update] | TAINT |
 | vector.cpp:389:8:389:13 | call to source | vector.cpp:389:2:389:15 | ... = ... |  |
 | vector.cpp:390:31:390:32 | call to iterator [post update] | vector.cpp:392:7:392:8 | v9 |  |
@@ -7365,6 +7535,7 @@
 | vector.cpp:403:6:403:6 | call to operator++ | vector.cpp:403:2:403:2 | call to operator* | TAINT |
 | vector.cpp:403:11:403:11 | 0 | vector.cpp:403:2:403:2 | call to operator* [post update] | TAINT |
 | vector.cpp:403:11:403:11 | 0 | vector.cpp:403:2:403:11 | ... = ... |  |
+| vector.cpp:404:2:404:2 | call to operator* [post update] | vector.cpp:404:3:404:5 | ref arg i12 | TAINT |
 | vector.cpp:404:2:404:2 | call to operator* [post update] | vector.cpp:405:7:405:9 | v12 |  |
 | vector.cpp:404:2:404:2 | call to operator* [post update] | vector.cpp:415:1:415:1 | v12 |  |
 | vector.cpp:404:2:404:16 | ... = ... | vector.cpp:404:2:404:2 | call to operator* [post update] |  |
@@ -7595,12 +7766,18 @@
 | vector.cpp:526:11:526:15 | call to begin | vector.cpp:530:3:530:4 | it |  |
 | vector.cpp:526:11:526:15 | call to begin | vector.cpp:531:9:531:10 | it |  |
 | vector.cpp:527:9:527:10 | it | vector.cpp:527:8:527:8 | call to operator* | TAINT |
+| vector.cpp:527:9:527:10 | ref arg it | vector.cpp:528:3:528:4 | it |  |
+| vector.cpp:527:9:527:10 | ref arg it | vector.cpp:529:9:529:10 | it |  |
+| vector.cpp:527:9:527:10 | ref arg it | vector.cpp:530:3:530:4 | it |  |
+| vector.cpp:527:9:527:10 | ref arg it | vector.cpp:531:9:531:10 | it |  |
 | vector.cpp:528:3:528:4 | it | vector.cpp:528:6:528:6 | call to operator+= |  |
 | vector.cpp:528:3:528:4 | ref arg it | vector.cpp:529:9:529:10 | it |  |
 | vector.cpp:528:3:528:4 | ref arg it | vector.cpp:530:3:530:4 | it |  |
 | vector.cpp:528:3:528:4 | ref arg it | vector.cpp:531:9:531:10 | it |  |
 | vector.cpp:528:9:528:9 | 1 | vector.cpp:528:3:528:4 | ref arg it | TAINT |
 | vector.cpp:529:9:529:10 | it | vector.cpp:529:8:529:8 | call to operator* | TAINT |
+| vector.cpp:529:9:529:10 | ref arg it | vector.cpp:530:3:530:4 | it |  |
+| vector.cpp:529:9:529:10 | ref arg it | vector.cpp:531:9:531:10 | it |  |
 | vector.cpp:530:3:530:4 | it | vector.cpp:530:6:530:6 | call to operator+= |  |
 | vector.cpp:530:3:530:4 | ref arg it | vector.cpp:531:9:531:10 | it |  |
 | vector.cpp:530:9:530:14 | call to source | vector.cpp:530:3:530:4 | ref arg it | TAINT |

cpp/ql/test/library-tests/dataflow/taint-tests/taint.cpp

@@ -655,5 +655,5 @@ public:
 void test_with_const_member(char* source) {
   C_const_member_function c;
   memcpy(c.data(), source, 16);
-  sink(c.data()); // $ MISSING: ast, ir
+  sink(c.data()); // $ ast MISSING: ir
 }

cpp/ql/test/query-tests/Security/CWE/CWE-014/MemsetMayBeDeleted.expected

@@ -0,0 +1,3 @@
+| test.cpp:48:5:48:10 | call to memset | Call to memset may be deleted by the compiler. |
+| test.cpp:79:5:79:10 | call to memset | Call to memset may be deleted by the compiler. |
+| test.cpp:208:2:208:7 | call to memset | Call to memset may be deleted by the compiler. |

cpp/ql/test/query-tests/Security/CWE/CWE-014/MemsetMayBeDeleted.qlref

@@ -0,0 +1 @@
+Security/CWE/CWE-014/MemsetMayBeDeleted.ql

cpp/ql/test/query-tests/Security/CWE/CWE-014/test.c

@@ -0,0 +1,140 @@
+typedef unsigned long long size_t;
+void *memset(void *s, int c, unsigned long n);
+void *__builtin_memset(void *s, int c, unsigned long n);
+typedef int errno_t;
+typedef unsigned int rsize_t;
+errno_t memset_s(void *dest, rsize_t destsz, int ch, rsize_t count);
+char *strcpy(char *dest, const char *src);
+
+extern void use_pw(char *pw);
+
+#define PW_SIZE 32
+
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.14 (WINE): deleted
+int func1(void) { 
+	char pw1[PW_SIZE];
+	use_pw(pw1);
+	memset(pw1, 0, PW_SIZE); // BAD [NOT DETECTED]
+	return 0;
+}
+
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.14 (WINE): not deleted
+int func1a(void) {  
+	char pw1a[PW_SIZE];
+	use_pw(pw1a);
+	__builtin_memset(pw1a, 0, PW_SIZE); // BAD [NOT DETECTED]
+	return 0;
+}
+
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.14 (WINE): deleted
+char *func1b(void) {  
+	char pw1b[PW_SIZE];
+	use_pw(pw1b);
+	memset(pw1b, 0, PW_SIZE); // BAD [NOT DETECTED]
+	pw1b[0] = pw1b[3] = 'a';
+	return 0;
+}
+
+// x86-64 gcc 9.2: not deleted
+// x86-64 clang 9.0.0: not deleted
+// x64 msvc v19.14 (WINE): not deleted
+int func1c(char pw1c[PW_SIZE]) {  
+	use_pw(pw1c);
+	memset(pw1c, 0, PW_SIZE); // GOOD
+	return 0;
+}
+
+// x86-64 gcc 9.2: not deleted
+// x86-64 clang 9.0.0: not deleted
+// x64 msvc v19.14 (WINE): not deleted
+char pw1d[PW_SIZE];
+int func1d() {  
+	use_pw(pw1d);
+	memset(pw1d, 0, PW_SIZE); // GOOD
+	return 0;
+}
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.14 (WINE): deleted
+char *func2(void) {  
+	char pw2[PW_SIZE];
+	use_pw(pw2);
+	memset(pw2, 1, PW_SIZE); // BAD [NOT DETECTED]
+	return pw2;
+}
+
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.14 (WINE): partially deleted
+int func3(void) { 
+	char pw3[PW_SIZE];
+	use_pw(pw3);
+	memset(pw3, 4, PW_SIZE); // BAD [NOT DETECTED]
+	return pw3[2];
+}
+
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.14 (WINE): not deleted
+int func4(void) { 
+	char pw1a[PW_SIZE];
+	use_pw(pw1a);
+	__builtin_memset(pw1a + 3, 0, PW_SIZE - 3); // BAD [NOT DETECTED]
+	return 0;
+}
+
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.14 (WINE): not deleted
+int func6(void) { 
+	char pw1a[PW_SIZE];
+	use_pw(pw1a);
+	__builtin_memset(&pw1a[3], 0, PW_SIZE - 3); // BAD [NOT DETECTED]
+	return pw1a[2];
+}
+
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.14 (WINE): not deleted
+int func5(void) { 
+	char pw1a[PW_SIZE];
+	use_pw(pw1a);
+	__builtin_memset(pw1a + 3, 0, PW_SIZE - 4); // GOOD
+	return pw1a[4];
+}
+
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.14 (WINE): not deleted
+int func7(void) { 
+	char pw1a[PW_SIZE];
+	use_pw(pw1a);
+	__builtin_memset(&pw1a[3], 0, PW_SIZE - 5); // BAD [NOT DETECTED]
+	return 0;
+}
+
+// x86-64 gcc 9.2: not deleted
+// x86-64 clang 9.0.0: not deleted
+// x64 msvc v19.14 (WINE): not deleted
+int func8(void) { 
+	char pw1a[PW_SIZE];
+	use_pw(pw1a);
+	__builtin_memset(pw1a + pw1a[3], 0, PW_SIZE - 4); // GOOD
+	return pw1a[4];
+}
+
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.14 (WINE): deleted
+char *func9(void) {
+	char pw1[PW_SIZE];
+	use_pw(pw1);
+	memset(pw1, 0, PW_SIZE); // BAD [NOT DETECTED]
+	return 0;
+}

cpp/ql/test/query-tests/Security/CWE/CWE-014/test.cpp

@@ -0,0 +1,401 @@
+extern "C" {
+	typedef unsigned long long size_t;
+	void *memset(void *s, int c, unsigned long n);
+	void *__builtin_memset(void *s, int c, unsigned long n);
+	typedef int errno_t;
+	typedef unsigned int rsize_t;
+	errno_t memset_s(void *dest, rsize_t destsz, int ch, rsize_t count);
+	char *strcpy(char *dest, const char *src);
+	void *memcpy(void *dest, const void *src, unsigned long n);
+	void *malloc(unsigned long size);
+	void free(void *ptr);
+	extern void use_pw(char *pw);
+	int printf(const char* format, ...);
+	char* gets(char * str);
+}
+
+#define PW_SIZE 32
+
+struct mem {
+	int a;
+	char b[PW_SIZE];
+	int c;
+};
+
+// x86-64 gcc 9.2: not deleted
+// x86-64 clang 9.0.0: not deleted
+// x64 msvc v19.22: not deleted
+void func(char buff[128], unsigned long long sz) {
+	gets(buff);
+  memset(buff, 0, PW_SIZE); // GOOD
+}
+
+// x86-64 gcc 9.2: not deleted
+// x86-64 clang 9.0.0: not deleted
+// x64 msvc v19.22: not deleted
+char *func2(char buff[128], unsigned long long sz) {
+	gets(buff);
+  memset(buff, 0, PW_SIZE); // GOOD
+  return buff;
+}
+
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.22: deleted
+void func3(unsigned long long sz) { 
+    char buff[128];
+		gets(buff);
+    memset(buff, 0, PW_SIZE); // BAD
+}
+
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.22: deleted
+void func4(unsigned long long sz) {
+    char buff[128];
+		gets(buff);
+    memset(buff, 0, PW_SIZE); // BAD [NOT DETECTED]
+    strcpy(buff, "Hello");
+}
+
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.22: deleted
+void func5(unsigned long long sz) {
+    char buff[128];
+		gets(buff);
+    memset(buff, 0, PW_SIZE); // BAD [NOT DETECTED]
+    if (sz > 5) {
+        strcpy(buff, "Hello");
+    }
+}
+
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.22: deleted
+void func6(unsigned long long sz) {
+    struct mem m;
+		gets(m.b);
+    memset(&m, 0, PW_SIZE); // BAD
+}
+
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.22: deleted
+void func7(unsigned long long sz) { 
+    struct mem m;
+		gets(m.b); 
+    memset(&m, 0, PW_SIZE); // BAD [NOT DETECTED]
+    m.a = 15;
+}
+
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.22: not deleted
+void func8(unsigned long long sz) {
+    struct mem *m = (struct mem *)malloc(sizeof(struct mem));
+		gets(m->b);
+    memset(m, 0, PW_SIZE); // BAD [NOT DETECTED]
+}
+
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.22: not deleted
+void func9(unsigned long long sz) {
+    struct mem *m = (struct mem *)malloc(sizeof(struct mem));
+		gets(m->b);
+    memset(m, 0, PW_SIZE); // BAD [NOT DETECTED]
+    free(m);
+}
+
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.22: not deleted
+void func10(unsigned long long sz) {
+    struct mem *m = (struct mem *)malloc(sizeof(struct mem));
+		gets(m->b);
+    memset(m, 0, PW_SIZE); // BAD [NOT DETECTED]
+    m->a = sz;
+    m->c = m->a + 1; 
+}
+
+// x86-64 gcc 9.2: deleted
+// x86-64 clang 9.0.0: deleted
+// x64 msvc v19.22: not deleted
+void func11(unsigned long long sz) {
+    struct mem *m = (struct mem *)malloc(sizeof(struct mem));
+		gets(m->b);
+    ::memset(m, 0, PW_SIZE); // BAD [NOT DETECTED]
+    if (sz > 5) {
+    	strcpy(m->b, "Hello");
+    }
+}
+
+// x86-64 gcc 9.2: not deleted
+// x86-64 clang 9.0.0: not deleted
+// x64 msvc v19.22: not deleted
+int func12(unsigned long long sz) {
+    struct mem *m = (struct mem *)malloc(sizeof(struct mem));
+		gets(m->b);
+    memset(m, 0, sz); // GOOD
+    return m->c;
+}
+
+int funcN1() {
+	char pw[PW_SIZE];
+	gets(pw);
+	char *pw_ptr = pw;
+	memset(pw, 0, PW_SIZE); // GOOD
+	use_pw(pw_ptr);
+	return 0;
+}
+
+char pw_global[PW_SIZE];
+int funcN2() {
+	gets(pw_global);
+	use_pw(pw_global);
+	memset(pw_global, 0, PW_SIZE); // GOOD
+	return 0;
+}
+
+int funcN3(unsigned long long sz) {
+    struct mem m;
+		gets(m.b);
+    memset(&m, 0, sizeof(m)); // GOOD
+    return m.a;
+}
+
+void funcN(int num) {
+	char pw[PW_SIZE];
+	int i;
+	for (i = 0; i < num; i++)
+	{
+		gets(pw);
+		use_pw(pw);
+		memset(pw, 0, PW_SIZE); // GOOD
+	}
+}
+
+class MyClass
+{
+public:
+	void set(int _x) {
+		x = _x;
+	}
+
+	int get()
+	{
+		return x;
+	}
+
+	void clear1() {
+		memset(&x, 0, sizeof(x)); // GOOD
+	}
+
+	void clear2() {
+		memset(&(this->x), 0, sizeof(this->x)); // GOOD
+	}
+
+private:
+	int x;
+};
+
+void badFunc0_0(){
+	unsigned char buff1[PW_SIZE];
+	for(int i = 0; i < PW_SIZE; i++) {
+		buff1[i] = 13;
+	}
+	memset(buff1, 0, PW_SIZE); // BAD
+}
+
+void nobadFunc1_0() {
+	char* buff1 = (char *) malloc(PW_SIZE);
+	gets(buff1);
+	memset(buff1, 0, PW_SIZE); // BAD [NOT DETECTED]
+}
+void badFunc1_0(){
+	char * buff1 = (char *) malloc(PW_SIZE);
+	gets(buff1);
+	memset(buff1, 0, PW_SIZE); // BAD [NOT DETECTED]
+	free(buff1);
+}
+void badFunc1_1(){
+	unsigned char buff1[PW_SIZE];
+	for(int i = 0; i < PW_SIZE; i++) {
+		buff1[i] = 'a' + i;	
+	}
+	memset(buff1, 0, PW_SIZE); // BAD [NOT DETECTED]
+	free(buff1);
+}
+void nobadFunc2_0_0(){
+	unsigned char buff1[PW_SIZE];
+	buff1[sizeof(buff1) - 1] = '\0';
+	memset(buff1, 0, PW_SIZE); // GOOD
+	printf("%s", buff1);
+}
+
+void nobadFunc2_0_1(){
+	char buff1[PW_SIZE];
+	gets(buff1);
+	memset(buff1, '\0', sizeof(buff1));
+	memset(buff1, 0, PW_SIZE); // GOOD
+	printf("%s", buff1 + 3);
+}
+
+void nobadFunc2_0_2(){
+	char buff1[PW_SIZE];
+	gets(buff1);
+	memset(buff1, 0, PW_SIZE); // GOOD
+	printf("%c", *buff1);
+}
+
+void nobadFunc2_0_3(char ch){
+	unsigned char buff1[PW_SIZE];
+	for(int i = 0; i < PW_SIZE; i++) {
+		buff1[i] = ch;
+	}
+	memset(buff1, 0, PW_SIZE); // GOOD
+	printf("%c", *(buff1 + 3));
+}
+
+char * nobadFunc2_0_4(){
+	char buff1[PW_SIZE];
+	gets(buff1);
+	memset(buff1, 0, PW_SIZE); // GOOD
+	return buff1;
+}
+
+char * nobadFunc2_0_5(){
+	char buff1[PW_SIZE];
+	gets(buff1);
+	memset(buff1, 0, PW_SIZE); // GOOD
+
+	return buff1+3;
+}
+
+unsigned char nobadFunc2_0_6(){
+	unsigned char buff1[PW_SIZE];
+	buff1[0] = 'z';
+	int i;
+	memset(buff1, 0, PW_SIZE); // GOOD
+
+	return *buff1;
+}
+
+unsigned char nobadFunc2_0_7(){
+	char buff1[PW_SIZE];
+	gets(buff1);
+	memset(buff1, 0, PW_SIZE); // GOOD
+
+	return *(buff1 + 3);
+}
+
+bool nobadFunc2_1_0(unsigned char ch){
+	char buff1[PW_SIZE];
+	gets(buff1);
+	memset(buff1, 0, PW_SIZE); // GOOD
+	if(*buff1 == ch) { return true; }
+	return false;
+}
+
+void nobadFunc2_1_2(){
+	char buff1[PW_SIZE];
+	gets(buff1);
+	memset(buff1, 0, PW_SIZE); // BAD [NOT DETECTED]
+	buff1[2] = 5;
+}
+
+void nobadFunc3_0(char * buffAll){
+	char * buff1 = buffAll;
+	gets(buff1);
+	memset(buff1, 0, PW_SIZE); // GOOD
+}
+
+void nobadFunc3_1(unsigned char * buffAll){
+	unsigned char * buff1 = buffAll + 3;
+	memset(buff1, 0, PW_SIZE); // GOOD
+}
+
+struct buffers
+{
+    char buff1[50];
+    unsigned char *buff2;
+};
+
+void nobadFunc3_2(struct buffers buffAll) {
+	char * buff1 = buffAll.buff1;
+	gets(buff1);
+	memset(buff1, 0, PW_SIZE); // GOOD
+}
+
+void nobadFunc3_3(struct buffers buffAll) {
+	unsigned char * buff1 = buffAll.buff2;
+	memset(buff1, 0, PW_SIZE); // GOOD
+}
+
+void nobadFunc3_4(struct buffers buffAll) {
+	unsigned char * buff1 = buffAll.buff2 + 3;
+	memset(buff1, 0, PW_SIZE); // GOOD
+}
+
+void nobadFunc3_5(struct buffers * buffAll) {
+	char * buff1 = buffAll->buff1;
+	memset(buff1, 0, PW_SIZE); // GOOD
+}
+
+void nobadFunc3_6(struct buffers *buffAll){
+	unsigned char * buff1 = buffAll->buff2;
+	memset(buff1, 0, PW_SIZE); // GOOD
+}
+
+char * globalBuff;
+
+void nobadFunc4(){
+	char * buff1 = globalBuff;
+	memset(buff1, 0, PW_SIZE); // GOOD
+}
+
+void nobadFunc4_0(){
+	char * buff1 = globalBuff;
+	gets(buff1);
+	memset(buff1, 0, PW_SIZE); // GOOD
+}
+void nobadFunc4_1(){
+	char * buff1 = globalBuff + 3;
+	memset(buff1, 0, PW_SIZE); // GOOD
+}
+
+buffers globalBuff1, *globalBuff2;
+
+void nobadFunc4_2(){
+	char * buff1 = globalBuff1.buff1;
+	memset(buff1, 0, PW_SIZE); // GOOD
+}
+
+void nobadFunc4_3(){
+	unsigned char * buff1 = globalBuff1.buff2;
+	memset(buff1, 0, PW_SIZE); // GOOD
+}
+
+void nobadFunc4_4(){
+	unsigned char * buff1 = globalBuff1.buff2+3;
+	memset(buff1, 0, PW_SIZE); // GOOD
+}
+
+void nobadFunc4_5(){
+	char * buff1 = globalBuff2->buff1;
+	memset(buff1, 0, PW_SIZE); // GOOD
+}
+
+void nobadFunc4_6(){
+	unsigned char * buff1 = globalBuff2->buff2;
+	memset(buff1, 0, PW_SIZE); // GOOD
+}
+
+extern void use_byte(unsigned char);
+
+void test_static_func() {
+	static unsigned char buffer[PW_SIZE] = {0};
+	use_byte(buffer[0]);
+	memset(buffer, 42, sizeof(buffer)); // GOOD
+}

csharp/change-notes/2021-02-12-with-expression.md

@@ -0,0 +1,4 @@
+lgtm,codescanning
+* C# 9 `with` expressions are now extracted. Data flow support has been added to
+handle flow through `with` expressions and also from `record` constructor arguments
+to its properties.

csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs

@@ -24,7 +24,7 @@ namespace Semmle.Extraction.CIL.Driver
         {
             var sw = new Stopwatch();
             sw.Start();
-            Entities.Assembly.ExtractCIL(layout, assemblyPath, logger, nocache, extractPdbs, trapCompression, out _, out _);
+            Analyser.ExtractCIL(layout, assemblyPath, logger, nocache, extractPdbs, trapCompression, out _, out _);
             sw.Stop();
             logger.Log(Severity.Info, "  {0} ({1})", assemblyPath, sw.Elapsed);
         }

csharp/extractor/Semmle.Extraction.CIL/Analyser.cs

@@ -0,0 +1,53 @@
+using Semmle.Util.Logging;
+using System;
+using Semmle.Util;
+using Semmle.Extraction.CIL.Entities;
+
+namespace Semmle.Extraction.CIL
+{
+    public static class Analyser
+    {
+        private static void ExtractCIL(Extractor extractor, TrapWriter trapWriter, bool extractPdbs)
+        {
+            using var cilContext = new Context(extractor, trapWriter, extractor.OutputPath, extractPdbs);
+            cilContext.Populate(new Assembly(cilContext));
+            cilContext.PopulateAll();
+        }
+
+        /// <summary>
+        /// Main entry point to the CIL extractor.
+        /// Call this to extract a given assembly.
+        /// </summary>
+        /// <param name="layout">The trap layout.</param>
+        /// <param name="assemblyPath">The full path of the assembly to extract.</param>
+        /// <param name="logger">The logger.</param>
+        /// <param name="nocache">True to overwrite existing trap file.</param>
+        /// <param name="extractPdbs">Whether to extract PDBs.</param>
+        /// <param name="trapFile">The path of the trap file.</param>
+        /// <param name="extracted">Whether the file was extracted (false=cached).</param>
+        public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs, TrapWriter.CompressionMode trapCompression, out string trapFile, out bool extracted)
+        {
+            trapFile = "";
+            extracted = false;
+            try
+            {
+                var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
+                var pathTransformer = new PathTransformer(canonicalPathCache);
+                var extractor = new Extractor(false, assemblyPath, logger, pathTransformer);
+                var transformedAssemblyPath = pathTransformer.Transform(assemblyPath);
+                var project = layout.LookupProjectOrDefault(transformedAssemblyPath);
+                using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), trapCompression, discardDuplicates: true);
+                trapFile = trapWriter.TrapFile;
+                if (nocache || !System.IO.File.Exists(trapFile))
+                {
+                    ExtractCIL(extractor, trapWriter, extractPdbs);
+                    extracted = true;
+                }
+            }
+            catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
+            {
+                logger.Log(Severity.Error, string.Format("Exception extracting {0}: {1}", assemblyPath, ex));
+            }
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CIL/Context.Factories.cs

@@ -9,11 +9,11 @@ namespace Semmle.Extraction.CIL
     /// <summary>
     /// Provides methods for creating and caching various entities.
     /// </summary>
-    public sealed partial class Context
+    internal sealed partial class Context
     {
         private readonly Dictionary<object, Label> ids = new Dictionary<object, Label>();
 
-        public T Populate<T>(T e) where T : IExtractedEntity
+        internal T Populate<T>(T e) where T : IExtractedEntity
         {
             if (e.Label.Valid)
             {
@@ -27,10 +27,10 @@ namespace Semmle.Extraction.CIL
             }
             else
             {
-                e.Label = Cx.GetNewLabel();
-                Cx.DefineLabel(e, Cx.TrapWriter.Writer, Cx.Extractor);
+                e.Label = GetNewLabel();
+                DefineLabel(e);
                 ids.Add(e, e.Label);
-                Cx.PopulateLater(() =>
+                PopulateLater(() =>
                 {
                     foreach (var c in e.Contents)
                         c.Extract(this);
@@ -42,7 +42,7 @@ namespace Semmle.Extraction.CIL
 
                 if (debugLabels.TryGetValue(id, out var previousEntity))
                 {
-                    Cx.Extractor.Message(new Message("Duplicate trap ID", id, null, severity: Util.Logging.Severity.Warning));
+                    Extractor.Message(new Message("Duplicate trap ID", id, null, severity: Util.Logging.Severity.Warning));
                 }
                 else
                 {
@@ -74,9 +74,9 @@ namespace Semmle.Extraction.CIL
             {
                 e = new PrimitiveType(this, code)
                 {
-                    Label = Cx.GetNewLabel()
+                    Label = GetNewLabel()
                 };
-                Cx.DefineLabel(e, Cx.TrapWriter.Writer, Cx.Extractor);
+                DefineLabel(e);
                 primitiveTypes[(int)code] = e;
             }
 

csharp/extractor/Semmle.Extraction.CIL/Context.cs

@@ -10,12 +10,10 @@ namespace Semmle.Extraction.CIL
     /// Adds additional context that is specific for CIL extraction.
     /// One context = one DLL/EXE.
     /// </summary>
-    public sealed partial class Context : IDisposable
+    internal sealed partial class Context : Extraction.Context, IDisposable
     {
         private readonly FileStream stream;
         private Entities.Assembly? assemblyNull;
-
-        public Extraction.Context Cx { get; }
         public MetadataReader MdReader { get; }
         public PEReader PeReader { get; }
         public string AssemblyPath { get; }
@@ -26,9 +24,9 @@ namespace Semmle.Extraction.CIL
         }
         public PDB.IPdb? Pdb { get; }
 
-        public Context(Extraction.Context cx, string assemblyPath, bool extractPdbs)
+        public Context(Extractor extractor, TrapWriter trapWriter, string assemblyPath, bool extractPdbs)
+            : base(extractor, trapWriter)
         {
-            this.Cx = cx;
             this.AssemblyPath = assemblyPath;
             stream = File.OpenRead(assemblyPath);
             PeReader = new PEReader(stream, PEStreamOptions.PrefetchEntireImage);
@@ -51,7 +49,7 @@ namespace Semmle.Extraction.CIL
                 Pdb = PDB.PdbReader.Create(assemblyPath, PeReader);
                 if (Pdb != null)
                 {
-                    cx.Extractor.Logger.Log(Util.Logging.Severity.Info, string.Format("Found PDB information for {0}", assemblyPath));
+                    Extractor.Logger.Log(Util.Logging.Severity.Info, string.Format("Found PDB information for {0}", assemblyPath));
                 }
             }
         }

csharp/extractor/Semmle.Extraction.CIL/EmptyContext.cs

@@ -5,7 +5,7 @@ namespace Semmle.Extraction.CIL
     /// <summary>
     /// A generic context which does not contain any type parameters.
     /// </summary>
-    public class EmptyContext : IGenericContext
+    internal class EmptyContext : IGenericContext
     {
         public EmptyContext(Context cx)
         {

csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs

@@ -1,18 +1,15 @@
 using System.Reflection;
 using System.Globalization;
 using System.Collections.Generic;
-using Semmle.Util.Logging;
-using System;
 using Semmle.Extraction.Entities;
 using System.IO;
-using Semmle.Util;
 
 namespace Semmle.Extraction.CIL.Entities
 {
     /// <summary>
     /// An assembly to extract.
     /// </summary>
-    public class Assembly : LabelledEntity, ILocation
+    internal class Assembly : LabelledEntity, ILocation
     {
         private readonly File file;
         private readonly AssemblyName assemblyName;
@@ -76,7 +73,7 @@ namespace Semmle.Extraction.CIL.Entities
                     }
                     catch (InternalError e)
                     {
-                        Cx.Cx.ExtractionError("Error processing type definition", e.Message, GeneratedLocation.Create(Cx.Cx), e.StackTrace);
+                        Cx.ExtractionError("Error processing type definition", e.Message, GeneratedLocation.Create(Cx), e.StackTrace);
                     }
 
                     // Limitation of C#: Cannot yield return inside a try-catch.
@@ -93,7 +90,7 @@ namespace Semmle.Extraction.CIL.Entities
                     }
                     catch (InternalError e)
                     {
-                        Cx.Cx.ExtractionError("Error processing bytecode", e.Message, GeneratedLocation.Create(Cx.Cx), e.StackTrace);
+                        Cx.ExtractionError("Error processing bytecode", e.Message, GeneratedLocation.Create(Cx), e.StackTrace);
                     }
 
                     if (product != null)
@@ -101,49 +98,5 @@ namespace Semmle.Extraction.CIL.Entities
                 }
             }
         }
-
-        private static void ExtractCIL(Extraction.Context cx, string assemblyPath, bool extractPdbs)
-        {
-            using var cilContext = new Context(cx, assemblyPath, extractPdbs);
-            cilContext.Populate(new Assembly(cilContext));
-            cilContext.Cx.PopulateAll();
-        }
-
-        /// <summary>
-        /// Main entry point to the CIL extractor.
-        /// Call this to extract a given assembly.
-        /// </summary>
-        /// <param name="layout">The trap layout.</param>
-        /// <param name="assemblyPath">The full path of the assembly to extract.</param>
-        /// <param name="logger">The logger.</param>
-        /// <param name="nocache">True to overwrite existing trap file.</param>
-        /// <param name="extractPdbs">Whether to extract PDBs.</param>
-        /// <param name="trapFile">The path of the trap file.</param>
-        /// <param name="extracted">Whether the file was extracted (false=cached).</param>
-        public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs, TrapWriter.CompressionMode trapCompression, out string trapFile, out bool extracted)
-        {
-            trapFile = "";
-            extracted = false;
-            try
-            {
-                var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
-                var pathTransformer = new PathTransformer(canonicalPathCache);
-                var extractor = new Extractor(false, assemblyPath, logger, pathTransformer);
-                var transformedAssemblyPath = pathTransformer.Transform(assemblyPath);
-                var project = layout.LookupProjectOrDefault(transformedAssemblyPath);
-                using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), trapCompression, discardDuplicates: true);
-                trapFile = trapWriter.TrapFile;
-                if (nocache || !System.IO.File.Exists(trapFile))
-                {
-                    var cx = new Extraction.Context(extractor, trapWriter);
-                    ExtractCIL(cx, assemblyPath, extractPdbs);
-                    extracted = true;
-                }
-            }
-            catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
-            {
-                logger.Log(Severity.Error, string.Format("Exception extracting {0}: {1}", assemblyPath, ex));
-            }
-        }
     }
 }

csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs

@@ -45,7 +45,7 @@ namespace Semmle.Extraction.CIL.Entities
                 }
                 catch
                 {
-                    Cx.Cx.Extractor.Logger.Log(Util.Logging.Severity.Info,
+                    Cx.Extractor.Logger.Log(Util.Logging.Severity.Info,
                         $"Attribute decoding is partial. Decoding attribute {constructor.DeclaringType.GetQualifiedName()} failed on {@object}.");
                     yield break;
                 }

csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractedEntity.cs

@@ -5,7 +5,7 @@ namespace Semmle.Extraction.CIL
     /// <summary>
     /// A CIL entity which has been extracted.
     /// </summary>
-    public interface IExtractedEntity : IExtractionProduct, IEntity
+    internal interface IExtractedEntity : IExtractionProduct, IEntity
     {
         /// <summary>
         /// The contents of the entity.

csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractionProduct.cs

@@ -12,7 +12,7 @@ namespace Semmle.Extraction.CIL
     /// - Enumerate Contents to produce more extraction products
     /// - Extract these until there is nothing left to extract
     /// </remarks>
-    public interface IExtractionProduct
+    internal interface IExtractionProduct
     {
         /// <summary>
         /// Perform further extraction/population of this item as necessary.

csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IGenericContext.cs

@@ -6,7 +6,7 @@ namespace Semmle.Extraction.CIL
     /// When we decode a type/method signature, we need access to
     /// generic parameters.
     /// </summary>
-    public interface IGenericContext
+    internal interface IGenericContext
     {
         Context Cx { get; }
 

csharp/extractor/Semmle.Extraction.CIL/Entities/Base/LabelledEntity.cs

@@ -8,14 +8,13 @@ namespace Semmle.Extraction.CIL
     /// An entity that needs to be populated during extraction.
     /// This assigns a key and optionally extracts its contents.
     /// </summary>
-    public abstract class LabelledEntity : Extraction.LabelledEntity, IExtractedEntity
+    internal abstract class LabelledEntity : Extraction.LabelledEntity, IExtractedEntity
     {
         // todo: with .NET 5 this can override the base context, and change the return type.
-        public Context Cx { get; }
+        public Context Cx => (Context)base.Context;
 
-        protected LabelledEntity(Context cx) : base(cx.Cx)
+        protected LabelledEntity(Context cx) : base(cx)
         {
-            this.Cx = cx;
         }
 
         public override Microsoft.CodeAnalysis.Location ReportingLocation => throw new NotImplementedException();

csharp/extractor/Semmle.Extraction.CIL/Entities/Base/Tuple.cs

@@ -14,7 +14,7 @@ namespace Semmle.Extraction.CIL
 
         public void Extract(Context cx)
         {
-            cx.Cx.Emit(tuple);
+            cx.TrapWriter.Emit(tuple);
         }
 
         public override string ToString() => tuple.ToString();

csharp/extractor/Semmle.Extraction.CIL/Entities/Base/UnlabelledEntity.cs

@@ -7,14 +7,13 @@ namespace Semmle.Extraction.CIL
     /// An entity that has contents to extract. There is no need to populate
     /// a key as it's done in the contructor.
     /// </summary>
-    public abstract class UnlabelledEntity : Extraction.UnlabelledEntity, IExtractedEntity
+    internal abstract class UnlabelledEntity : Extraction.UnlabelledEntity, IExtractedEntity
     {
         // todo: with .NET 5 this can override the base context, and change the return type.
-        public Context Cx { get; }
+        public Context Cx => (Context)base.Context;
 
-        protected UnlabelledEntity(Context cx) : base(cx.Cx)
+        protected UnlabelledEntity(Context cx) : base(cx)
         {
-            Cx = cx;
         }
 
         public override Microsoft.CodeAnalysis.Location ReportingLocation => throw new NotImplementedException();

csharp/extractor/Semmle.Extraction.CIL/Entities/ConstructedType.cs

@@ -9,7 +9,7 @@ namespace Semmle.Extraction.CIL.Entities
     /// <summary>
     /// A constructed type.
     /// </summary>
-    public sealed class ConstructedType : Type
+    internal sealed class ConstructedType : Type
     {
         private readonly Type unboundGenericType;
 

csharp/extractor/Semmle.Extraction.CIL/Entities/CustomAttributeDecoder.cs

@@ -40,11 +40,11 @@ namespace Semmle.Extraction.CIL.Entities
 
             if (wellKnownEnums.TryGetValue(name, out var code))
             {
-                cx.Cx.Extractor.Logger.Log(Util.Logging.Severity.Debug, $"Using hard coded underlying enum type for {name}");
+                cx.Extractor.Logger.Log(Util.Logging.Severity.Debug, $"Using hard coded underlying enum type for {name}");
                 return code;
             }
 
-            cx.Cx.Extractor.Logger.Log(Util.Logging.Severity.Info, $"Couldn't get underlying enum type for {name}");
+            cx.Extractor.Logger.Log(Util.Logging.Severity.Info, $"Couldn't get underlying enum type for {name}");
 
             // We can't fall back to Int32, because the type returned here defines how many bytes are read from the
             // stream and how those bytes are interpreted.
@@ -56,7 +56,8 @@ namespace Semmle.Extraction.CIL.Entities
         private static readonly Dictionary<string, PrimitiveTypeCode> wellKnownEnums = new Dictionary<string, PrimitiveTypeCode>
         {
             { "System.AttributeTargets", PrimitiveTypeCode.Int32 },
-            { "System.ComponentModel.EditorBrowsableState", PrimitiveTypeCode.Int32 }
+            { "System.ComponentModel.EditorBrowsableState", PrimitiveTypeCode.Int32 },
+            { "System.Diagnostics.DebuggerBrowsableState", PrimitiveTypeCode.Int32 }
         };
     }
 }

csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs

@@ -3,7 +3,7 @@ using System.IO;
 
 namespace Semmle.Extraction.CIL.Entities
 {
-    public class File : LabelledEntity, IFileOrFolder
+    internal class File : LabelledEntity, IFileOrFolder
     {
         protected string OriginalPath { get; }
         protected PathTransformer.ITransformedPath TransformedPath { get; }
@@ -11,7 +11,7 @@ namespace Semmle.Extraction.CIL.Entities
         public File(Context cx, string path) : base(cx)
         {
             this.OriginalPath = path;
-            TransformedPath = cx.Cx.Extractor.PathTransformer.Transform(OriginalPath);
+            TransformedPath = Cx.Extractor.PathTransformer.Transform(OriginalPath);
         }
 
         public override void WriteId(TextWriter trapFile)

csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs

@@ -3,7 +3,7 @@ using System.IO;
 
 namespace Semmle.Extraction.CIL.Entities
 {
-    public sealed class Folder : LabelledEntity, IFileOrFolder
+    internal sealed class Folder : LabelledEntity, IFileOrFolder
     {
         private readonly PathTransformer.ITransformedPath transformedPath;
 

csharp/extractor/Semmle.Extraction.CIL/Entities/Instruction.cs

@@ -476,7 +476,7 @@ namespace Semmle.Extraction.CIL.Entities
                 // TODO: Find a solution to this.
 
                 // For now, just log the error
-                Cx.Cx.ExtractionError("A CIL instruction jumps outside the current method", null, Extraction.Entities.GeneratedLocation.Create(Cx.Cx), "", Util.Logging.Severity.Warning);
+                Cx.ExtractionError("A CIL instruction jumps outside the current method", null, Extraction.Entities.GeneratedLocation.Create(Cx), "", Util.Logging.Severity.Warning);
             }
         }
     }

csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs

@@ -7,7 +7,7 @@ namespace Semmle.Extraction.CIL.Entities
     /// <summary>
     /// A namespace.
     /// </summary>
-    public sealed class Namespace : TypeContainer
+    internal sealed class Namespace : TypeContainer
     {
         public Namespace? ParentNamespace { get; }
         public string Name { get; }

csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs

@@ -2,7 +2,7 @@ using System.Collections.Generic;
 
 namespace Semmle.Extraction.CIL.Entities
 {
-    public class PdbSourceFile : File
+    internal class PdbSourceFile : File
     {
         private readonly PDB.ISourceFile file;
 
@@ -21,9 +21,9 @@ namespace Semmle.Extraction.CIL.Entities
                 var text = file.Contents;
 
                 if (text == null)
-                    Cx.Cx.Extractor.Logger.Log(Util.Logging.Severity.Warning, string.Format("PDB source file {0} could not be found", OriginalPath));
+                    Cx.Extractor.Logger.Log(Util.Logging.Severity.Warning, string.Format("PDB source file {0} could not be found", OriginalPath));
                 else
-                    Cx.Cx.TrapWriter.Archive(TransformedPath, text);
+                    Cx.TrapWriter.Archive(TransformedPath, text);
 
                 yield return Tuples.file_extraction_mode(this, 2);
             }

csharp/extractor/Semmle.Extraction.CIL/Entities/PrimitiveType.cs

@@ -5,7 +5,7 @@ using System.IO;
 
 namespace Semmle.Extraction.CIL.Entities
 {
-    public sealed class PrimitiveType : Type
+    internal sealed class PrimitiveType : Type
     {
         private readonly PrimitiveTypeCode typeCode;
         public PrimitiveType(Context cx, PrimitiveTypeCode tc) : base(cx)

csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs

@@ -4,7 +4,7 @@ using Semmle.Extraction.PDB;
 
 namespace Semmle.Extraction.CIL.Entities
 {
-    public sealed class PdbSourceLocation : LabelledEntity, ILocation
+    internal sealed class PdbSourceLocation : LabelledEntity, ILocation
     {
         private readonly Location location;
         private readonly PdbSourceFile file;

csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs

@@ -9,7 +9,7 @@ namespace Semmle.Extraction.CIL.Entities
     /// <summary>
     /// A type.
     /// </summary>
-    public abstract class Type : TypeContainer, IMember
+    internal abstract class Type : TypeContainer, IMember
     {
         internal const string AssemblyTypeNameSeparator = "::";
         internal const string PrimitiveTypePrefix = "builtin" + AssemblyTypeNameSeparator + "System.";

csharp/extractor/Semmle.Extraction.CIL/Entities/TypeContainer.cs

@@ -6,7 +6,7 @@ namespace Semmle.Extraction.CIL.Entities
     /// <summary>
     /// Base class for all type containers (namespaces, types, methods).
     /// </summary>
-    public abstract class TypeContainer : LabelledEntity, IGenericContext
+    internal abstract class TypeContainer : LabelledEntity, IGenericContext
     {
         protected TypeContainer(Context cx) : base(cx)
         {

csharp/extractor/Semmle.Extraction.CIL/Entities/TypeDefinitionType.cs

@@ -11,7 +11,7 @@ namespace Semmle.Extraction.CIL.Entities
     /// <summary>
     /// A type defined in the current assembly.
     /// </summary>
-    public sealed class TypeDefinitionType : Type
+    internal sealed class TypeDefinitionType : Type
     {
         private readonly TypeDefinitionHandle handle;
         private readonly TypeDefinition td;

csharp/extractor/Semmle.Extraction.CIL/Entities/TypeReferenceType.cs

@@ -9,7 +9,7 @@ namespace Semmle.Extraction.CIL.Entities
     /// <summary>
     /// A type reference, to a type in a referenced assembly.
     /// </summary>
-    public sealed class TypeReferenceType : Type
+    internal sealed class TypeReferenceType : Type
     {
         private readonly TypeReferenceHandle handle;
         private readonly TypeReference tr;

csharp/extractor/Semmle.Extraction.CIL/Entities/TypeSignatureDecoder.cs

@@ -8,7 +8,7 @@ namespace Semmle.Extraction.CIL.Entities
     /// <summary>
     /// Decodes a type signature and produces a Type, for use by DecodeSignature() and friends.
     /// </summary>
-    public class TypeSignatureDecoder : ISignatureTypeProvider<Type, IGenericContext>
+    internal class TypeSignatureDecoder : ISignatureTypeProvider<Type, IGenericContext>
     {
         private readonly Context cx;
 

csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs

@@ -292,7 +292,7 @@ namespace Semmle.Extraction.CSharp
                             AnalyseNamespace(cx, module.GlobalNamespace);
                         }
 
-                        Entities.Attribute.ExtractAttributes(cx, assembly, Extraction.Entities.Assembly.Create(cx, assembly.GetSymbolLocation()));
+                        Entities.Attribute.ExtractAttributes(cx, assembly, Entities.Assembly.Create(cx, assembly.GetSymbolLocation()));
 
                         cx.PopulateAll();
                     }
@@ -310,7 +310,7 @@ namespace Semmle.Extraction.CSharp
         {
             var stopwatch = new Stopwatch();
             stopwatch.Start();
-            CIL.Entities.Assembly.ExtractCIL(layout, r.FilePath, Logger, !options.Cache, options.PDB, options.TrapCompression, out var trapFile, out var extracted);
+            CIL.Analyser.ExtractCIL(layout, r.FilePath, Logger, !options.Cache, options.PDB, options.TrapCompression, out var trapFile, out var extracted);
             stopwatch.Stop();
             ReportProgress(r.FilePath, trapFile, stopwatch.Elapsed, extracted ? AnalysisAction.Extracted : AnalysisAction.UpToDate);
         }
@@ -374,7 +374,7 @@ namespace Semmle.Extraction.CSharp
                         var cx = new Context(extractor, compilation.Clone(), trapWriter, new SourceScope(tree), AddAssemblyTrapPrefix);
                         // Ensure that the file itself is populated in case the source file is totally empty
                         var root = tree.GetRoot();
-                        Extraction.Entities.File.Create(cx, root.SyntaxTree.FilePath);
+                        Entities.File.Create(cx, root.SyntaxTree.FilePath);
 
                         var csNode = (CSharpSyntaxNode)root;
                         csNode.Accept(new CompilationUnitVisitor(cx));

csharp/extractor/Semmle.Extraction.CSharp/CachedEntityFactory.cs

@@ -0,0 +1,19 @@
+namespace Semmle.Extraction.CSharp
+{
+    /// <summary>
+    /// A factory for creating cached entities.
+    /// </summary>
+    internal abstract class CachedEntityFactory<TInit, TEntity>
+        : Extraction.CachedEntityFactory<TInit, TEntity> where TEntity : CachedEntity
+    {
+        /// <summary>
+        /// Initializes the entity, but does not generate any trap code.
+        /// </summary>
+        public sealed override TEntity Create(Extraction.Context cx, TInit init)
+        {
+            return Create((Context)cx, init);
+        }
+
+        public abstract TEntity Create(Context cx, TInit init);
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentBinding.cs

@@ -0,0 +1,13 @@
+namespace Semmle.Extraction.CSharp
+{
+    /// <summary>
+    /// Describes the relationship between a comment and a program element.
+    /// </summary>
+    public enum CommentBinding
+    {
+        Parent,     // The parent element of a comment
+        Best,       // The most likely element associated with a comment
+        Before,     // The element before the comment
+        After       // The element after the comment
+    };
+}

csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentBlock.cs

@@ -0,0 +1,59 @@
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.Text;
+using Semmle.Extraction.CSharp.Entities;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Semmle.Extraction.CSharp.Comments
+{
+    internal class CommentBlock
+    {
+        private readonly List<CommentLine> lines;
+
+        public IEnumerable<CommentLine> CommentLines => lines;
+
+        public Location Location { get; private set; }
+
+        public CommentBlock(CommentLine firstLine)
+        {
+            lines = new List<CommentLine> { firstLine };
+            Location = firstLine.Location;
+        }
+
+        /// <summary>
+        ///     Determine whether commentlines should be merged.
+        /// </summary>
+        /// <param name="newLine">A comment line to be appended to this comment block.</param>
+        /// <returns>Whether the new line should be appended to this block.</returns>
+        public bool CombinesWith(CommentLine newLine)
+        {
+            if (!CommentLines.Any())
+                return true;
+
+            var sameFile = Location.SourceTree == newLine.Location.SourceTree;
+            var sameRow = Location.EndLine() == newLine.Location.StartLine();
+            var sameColumn = Location.EndLine() + 1 == newLine.Location.StartLine();
+            var nextRow = Location.StartColumn() == newLine.Location.StartColumn();
+            var adjacent = sameFile && (sameRow || (sameColumn && nextRow));
+
+            return
+                newLine.Type == CommentLineType.MultilineContinuation ||
+                adjacent;
+        }
+
+        /// <summary>
+        /// Adds a comment line to the this comment block.
+        /// </summary>
+        /// <param name="line">The line to add.</param>
+        public void AddCommentLine(CommentLine line)
+        {
+            Location = !lines.Any()
+                ? line.Location
+                : Location.Create(
+                    line.Location.SourceTree!,
+                    new TextSpan(Location.SourceSpan.Start, line.Location.SourceSpan.End - Location.SourceSpan.Start));
+
+            lines.Add(line);
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentLineType.cs

@@ -0,0 +1,13 @@
+namespace Semmle.Extraction.CSharp
+{
+    /// <summary>
+    /// The type of a single comment line.
+    /// </summary>
+    public enum CommentLineType
+    {
+        Singleline,             // Comment starting // ...
+        XmlDoc,                 // Comment starting /// ...
+        Multiline,              // Comment starting /* ..., even if the comment only spans one line.
+        MultilineContinuation   // The second and subsequent lines of comment in a multiline comment.
+    };
+}

csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentProcessor.cs

@@ -1,32 +1,32 @@
 using Microsoft.CodeAnalysis;
-using Microsoft.CodeAnalysis.Text;
+using Semmle.Extraction.CSharp.Entities;
 using Semmle.Util;
 using System.Collections.Generic;
 using System.Linq;
 
-namespace Semmle.Extraction.CommentProcessing
+namespace Semmle.Extraction.CSharp
 {
     /// <summary>
     /// Implements the comment processor for associating comments with program elements.
     /// Registers locations of comments and program elements,
     /// then generates binding information.
     /// </summary>
-    internal class CommentProcessor : ICommentGenerator
+    internal class CommentProcessor
     {
-        public void AddComment(ICommentLine comment)
+        public void AddComment(CommentLine comment)
         {
             comments[comment.Location] = comment;
         }
 
         // Comments sorted by location.
-        private readonly SortedDictionary<Location, ICommentLine> comments = new SortedDictionary<Location, ICommentLine>(new LocationComparer());
+        private readonly SortedDictionary<Location, CommentLine> comments = new SortedDictionary<Location, CommentLine>(new LocationComparer());
 
         // Program elements sorted by location.
         private readonly SortedDictionary<Location, Label> elements = new SortedDictionary<Location, Label>(new LocationComparer());
 
         private readonly Dictionary<Label, Key> duplicationGuardKeys = new Dictionary<Label, Key>();
 
-        private Key? GetDuplicationGuardKey(Label label)
+        private Key GetDuplicationGuardKey(Label label)
         {
             if (duplicationGuardKeys.TryGetValue(label, out var duplicationGuardKey))
                 return duplicationGuardKey;
@@ -35,7 +35,7 @@ namespace Semmle.Extraction.CommentProcessing
 
         private class LocationComparer : IComparer<Location>
         {
-            public int Compare(Location? l1, Location? l2) => CommentProcessor.Compare(l1, l2);
+            public int Compare(Location l1, Location l2) => CommentProcessor.Compare(l1, l2);
         }
 
         /// <summary>
@@ -44,7 +44,7 @@ namespace Semmle.Extraction.CommentProcessing
         /// <param name="l1">First location</param>
         /// <param name="l2">Second location</param>
         /// <returns>&lt;0 if l1 before l2, &gt;0 if l1 after l2, else 0.</returns>
-        private static int Compare(Location? l1, Location? l2)
+        private static int Compare(Location l1, Location l2)
         {
             if (object.ReferenceEquals(l1, l2))
                 return 0;
@@ -68,7 +68,7 @@ namespace Semmle.Extraction.CommentProcessing
         /// <param name="elementLabel">The label of the element in the trap file.</param>
         /// <param name="duplicationGuardKey">The duplication guard key of the element, if any.</param>
         /// <param name="loc">The location of the element.</param>
-        public void AddElement(Label elementLabel, Key? duplicationGuardKey, Location loc)
+        public void AddElement(Label elementLabel, Key duplicationGuardKey, Location loc)
         {
             if (loc != null && loc.IsInSource)
                 elements[loc] = elementLabel;
@@ -78,7 +78,7 @@ namespace Semmle.Extraction.CommentProcessing
 
         // Ensure that commentBlock and element refer to the same file
         // which can happen when processing multiple files.
-        private static void EnsureSameFile(ICommentBlock commentBlock, ref KeyValuePair<Location, Label>? element)
+        private static void EnsureSameFile(Comments.CommentBlock commentBlock, ref KeyValuePair<Location, Label>? element)
         {
             if (element != null && element.Value.Key.SourceTree != commentBlock.Location.SourceTree)
                 element = null;
@@ -95,7 +95,7 @@ namespace Semmle.Extraction.CommentProcessing
         /// <param name="parentElement">The parent element of the comment block.</param>
         /// <param name="callback">Output binding information.</param>
         private void GenerateBindings(
-            ICommentBlock commentBlock,
+            Comments.CommentBlock commentBlock,
             KeyValuePair<Location, Label>? previousElement,
             KeyValuePair<Location, Label>? nextElement,
             KeyValuePair<Location, Label>? parentElement,
@@ -231,7 +231,7 @@ namespace Semmle.Extraction.CommentProcessing
 
         // Generate binding information for one CommentBlock.
         private void GenerateBindings(
-            ICommentBlock block,
+            Comments.CommentBlock block,
             ElementStack elementStack,
             KeyValuePair<Location, Label>? nextElement,
             CommentBindingCallback cb
@@ -259,25 +259,25 @@ namespace Semmle.Extraction.CommentProcessing
         /// <param name="cb">Where to send the results.</param>
         /// <returns>true if there are more comments to process, false otherwise.</returns>
         private bool GenerateBindings(
-            IEnumerator<KeyValuePair<Location, ICommentLine>> commentEnumerator,
+            IEnumerator<KeyValuePair<Location, CommentLine>> commentEnumerator,
             KeyValuePair<Location, Label>? nextElement,
             ElementStack elementStack,
             CommentBindingCallback cb
             )
         {
-            CommentBlock? block = null;
+            Comments.CommentBlock block = null;
 
             // Iterate comments until the commentEnumerator has gone past nextElement
             while (nextElement == null || Compare(commentEnumerator.Current.Value.Location, nextElement.Value.Key) < 0)
             {
                 if (block is null)
-                    block = new CommentBlock(commentEnumerator.Current.Value);
+                    block = new Comments.CommentBlock(commentEnumerator.Current.Value);
 
                 if (!block.CombinesWith(commentEnumerator.Current.Value))
                 {
                     // Start of a new block, so generate the bindings for the old block first.
                     GenerateBindings(block, elementStack, nextElement, cb);
-                    block = new CommentBlock(commentEnumerator.Current.Value);
+                    block = new Comments.CommentBlock(commentEnumerator.Current.Value);
                 }
                 else
                 {
@@ -320,7 +320,7 @@ namespace Semmle.Extraction.CommentProcessing
             var elementStack = new ElementStack();
 
             using IEnumerator<KeyValuePair<Location, Label>> elementEnumerator = elements.GetEnumerator();
-            using IEnumerator<KeyValuePair<Location, ICommentLine>> commentEnumerator = comments.GetEnumerator();
+            using IEnumerator<KeyValuePair<Location, CommentLine>> commentEnumerator = comments.GetEnumerator();
             if (!commentEnumerator.MoveNext())
             {
                 // There are no comments to process.
@@ -343,54 +343,12 @@ namespace Semmle.Extraction.CommentProcessing
         }
     }
 
-    internal class CommentBlock : ICommentBlock
-    {
-        private readonly List<ICommentLine> lines;
-
-        public IEnumerable<ICommentLine> CommentLines => lines;
-
-        public Location Location { get; private set; }
-
-        public CommentBlock(ICommentLine firstLine)
-        {
-            lines = new List<ICommentLine> { firstLine };
-            Location = firstLine.Location;
-        }
-
-        /// <summary>
-        ///     Determine whether commentlines should be merged.
-        /// </summary>
-        /// <param name="newLine">A comment line to be appended to this comment block.</param>
-        /// <returns>Whether the new line should be appended to this block.</returns>
-        public bool CombinesWith(ICommentLine newLine)
-        {
-            if (!CommentLines.Any())
-                return true;
-
-            var sameFile = Location.SourceTree == newLine.Location.SourceTree;
-            var sameRow = Location.EndLine() == newLine.Location.StartLine();
-            var sameColumn = Location.EndLine() + 1 == newLine.Location.StartLine();
-            var nextRow = Location.StartColumn() == newLine.Location.StartColumn();
-            var adjacent = sameFile && (sameRow || (sameColumn && nextRow));
-
-            return
-                newLine.Type == CommentLineType.MultilineContinuation ||
-                adjacent;
-        }
-
-        /// <summary>
-        /// Adds a comment line to the this comment block.
-        /// </summary>
-        /// <param name="line">The line to add.</param>
-        public void AddCommentLine(ICommentLine line)
-        {
-            Location = !lines.Any()
-                ? line.Location
-                : Location.Create(
-                    line.Location.SourceTree!,
-                    new TextSpan(Location.SourceSpan.Start, line.Location.SourceSpan.End - Location.SourceSpan.Start));
-
-            lines.Add(line);
-        }
-    }
+    /// <summary>
+    /// Callback for generated comment associations.
+    /// </summary>
+    /// <param name="elementLabel">The label of the element</param>
+    /// <param name="duplicationGuardKey">The duplication guard key of the element, if any</param>
+    /// <param name="commentBlock">The comment block associated with the element</param>
+    /// <param name="binding">The relationship between the commentblock and the element</param>
+    internal delegate void CommentBindingCallback(Label elementLabel, Key duplicationGuardKey, Comments.CommentBlock commentBlock, CommentBinding binding);
 }

csharp/extractor/Semmle.Extraction.CSharp/Context.cs

@@ -0,0 +1,144 @@
+using Microsoft.CodeAnalysis;
+using System;
+using System.Diagnostics.CodeAnalysis;
+using Semmle.Extraction.Entities;
+using System.Collections.Generic;
+
+namespace Semmle.Extraction.CSharp
+{
+    /// <summary>
+    /// State that needs to be available throughout the extraction process.
+    /// There is one Context object per trap output file.
+    /// </summary>
+    internal class Context : Extraction.Context
+    {
+        /// <summary>
+        /// The program database provided by Roslyn.
+        /// There's one per syntax tree, which makes things awkward.
+        /// </summary>
+        public SemanticModel GetModel(SyntaxNode node)
+        {
+            // todo: when this context belongs to a SourceScope, the syntax tree can be retrieved from the scope, and
+            // the node parameter could be removed. Is there any case when we pass in a node that's not from the current
+            // tree?
+            if (cachedModel == null || node.SyntaxTree != cachedModel.SyntaxTree)
+            {
+                cachedModel = Compilation.GetSemanticModel(node.SyntaxTree);
+            }
+
+            return cachedModel;
+        }
+
+        private SemanticModel cachedModel;
+
+        /// <summary>
+        /// The current compilation unit.
+        /// </summary>
+        public Compilation Compilation { get; }
+
+        internal CommentProcessor CommentGenerator { get; } = new CommentProcessor();
+
+        public Context(Extraction.Extractor e, Compilation c, TrapWriter trapWriter, IExtractionScope scope, bool addAssemblyTrapPrefix)
+            : base(e, trapWriter, addAssemblyTrapPrefix)
+        {
+            Compilation = c;
+            this.scope = scope;
+        }
+
+        public bool FromSource => scope is SourceScope;
+
+        private readonly IExtractionScope scope;
+
+        public bool IsAssemblyScope => scope is AssemblyScope;
+
+        private SyntaxTree SourceTree => scope is SourceScope sc ? sc.SourceTree : null;
+
+        /// <summary>
+        ///     Whether the given symbol needs to be defined in this context.
+        ///     This is the case if the symbol is contained in the source/assembly, or
+        ///     of the symbol is a constructed generic.
+        /// </summary>
+        /// <param name="symbol">The symbol to populate.</param>
+        public bool Defines(ISymbol symbol) =>
+            !SymbolEqualityComparer.Default.Equals(symbol, symbol.OriginalDefinition) ||
+            scope.InScope(symbol);
+
+        public override void WithDuplicationGuard(Key key, Action a)
+        {
+            if (IsAssemblyScope)
+            {
+                // No need for a duplication guard when extracting assemblies,
+                // and the duplication guard could lead to method bodies being missed
+                // depending on trap import order.
+                a();
+            }
+            else
+            {
+                base.WithDuplicationGuard(key, a);
+            }
+        }
+
+        public override Extraction.Entities.Location CreateLocation()
+        {
+            return SourceTree == null
+                ? GeneratedLocation.Create(this)
+                : CreateLocation(Microsoft.CodeAnalysis.Location.Create(SourceTree, Microsoft.CodeAnalysis.Text.TextSpan.FromBounds(0, 0)));
+        }
+
+        public override Extraction.Entities.Location CreateLocation(Microsoft.CodeAnalysis.Location location)
+        {
+            return (location == null || location.Kind == LocationKind.None)
+                ? GeneratedLocation.Create(this)
+                : location.IsInSource
+                    ? Entities.NonGeneratedSourceLocation.Create(this, location)
+                    : Entities.Assembly.Create(this, location);
+        }
+
+        /// <summary>
+        /// Register a program entity which can be bound to comments.
+        /// </summary>
+        /// <param name="cx">Extractor context.</param>
+        /// <param name="entity">Program entity.</param>
+        /// <param name="l">Location of the entity.</param>
+        public void BindComments(Entity entity, Microsoft.CodeAnalysis.Location l)
+        {
+            var duplicationGuardKey = GetCurrentTagStackKey();
+            CommentGenerator.AddElement(entity.Label, duplicationGuardKey, l);
+        }
+
+        protected override bool IsEntityDuplicationGuarded(IEntity entity, [NotNullWhen(true)] out Extraction.Entities.Location loc)
+        {
+            if (CreateLocation(entity.ReportingLocation) is Entities.NonGeneratedSourceLocation l)
+            {
+                loc = l;
+                return true;
+            }
+
+            loc = null;
+            return false;
+        }
+
+        private readonly HashSet<Label> extractedGenerics = new HashSet<Label>();
+
+        /// <summary>
+        /// Should the given entity be extracted?
+        /// A second call to this method will always return false,
+        /// on the assumption that it would have been extracted on the first call.
+        ///
+        /// This is used to track the extraction of generics, which cannot be extracted
+        /// in a top-down manner.
+        /// </summary>
+        /// <param name="entity">The entity to extract.</param>
+        /// <returns>True only on the first call for a particular entity.</returns>
+        internal bool ExtractGenerics(CachedEntity entity)
+        {
+            if (extractedGenerics.Contains(entity.Label))
+            {
+                return false;
+            }
+
+            extractedGenerics.Add(entity.Label);
+            return true;
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp/Entities/Accessor.cs

@@ -86,11 +86,11 @@ namespace Semmle.Extraction.CSharp.Entities
         public static new Accessor Create(Context cx, IMethodSymbol symbol) =>
             AccessorFactory.Instance.CreateEntityFromSymbol(cx, symbol);
 
-        private class AccessorFactory : ICachedEntityFactory<IMethodSymbol, Accessor>
+        private class AccessorFactory : CachedEntityFactory<IMethodSymbol, Accessor>
         {
             public static AccessorFactory Instance { get; } = new AccessorFactory();
 
-            public Accessor Create(Context cx, IMethodSymbol init) => new Accessor(cx, init);
+            public override Accessor Create(Context cx, IMethodSymbol init) => new Accessor(cx, init);
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Assembly.cs

@@ -1,14 +1,18 @@
 using Microsoft.CodeAnalysis;
+using Semmle.Extraction.CSharp;
 using System.IO;
 
-namespace Semmle.Extraction.Entities
+namespace Semmle.Extraction.CSharp.Entities
 {
-    public class Assembly : Location
+    internal class Assembly : Extraction.Entities.Location
     {
+        // todo: this can be changed to an override after the .NET 5 upgrade
+        private new Context Context => (Context)base.Context;
+
         private readonly string assemblyPath;
         private readonly IAssemblySymbol assembly;
 
-        private Assembly(Context cx, Microsoft.CodeAnalysis.Location? init)
+        private Assembly(Context cx, Microsoft.CodeAnalysis.Location init)
             : base(cx, init)
         {
             if (init == null)
@@ -40,7 +44,7 @@ namespace Semmle.Extraction.Entities
         public override int GetHashCode() =>
             Symbol == null ? 91187354 : Symbol.GetHashCode();
 
-        public override bool Equals(object? obj)
+        public override bool Equals(object obj)
         {
             if (obj is Assembly other && other.GetType() == typeof(Assembly))
                 return Equals(Symbol, other.Symbol);
@@ -48,13 +52,13 @@ namespace Semmle.Extraction.Entities
             return false;
         }
 
-        public static Location Create(Context cx, Microsoft.CodeAnalysis.Location loc) => AssemblyConstructorFactory.Instance.CreateEntity(cx, loc, loc);
+        public static Extraction.Entities.Location Create(Context cx, Microsoft.CodeAnalysis.Location loc) => AssemblyConstructorFactory.Instance.CreateEntity(cx, loc, loc);
 
-        private class AssemblyConstructorFactory : ICachedEntityFactory<Microsoft.CodeAnalysis.Location?, Assembly>
+        private class AssemblyConstructorFactory : CachedEntityFactory<Microsoft.CodeAnalysis.Location, Assembly>
         {
             public static AssemblyConstructorFactory Instance { get; } = new AssemblyConstructorFactory();
 
-            public Assembly Create(Context cx, Microsoft.CodeAnalysis.Location? init) => new Assembly(cx, init);
+            public override Assembly Create(Context cx, Microsoft.CodeAnalysis.Location init) => new Assembly(cx, init);
         }
 
         private static readonly object outputAssemblyCacheKey = new object();

csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs

@@ -136,11 +136,11 @@ namespace Semmle.Extraction.CSharp.Entities
             return AttributeFactory.Instance.CreateEntity(cx, attributeData, init);
         }
 
-        private class AttributeFactory : ICachedEntityFactory<(AttributeData attributeData, IEntity receiver), Attribute>
+        private class AttributeFactory : CachedEntityFactory<(AttributeData attributeData, IEntity receiver), Attribute>
         {
             public static readonly AttributeFactory Instance = new AttributeFactory();
 
-            public Attribute Create(Context cx, (AttributeData attributeData, IEntity receiver) init) =>
+            public override Attribute Create(Context cx, (AttributeData attributeData, IEntity receiver) init) =>
                 new Attribute(cx, init.attributeData, init.receiver);
         }
     }

csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedEntity.cs

@@ -0,0 +1,12 @@
+namespace Semmle.Extraction.CSharp.Entities
+{
+    internal abstract class CachedEntity<T> : Extraction.CachedEntity<T>
+    {
+        // todo: this can be changed to an override after the .NET 5 upgrade
+        protected new Context Context => (Context)base.Context;
+
+        protected CachedEntity(Context context, T symbol) : base(context, symbol)
+        {
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedSymbol.cs

@@ -9,10 +9,15 @@ using System.Reflection.Metadata.Ecma335;
 
 namespace Semmle.Extraction.CSharp.Entities
 {
-    public abstract class CachedSymbol<T> : CachedEntity<T> where T : ISymbol
+    internal abstract class CachedSymbol<T> : CachedEntity<T> where T : ISymbol
     {
+        // todo: this can be changed to an override after the .NET 5 upgrade
+        protected new Context Context => (Context)base.Context;
+
         protected CachedSymbol(Context cx, T init)
-            : base(cx, init) { }
+            : base(cx, init)
+        {
+        }
 
         public virtual Type ContainingType => Symbol.ContainingType != null ? Type.Create(Context, Symbol.ContainingType) : null;
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentBlock.cs

@@ -1,12 +1,11 @@
-using Semmle.Extraction.CommentProcessing;
 using Semmle.Extraction.Entities;
 using System.IO;
 
 namespace Semmle.Extraction.CSharp.Entities
 {
-    internal class CommentBlock : CachedEntity<ICommentBlock>
+    internal class CommentBlock : CachedEntity<Comments.CommentBlock>
     {
-        private CommentBlock(Context cx, ICommentBlock init)
+        private CommentBlock(Context cx, Comments.CommentBlock init)
             : base(cx, init) { }
 
         public override void Populate(TextWriter trapFile)
@@ -35,13 +34,13 @@ namespace Semmle.Extraction.CSharp.Entities
             Context.TrapWriter.Writer.commentblock_binding(this, entity, binding);
         }
 
-        public static CommentBlock Create(Context cx, ICommentBlock block) => CommentBlockFactory.Instance.CreateEntity(cx, block, block);
+        public static CommentBlock Create(Context cx, Comments.CommentBlock block) => CommentBlockFactory.Instance.CreateEntity(cx, block, block);
 
-        private class CommentBlockFactory : ICachedEntityFactory<ICommentBlock, CommentBlock>
+        private class CommentBlockFactory : CachedEntityFactory<Comments.CommentBlock, CommentBlock>
         {
             public static CommentBlockFactory Instance { get; } = new CommentBlockFactory();
 
-            public CommentBlock Create(Context cx, ICommentBlock init) => new CommentBlock(cx, init);
+            public override CommentBlock Create(Context cx, Comments.CommentBlock init) => new CommentBlock(cx, init);
         }
 
         public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel;

csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs

@@ -1,10 +1,9 @@
-using Semmle.Extraction.CommentProcessing;
 using Semmle.Extraction.Entities;
 using System.IO;
 
 namespace Semmle.Extraction.CSharp.Entities
 {
-    internal class CommentLine : CachedEntity<(Microsoft.CodeAnalysis.Location, string)>, ICommentLine
+    internal class CommentLine : CachedEntity<(Microsoft.CodeAnalysis.Location, string)>
     {
         private CommentLine(Context cx, Microsoft.CodeAnalysis.Location loc, CommentLineType type, string text, string raw)
             : base(cx, (loc, text))
@@ -44,11 +43,11 @@ namespace Semmle.Extraction.CSharp.Entities
             return CommentLineFactory.Instance.CreateEntity(cx, init, init);
         }
 
-        private class CommentLineFactory : ICachedEntityFactory<(Microsoft.CodeAnalysis.Location, CommentLineType, string, string), CommentLine>
+        private class CommentLineFactory : CachedEntityFactory<(Microsoft.CodeAnalysis.Location, CommentLineType, string, string), CommentLine>
         {
             public static CommentLineFactory Instance { get; } = new CommentLineFactory();
 
-            public CommentLine Create(Context cx, (Microsoft.CodeAnalysis.Location, CommentLineType, string, string) init) =>
+            public override CommentLine Create(Context cx, (Microsoft.CodeAnalysis.Location, CommentLineType, string, string) init) =>
                 new CommentLine(cx, init.Item1, init.Item2, init.Item3, init.Item4);
         }
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs

@@ -6,7 +6,7 @@ using Semmle.Util;
 
 namespace Semmle.Extraction.CSharp.Entities
 {
-    public class Compilation : CachedEntity<object>
+    internal class Compilation : CachedEntity<object>
     {
         private static (string Cwd, string[] Args) settings;
         private static int hashCode;
@@ -31,7 +31,7 @@ namespace Semmle.Extraction.CSharp.Entities
 
         public override void Populate(TextWriter trapFile)
         {
-            var assembly = Extraction.Entities.Assembly.CreateOutputAssembly(Context);
+            var assembly = Assembly.CreateOutputAssembly(Context);
 
             trapFile.compilations(this, FileUtils.ConvertToUnix(Compilation.Settings.Cwd));
             trapFile.compilation_assembly(this, assembly);
@@ -45,7 +45,7 @@ namespace Semmle.Extraction.CSharp.Entities
 
             // Files
             index = 0;
-            foreach (var file in Context.Compilation.SyntaxTrees.Select(tree => Extraction.Entities.File.Create(Context, tree.FilePath)))
+            foreach (var file in Context.Compilation.SyntaxTrees.Select(tree => File.Create(Context, tree.FilePath)))
             {
                 trapFile.compilation_compiling_files(this, index++, file);
             }
@@ -54,7 +54,7 @@ namespace Semmle.Extraction.CSharp.Entities
             index = 0;
             foreach (var file in Context.Compilation.References
                 .OfType<PortableExecutableReference>()
-                .Select(r => Extraction.Entities.File.Create(Context, r.FilePath)))
+                .Select(r => File.Create(Context, r.FilePath)))
             {
                 trapFile.compilation_referencing_files(this, index++, file);
             }
@@ -90,11 +90,11 @@ namespace Semmle.Extraction.CSharp.Entities
 
         public override bool NeedsPopulation => Context.IsAssemblyScope;
 
-        private class CompilationFactory : ICachedEntityFactory<object, Compilation>
+        private class CompilationFactory : CachedEntityFactory<object, Compilation>
         {
             public static CompilationFactory Instance { get; } = new CompilationFactory();
 
-            public Compilation Create(Context cx, object init) => new Compilation(cx);
+            public override Compilation Create(Context cx, object init) => new Compilation(cx);
         }
 
         private static readonly object compilationCacheKey = new object();

csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs

@@ -8,7 +8,7 @@ using System.IO;
 
 namespace Semmle.Extraction.CSharp.Entities
 {
-    public class Constructor : Method
+    internal class Constructor : Method
     {
         private Constructor(Context cx, IMethodSymbol init)
             : base(cx, init) { }
@@ -145,11 +145,11 @@ namespace Semmle.Extraction.CSharp.Entities
             }
         }
 
-        private class ConstructorFactory : ICachedEntityFactory<IMethodSymbol, Constructor>
+        private class ConstructorFactory : CachedEntityFactory<IMethodSymbol, Constructor>
         {
             public static ConstructorFactory Instance { get; } = new ConstructorFactory();
 
-            public Constructor Create(Context cx, IMethodSymbol init) => new Constructor(cx, init);
+            public override Constructor Create(Context cx, IMethodSymbol init) => new Constructor(cx, init);
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Conversion.cs

@@ -26,11 +26,11 @@ namespace Semmle.Extraction.CSharp.Entities
             }
         }
 
-        private class ConversionFactory : ICachedEntityFactory<IMethodSymbol, Conversion>
+        private class ConversionFactory : CachedEntityFactory<IMethodSymbol, Conversion>
         {
             public static ConversionFactory Instance { get; } = new ConversionFactory();
 
-            public Conversion Create(Context cx, IMethodSymbol init) => new Conversion(cx, init);
+            public override Conversion Create(Context cx, IMethodSymbol init) => new Conversion(cx, init);
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Destructor.cs

@@ -26,11 +26,11 @@ namespace Semmle.Extraction.CSharp.Entities
         public static new Destructor Create(Context cx, IMethodSymbol symbol) =>
             DestructorFactory.Instance.CreateEntityFromSymbol(cx, symbol);
 
-        private class DestructorFactory : ICachedEntityFactory<IMethodSymbol, Destructor>
+        private class DestructorFactory : CachedEntityFactory<IMethodSymbol, Destructor>
         {
             public static DestructorFactory Instance { get; } = new DestructorFactory();
 
-            public Destructor Create(Context cx, IMethodSymbol init) => new Destructor(cx, init);
+            public override Destructor Create(Context cx, IMethodSymbol init) => new Destructor(cx, init);
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs

@@ -65,11 +65,11 @@ namespace Semmle.Extraction.CSharp.Entities
 
         public static Event Create(Context cx, IEventSymbol symbol) => EventFactory.Instance.CreateEntityFromSymbol(cx, symbol);
 
-        private class EventFactory : ICachedEntityFactory<IEventSymbol, Event>
+        private class EventFactory : CachedEntityFactory<IEventSymbol, Event>
         {
             public static EventFactory Instance { get; } = new EventFactory();
 
-            public Event Create(Context cx, IEventSymbol init) => new Event(cx, init);
+            public override Event Create(Context cx, IEventSymbol init) => new Event(cx, init);
         }
 
         public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel;

csharp/extractor/Semmle.Extraction.CSharp/Entities/EventAccessor.cs

@@ -55,11 +55,11 @@ namespace Semmle.Extraction.CSharp.Entities
         public static new EventAccessor Create(Context cx, IMethodSymbol symbol) =>
             EventAccessorFactory.Instance.CreateEntityFromSymbol(cx, symbol);
 
-        private class EventAccessorFactory : ICachedEntityFactory<IMethodSymbol, EventAccessor>
+        private class EventAccessorFactory : CachedEntityFactory<IMethodSymbol, EventAccessor>
         {
             public static EventAccessorFactory Instance { get; } = new EventAccessorFactory();
 
-            public EventAccessor Create(Context cx, IMethodSymbol init) => new EventAccessor(cx, init);
+            public override EventAccessor Create(Context cx, IMethodSymbol init) => new EventAccessor(cx, init);
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Binary.cs

@@ -23,12 +23,13 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
 
         private static ExprKind GetKind(Context cx, BinaryExpressionSyntax node)
         {
-            var k = GetBinaryTokenKind(cx, node.OperatorToken.Kind());
+            var k = GetBinaryTokenKind(cx, node);
             return GetCallType(cx, node).AdjustKind(k);
         }
 
-        private static ExprKind GetBinaryTokenKind(Context cx, SyntaxKind kind)
+        private static ExprKind GetBinaryTokenKind(Context cx, BinaryExpressionSyntax node)
         {
+            var kind = node.OperatorToken.Kind();
             switch (kind)
             {
                 case SyntaxKind.LessThanToken: return ExprKind.LT;
@@ -54,7 +55,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                 case SyntaxKind.QuestionQuestionToken: return ExprKind.NULL_COALESCING;
                 // !! And the rest
                 default:
-                    cx.ModelError($"Unhandled operator type {kind}");
+                    cx.ModelError(node, $"Unhandled operator type {kind}");
                     return ExprKind.UNKNOWN;
             }
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Factory.cs

@@ -250,6 +250,9 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                     case SyntaxKind.SuppressNullableWarningExpression:
                         return PostfixUnary.Create(info.SetKind(ExprKind.SUPPRESS_NULLABLE_WARNING), ((PostfixUnaryExpressionSyntax)info.Node).Operand);
 
+                    case SyntaxKind.WithExpression:
+                        return WithExpression.Create(info);
+
                     default:
                         info.Context.ModelError(info.Node, $"Unhandled expression '{info.Node}' of kind '{info.Node.Kind()}'");
                         return new Unknown(info);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Initializer.cs

@@ -12,7 +12,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
         protected Initializer(ExpressionNodeInfo info) : base(info) { }
     }
 
-    internal class ArrayInitializer : Expression<InitializerExpressionSyntax>
+    internal class ArrayInitializer : Initializer
     {
         private ArrayInitializer(ExpressionNodeInfo info) : base(info.SetType(null).SetKind(ExprKind.ARRAY_INIT)) { }
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Name.cs

@@ -13,6 +13,13 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
 
             var target = symbolInfo.Symbol;
 
+            if (target == null &&
+                symbolInfo.CandidateReason == CandidateReason.OverloadResolutionFailure &&
+                info.Node.Parent.IsKind(SyntaxKind.SuppressNullableWarningExpression))
+            {
+                target = symbolInfo.CandidateSymbols.FirstOrDefault();
+            }
+
             if (target == null && symbolInfo.CandidateReason == CandidateReason.OverloadResolutionFailure)
             {
                 // The expression is probably a cast

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ObjectCreation/BaseObjectCreation.cs

@@ -52,7 +52,7 @@ namespace Semmle.Extraction.CSharp.Entities.Expressions
                         ObjectInitializer.Create(new ExpressionNodeInfo(Context, Syntax.Initializer, this, -1).SetType(Type));
                         break;
                     default:
-                        Context.ModelError("Unhandled initializer in object creation");
+                        Context.ModelError(Syntax.Initializer, "Unhandled initializer in object creation");
                         break;
                 }
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/WithExpression.cs

@@ -0,0 +1,20 @@
+using Microsoft.CodeAnalysis.CSharp.Syntax;
+using Semmle.Extraction.Kinds;
+using System.IO;
+
+namespace Semmle.Extraction.CSharp.Entities.Expressions
+{
+    internal class WithExpression : Expression<WithExpressionSyntax>
+    {
+        private WithExpression(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.WITH)) { }
+
+        public static Expression Create(ExpressionNodeInfo info) => new WithExpression(info).TryPopulate();
+
+        protected override void PopulateExpression(TextWriter trapFile)
+        {
+            Create(Context, Syntax.Expression, this, 0);
+
+            ObjectInitializer.Create(new ExpressionNodeInfo(Context, Syntax.Initializer, this, 1).SetType(Type));
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs

@@ -130,11 +130,11 @@ namespace Semmle.Extraction.CSharp.Entities
 
         bool IExpressionParentEntity.IsTopLevelParent => true;
 
-        private class FieldFactory : ICachedEntityFactory<IFieldSymbol, Field>
+        private class FieldFactory : CachedEntityFactory<IFieldSymbol, Field>
         {
             public static FieldFactory Instance { get; } = new FieldFactory();
 
-            public Field Create(Context cx, IFieldSymbol init) => new Field(cx, init);
+            public override Field Create(Context cx, IFieldSymbol init) => new Field(cx, init);
         }
         public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.PushesLabel;
     }

csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs

@@ -0,0 +1,83 @@
+using Microsoft.CodeAnalysis;
+using Semmle.Util;
+using System;
+using System.IO;
+using System.Linq;
+
+namespace Semmle.Extraction.CSharp.Entities
+{
+    internal class File : Extraction.Entities.File
+    {
+        // todo: this can be changed to an override after the .NET 5 upgrade
+        private new Context Context => (Context)base.Context;
+
+        protected File(Context cx, string path)
+            : base(cx, path)
+        {
+        }
+
+        public override void Populate(TextWriter trapFile)
+        {
+            trapFile.files(this, TransformedPath.Value, TransformedPath.NameWithoutExtension, TransformedPath.Extension);
+
+            if (TransformedPath.ParentDirectory is PathTransformer.ITransformedPath dir)
+                trapFile.containerparent(Extraction.Entities.Folder.Create(Context, dir), this);
+
+            var trees = Context.Compilation.SyntaxTrees.Where(t => t.FilePath == originalPath);
+
+            if (trees.Any())
+            {
+                foreach (var text in trees.Select(tree => tree.GetText()))
+                {
+                    var rawText = text.ToString() ?? "";
+                    var lineCounts = LineCounter.ComputeLineCounts(rawText);
+                    if (rawText.Length > 0 && rawText[rawText.Length - 1] != '\n')
+                        lineCounts.Total++;
+
+                    trapFile.numlines(this, lineCounts);
+                    Context.TrapWriter.Archive(originalPath, TransformedPath, text.Encoding ?? System.Text.Encoding.Default);
+                }
+            }
+            else if (IsPossiblyTextFile())
+            {
+                try
+                {
+                    System.Text.Encoding encoding;
+                    var lineCount = 0;
+                    using (var sr = new StreamReader(originalPath, detectEncodingFromByteOrderMarks: true))
+                    {
+                        while (sr.ReadLine() != null)
+                        {
+                            lineCount++;
+                        }
+                        encoding = sr.CurrentEncoding;
+                    }
+
+                    trapFile.numlines(this, new LineCounts() { Total = lineCount, Code = 0, Comment = 0 });
+                    Context.TrapWriter.Archive(originalPath, TransformedPath, encoding ?? System.Text.Encoding.Default);
+                }
+                catch (Exception exc)
+                {
+                    Context.ExtractionError($"Couldn't read file: {originalPath}. {exc.Message}", null, null, exc.StackTrace);
+                }
+            }
+
+            trapFile.file_extraction_mode(this, Context.Extractor.Standalone ? 1 : 0);
+        }
+
+        private bool IsPossiblyTextFile()
+        {
+            var extension = TransformedPath.Extension.ToLowerInvariant();
+            return !extension.Equals("dll") && !extension.Equals("exe");
+        }
+
+        public static File Create(Context cx, string path) => FileFactory.Instance.CreateEntity(cx, (typeof(File), path), path);
+
+        private class FileFactory : CachedEntityFactory<string, File>
+        {
+            public static FileFactory Instance { get; } = new FileFactory();
+
+            public override File Create(Context cx, string init) => new File(cx, init);
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp/Entities/FreshEntity.cs

@@ -0,0 +1,13 @@
+namespace Semmle.Extraction.CSharp.Entities
+{
+    internal abstract class FreshEntity : Extraction.FreshEntity
+    {
+        // todo: this can be changed to an override after the .NET 5 upgrade
+        protected new Context Context => (Context)base.Context;
+
+        protected FreshEntity(Context cx)
+            : base(cx)
+        {
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs

@@ -98,11 +98,11 @@ namespace Semmle.Extraction.CSharp.Entities
 
         bool IExpressionParentEntity.IsTopLevelParent => true;
 
-        private class IndexerFactory : ICachedEntityFactory<IPropertySymbol, Indexer>
+        private class IndexerFactory : CachedEntityFactory<IPropertySymbol, Indexer>
         {
             public static IndexerFactory Instance { get; } = new IndexerFactory();
 
-            public Indexer Create(Context cx, IPropertySymbol init) => new Indexer(cx, init);
+            public override Indexer Create(Context cx, IPropertySymbol init) => new Indexer(cx, init);
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalFunction.cs

@@ -22,11 +22,11 @@ namespace Semmle.Extraction.CSharp.Entities
 
         public static new LocalFunction Create(Context cx, IMethodSymbol field) => LocalFunctionFactory.Instance.CreateEntityFromSymbol(cx, field);
 
-        private class LocalFunctionFactory : ICachedEntityFactory<IMethodSymbol, LocalFunction>
+        private class LocalFunctionFactory : CachedEntityFactory<IMethodSymbol, LocalFunction>
         {
             public static LocalFunctionFactory Instance { get; } = new LocalFunctionFactory();
 
-            public LocalFunction Create(Context cx, IMethodSymbol init) => new LocalFunction(cx, init);
+            public override LocalFunction Create(Context cx, IMethodSymbol init) => new LocalFunction(cx, init);
         }
 
         public override void Populate(TextWriter trapFile)

csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs

@@ -53,11 +53,11 @@ namespace Semmle.Extraction.CSharp.Entities
             }
         }
 
-        private class LocalVariableFactory : ICachedEntityFactory<ISymbol, LocalVariable>
+        private class LocalVariableFactory : CachedEntityFactory<ISymbol, LocalVariable>
         {
             public static LocalVariableFactory Instance { get; } = new LocalVariableFactory();
 
-            public LocalVariable Create(Context cx, ISymbol init) => new LocalVariable(cx, init);
+            public override LocalVariable Create(Context cx, ISymbol init) => new LocalVariable(cx, init);
         }
 
         public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NeedsLabel;

csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs

@@ -8,7 +8,7 @@ using System.Linq;
 
 namespace Semmle.Extraction.CSharp.Entities
 {
-    public abstract class Method : CachedSymbol<IMethodSymbol>, IExpressionParentEntity, IStatementParentEntity
+    internal abstract class Method : CachedSymbol<IMethodSymbol>, IExpressionParentEntity, IStatementParentEntity
     {
         protected Method(Context cx, IMethodSymbol init)
             : base(cx, init) { }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs

@@ -140,11 +140,11 @@ namespace Semmle.Extraction.CSharp.Entities
             return ModifierFactory.Instance.CreateEntity(cx, (typeof(Modifier), modifier), modifier);
         }
 
-        private class ModifierFactory : ICachedEntityFactory<string, Modifier>
+        private class ModifierFactory : CachedEntityFactory<string, Modifier>
         {
             public static ModifierFactory Instance { get; } = new ModifierFactory();
 
-            public Modifier Create(Context cx, string init) => new Modifier(cx, init);
+            public override Modifier Create(Context cx, string init) => new Modifier(cx, init);
         }
         public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.OptionalLabel;
     }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Namespace.cs

@@ -36,11 +36,11 @@ namespace Semmle.Extraction.CSharp.Entities
 
         public static Namespace Create(Context cx, INamespaceSymbol ns) => NamespaceFactory.Instance.CreateEntityFromSymbol(cx, ns);
 
-        private class NamespaceFactory : ICachedEntityFactory<INamespaceSymbol, Namespace>
+        private class NamespaceFactory : CachedEntityFactory<INamespaceSymbol, Namespace>
         {
             public static NamespaceFactory Instance { get; } = new NamespaceFactory();
 
-            public Namespace Create(Context cx, INamespaceSymbol init) => new Namespace(cx, init);
+            public override Namespace Create(Context cx, INamespaceSymbol init) => new Namespace(cx, init);
         }
 
         public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel;

csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs

@@ -52,11 +52,11 @@ namespace Semmle.Extraction.CSharp.Entities
             return NamespaceDeclarationFactory.Instance.CreateEntity(cx, decl, init);
         }
 
-        private class NamespaceDeclarationFactory : ICachedEntityFactory<(NamespaceDeclarationSyntax decl, NamespaceDeclaration parent), NamespaceDeclaration>
+        private class NamespaceDeclarationFactory : CachedEntityFactory<(NamespaceDeclarationSyntax decl, NamespaceDeclaration parent), NamespaceDeclaration>
         {
             public static readonly NamespaceDeclarationFactory Instance = new NamespaceDeclarationFactory();
 
-            public NamespaceDeclaration Create(Context cx, (NamespaceDeclarationSyntax decl, NamespaceDeclaration parent) init) =>
+            public override NamespaceDeclaration Create(Context cx, (NamespaceDeclarationSyntax decl, NamespaceDeclaration parent) init) =>
                 new NamespaceDeclaration(cx, init.decl, init.parent);
         }
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/NonGeneratedSourceLocation.cs

@@ -0,0 +1,66 @@
+using Microsoft.CodeAnalysis;
+using System.IO;
+
+namespace Semmle.Extraction.CSharp.Entities
+{
+    internal class NonGeneratedSourceLocation : Extraction.Entities.SourceLocation
+    {
+        // todo: this can be changed to an override after the .NET 5 upgrade
+        private new Context Context => (Context)base.Context;
+
+        protected NonGeneratedSourceLocation(Context cx, Location init)
+            : base(cx, init)
+        {
+            Position = init.GetLineSpan();
+            FileEntity = File.Create(Context, Position.Path);
+        }
+
+        public static NonGeneratedSourceLocation Create(Context cx, Location loc) => SourceLocationFactory.Instance.CreateEntity(cx, loc, loc);
+
+        public override void Populate(TextWriter trapFile)
+        {
+            trapFile.locations_default(this, FileEntity,
+                Position.Span.Start.Line + 1, Position.Span.Start.Character + 1,
+                Position.Span.End.Line + 1, Position.Span.End.Character);
+
+            var mapped = Symbol!.GetMappedLineSpan();
+            if (mapped.HasMappedPath && mapped.IsValid)
+            {
+                var mappedLoc = Create(Context, Location.Create(mapped.Path, default, mapped.Span));
+
+                trapFile.locations_mapped(this, mappedLoc);
+            }
+        }
+
+        public FileLinePositionSpan Position
+        {
+            get;
+        }
+
+        public File FileEntity
+        {
+            get;
+        }
+
+        public override void WriteId(TextWriter trapFile)
+        {
+            trapFile.Write("loc,");
+            trapFile.WriteSubId(FileEntity);
+            trapFile.Write(',');
+            trapFile.Write(Position.Span.Start.Line + 1);
+            trapFile.Write(',');
+            trapFile.Write(Position.Span.Start.Character + 1);
+            trapFile.Write(',');
+            trapFile.Write(Position.Span.End.Line + 1);
+            trapFile.Write(',');
+            trapFile.Write(Position.Span.End.Character);
+        }
+
+        private class SourceLocationFactory : CachedEntityFactory<Location, NonGeneratedSourceLocation>
+        {
+            public static SourceLocationFactory Instance { get; } = new SourceLocationFactory();
+
+            public override NonGeneratedSourceLocation Create(Context cx, Location init) => new NonGeneratedSourceLocation(cx, init);
+        }
+    }
+}