hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-18 13:34:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into unreachable2

Browse Source
This commit is contained in:
Simon Friis Vindum
2024-10-18 15:13:40 +02:00
committed by GitHub
parent e1aaa136b2 41e811773c
commit b665f45ffa
88 changed files with 56104 additions and 51730 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cpp/ql/lib/change-notes/2024-10-16-function-pointer-resolution.md Normal file
View File

@@ -0,0 +1,4 @@
---
category: minorAnalysis
---
* The function call target resolution algorithm has been improved to resolve more calls through function pointers. As a result, dataflow queries may have more results.

4
cpp/ql/lib/change-notes/2024-10-16-new-api-for-call-target-resolution.md Normal file
View File

@@ -0,0 +1,4 @@
---
category: feature
---
* Added a new predicate `DataFlow::getARuntimeTarget` for getting a function that may be invoked by a `Call` expression. Unlike `Call.getTarget` this new predicate may also resolve function pointers.

21
cpp/ql/lib/semmle/code/cpp/Element.qll
View File

@@ -129,7 +129,7 @@ class Element extends ElementBase {
* or certain kinds of `Statement`.
*/
Element getParentScope() {
// result instanceof class
// result instanceof Class
exists(Declaration m |
m = this and
result = m.getDeclaringType() and
@@ -138,31 +138,40 @@ class Element extends ElementBase {
or
exists(TemplateClass tc | this = tc.getATemplateArgument() and result = tc)
or
// result instanceof namespace
// result instanceof Namespace
exists(Namespace n | result = n and n.getADeclaration() = this)
or
exists(FriendDecl d, Namespace n | this = d and n.getADeclaration() = d and result = n)
or
exists(Namespace n | this = n and result = n.getParentNamespace())
or
// result instanceof stmt
// result instanceof Stmt
exists(LocalVariable v |
this = v and
exists(DeclStmt ds | ds.getADeclaration() = v and result = ds.getParent())
)
or
exists(Parameter p | this = p and result = p.getFunction())
exists(Parameter p |
this = p and
(
result = p.getFunction() or
result = p.getCatchBlock().getParent().(Handler).getParent().(TryStmt).getParent() or
result = p.getRequiresExpr().getEnclosingStmt().getParent()
)
)
or
exists(GlobalVariable g, Namespace n | this = g and n.getADeclaration() = g and result = n)
or
exists(TemplateVariable tv | this = tv.getATemplateArgument() and result = tv)
or
exists(EnumConstant e | this = e and result = e.getDeclaringEnum())
or
// result instanceof block|function
// result instanceof Block|Function
exists(BlockStmt b | this = b and blockscope(unresolveElement(b), unresolveElement(result)))
or
exists(TemplateFunction tf | this = tf.getATemplateArgument() and result = tf)
or
// result instanceof stmt
// result instanceof Stmt
exists(ControlStructure s | this = s and result = s.getParent())
or
using_container(unresolveElement(result), underlyingElement(this))

3
cpp/ql/lib/semmle/code/cpp/Parameter.qll
View File

@@ -73,7 +73,8 @@ class Parameter extends LocalScopeVariable, @parameter {
}
private VariableDeclarationEntry getANamedDeclarationEntry() {
result = this.getAnEffectiveDeclarationEntry() and result.getName() != ""
result = this.getAnEffectiveDeclarationEntry() and
exists(string name | var_decls(unresolveElement(result), _, _, name, _) | name != "")
}
/**

20
cpp/ql/lib/semmle/code/cpp/Variable.qll
View File

@@ -241,6 +241,10 @@ class VariableDeclarationEntry extends DeclarationEntry, @var_decl {
name != "" and result = name
or
name = "" and result = this.getVariable().(LocalVariable).getName()
or
name = "" and
not this instanceof ParameterDeclarationEntry and
result = this.getVariable().(Parameter).getName()
)
)
}
@@ -295,19 +299,11 @@ class ParameterDeclarationEntry extends VariableDeclarationEntry {
private string getAnonymousParameterDescription() {
not exists(this.getName()) and
exists(string idx |
idx =
((this.getIndex() + 1).toString() + "th")
.replaceAll("1th", "1st")
.replaceAll("2th", "2nd")
.replaceAll("3th", "3rd")
.replaceAll("11st", "11th")
.replaceAll("12nd", "12th")
.replaceAll("13rd", "13th") and
exists(string anon |
anon = "(unnamed parameter " + this.getIndex().toString() + ")" and
if exists(this.getCanonicalName())
then
result = "declaration of " + this.getCanonicalName() + " as anonymous " + idx + " parameter"
else result = "declaration of " + idx + " parameter"
then result = "declaration of " + this.getCanonicalName() + " as " + anon
else result = "declaration of " + anon
)
}

5
cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll
View File

@@ -1328,7 +1328,10 @@ predicate lambdaCreation(Node creation, LambdaCallKind kind, DataFlowCallable c)
/** Holds if `call` is a lambda call of kind `kind` where `receiver` is the lambda expression. */
predicate lambdaCall(DataFlowCall call, LambdaCallKind kind, Node receiver) {
call.(SummaryCall).getReceiver() = receiver.(FlowSummaryNode).getSummaryNode() and
(
call.(SummaryCall).getReceiver() = receiver.(FlowSummaryNode).getSummaryNode() or
call.asCallInstruction().getCallTargetOperand() = receiver.asOperand()
) and
exists(kind)
}

14
cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll
View File

@@ -17,6 +17,7 @@ private import SsaInternals as Ssa
private import DataFlowImplCommon as DataFlowImplCommon
private import codeql.util.Unit
private import Node0ToString
private import DataFlowDispatch as DataFlowDispatch
import ExprNodes
/**
@@ -2497,3 +2498,16 @@ class AdditionalCallTarget extends Unit {
*/
abstract Declaration viableTarget(Call call);
}
/**
* Gets a function that may be called by `call`.
*
* Note that `call` may be a call to a function pointer expression.
*/
Function getARuntimeTarget(Call call) {
exists(DataFlowCall dfCall | dfCall.asCallInstruction().getUnconvertedResultExpression() = call |
result = DataFlowDispatch::viableCallable(dfCall).asSourceCallable()
or
result = DataFlowImplCommon::viableCallableLambda(dfCall, _).asSourceCallable()
)
}

3
cpp/ql/test/library-tests/dataflow/dataflow-tests/dataflow-consistency.expected
View File

@@ -3,6 +3,9 @@ uniqueEnclosingCallable
| test.cpp:864:47:864:54 | call to source | Node should have one enclosing callable but has 0. |
| test.cpp:872:46:872:51 | call to source | Node should have one enclosing callable but has 0. |
| test.cpp:872:53:872:56 | 1 | Node should have one enclosing callable but has 0. |
| test.cpp:1126:33:1129:1 | {...} | Node should have one enclosing callable but has 0. |
| test.cpp:1127:3:1127:13 | reads_input | Node should have one enclosing callable but has 0. |
| test.cpp:1128:3:1128:21 | not_does_read_input | Node should have one enclosing callable but has 0. |
uniqueCallEnclosingCallable
| test.cpp:864:47:864:54 | call to source | Call should have one enclosing callable but has 0. |
| test.cpp:872:46:872:51 | call to source | Call should have one enclosing callable but has 0. |

1
cpp/ql/test/library-tests/dataflow/dataflow-tests/test-source-sink.expected
View File

@@ -323,6 +323,7 @@ irFlow
| test.cpp:1069:9:1069:14 | call to source | test.cpp:1074:10:1074:10 | i |
| test.cpp:1069:9:1069:14 | call to source | test.cpp:1081:10:1081:10 | i |
| test.cpp:1117:27:1117:34 | call to source | test.cpp:1117:27:1117:34 | call to source |
| test.cpp:1132:11:1132:16 | call to source | test.cpp:1121:8:1121:8 | x |
| true_upon_entry.cpp:9:11:9:16 | call to source | true_upon_entry.cpp:13:8:13:8 | x |
| true_upon_entry.cpp:17:11:17:16 | call to source | true_upon_entry.cpp:21:8:21:8 | x |
| true_upon_entry.cpp:27:9:27:14 | call to source | true_upon_entry.cpp:29:8:29:8 | x |

16
cpp/ql/test/library-tests/dataflow/dataflow-tests/test.cpp
View File

@@ -1115,4 +1115,20 @@ void indirect_sink_const_ref(const T&);
void test_temp_with_conversion_from_materialization() {
indirect_sink_const_ref(source()); // $ ir MISSING: ast
}
void reads_input(int x) {
sink(x); // $ ir MISSING: ast
}
void not_does_read_input(int x);
void (*dispatch_table[])(int) = {
reads_input,
not_does_read_input
};
void test_dispatch_table(int i) {
int x = source();
dispatch_table[i](x);
}

4
cpp/ql/test/library-tests/declarationEntry/declarationEntry/roundTrip.expected
View File

@@ -25,8 +25,8 @@
| declarationEntry.cpp:39:7:39:7 | declaration of operator= | declarationEntry.cpp:39:7:39:7 | operator= | yes |
| declarationEntry.cpp:39:7:39:13 | definition of myClass | declarationEntry.cpp:39:7:39:13 | myClass | yes |
| declarationEntry.cpp:42:6:42:21 | definition of myMemberVariable | declarationEntry.cpp:42:6:42:21 | myMemberVariable | yes |
| file://:0:0:0:0 | declaration of 1st parameter | file://:0:0:0:0 | (unnamed parameter 0) | yes |
| file://:0:0:0:0 | declaration of 1st parameter | file://:0:0:0:0 | (unnamed parameter 0) | yes |
| file://:0:0:0:0 | declaration of (unnamed parameter 0) | file://:0:0:0:0 | (unnamed parameter 0) | yes |
| file://:0:0:0:0 | declaration of (unnamed parameter 0) | file://:0:0:0:0 | (unnamed parameter 0) | yes |
| file://:0:0:0:0 | definition of fp_offset | file://:0:0:0:0 | fp_offset | yes |
| file://:0:0:0:0 | definition of gp_offset | file://:0:0:0:0 | gp_offset | yes |
| file://:0:0:0:0 | definition of overflow_arg_area | file://:0:0:0:0 | overflow_arg_area | yes |

8
cpp/ql/test/library-tests/declarationEntry/more/declarationEntry.expected
View File

@@ -1,7 +1,7 @@
| file://:0:0:0:0 | declaration of 1st parameter |
| file://:0:0:0:0 | declaration of 1st parameter |
| file://:0:0:0:0 | declaration of 1st parameter |
| file://:0:0:0:0 | declaration of 1st parameter |
| file://:0:0:0:0 | declaration of (unnamed parameter 0) |
| file://:0:0:0:0 | declaration of (unnamed parameter 0) |
| file://:0:0:0:0 | declaration of (unnamed parameter 0) |
| file://:0:0:0:0 | declaration of (unnamed parameter 0) |
| file://:0:0:0:0 | definition of fp_offset |
| file://:0:0:0:0 | definition of gp_offset |
| file://:0:0:0:0 | definition of overflow_arg_area |

16
cpp/ql/test/library-tests/parameters/toStrings/params.expected
View File

@@ -1,11 +1,11 @@
| test.c:2:8:2:10 | declaration of 1st parameter |
| test.c:2:13:2:15 | declaration of 2nd parameter |
| test.c:2:18:2:20 | declaration of 3rd parameter |
| test.c:2:23:2:25 | declaration of 4th parameter |
| test.c:3:8:3:10 | declaration of y1 as anonymous 1st parameter |
| test.c:3:13:3:15 | declaration of y2 as anonymous 2nd parameter |
| test.c:3:18:3:20 | declaration of y3 as anonymous 3rd parameter |
| test.c:3:23:3:25 | declaration of y4 as anonymous 4th parameter |
| test.c:2:8:2:10 | declaration of (unnamed parameter 0) |
| test.c:2:13:2:15 | declaration of (unnamed parameter 1) |
| test.c:2:18:2:20 | declaration of (unnamed parameter 2) |
| test.c:2:23:2:25 | declaration of (unnamed parameter 3) |
| test.c:3:8:3:10 | declaration of y1 as (unnamed parameter 0) |
| test.c:3:13:3:15 | declaration of y2 as (unnamed parameter 1) |
| test.c:3:18:3:20 | declaration of y3 as (unnamed parameter 2) |
| test.c:3:23:3:25 | declaration of y4 as (unnamed parameter 3) |
| test.c:4:12:4:13 | declaration of x1 |
| test.c:4:20:4:21 | declaration of x2 |
| test.c:4:28:4:29 | declaration of x3 |

9
cpp/ql/test/library-tests/scopes/parents/parents.cpp
View File

@@ -14,6 +14,13 @@ namespace foo {
}
}
template<typename T>
T var = 42;
int g() {
requires(int l) { l; };
return var<int>;
}
// semmle-extractor-options: -std=c++20

7
cpp/ql/test/library-tests/scopes/parents/parents.expected
View File

@@ -1,5 +1,8 @@
| 0 | file://:0:0:0:0 | (global namespace) | file://:0:0:0:0 | __va_list_tag |
| 0 | file://:0:0:0:0 | (global namespace) | parents.cpp:2:11:2:13 | foo |
| 0 | file://:0:0:0:0 | (global namespace) | parents.cpp:18:3:18:3 | var |
| 0 | file://:0:0:0:0 | (global namespace) | parents.cpp:18:7:18:7 | var |
| 0 | file://:0:0:0:0 | (global namespace) | parents.cpp:20:5:20:5 | g |
| 1 | file://:0:0:0:0 | __va_list_tag | file://:0:0:0:0 | fp_offset |
| 1 | file://:0:0:0:0 | __va_list_tag | file://:0:0:0:0 | gp_offset |
| 1 | file://:0:0:0:0 | __va_list_tag | file://:0:0:0:0 | operator= |
@@ -14,7 +17,11 @@
| 1 | parents.cpp:4:10:4:10 | f | parents.cpp:4:19:13:5 | { ... } |
| 1 | parents.cpp:4:19:13:5 | { ... } | parents.cpp:5:11:5:11 | j |
| 1 | parents.cpp:4:19:13:5 | { ... } | parents.cpp:6:11:10:7 | { ... } |
| 1 | parents.cpp:4:19:13:5 | { ... } | parents.cpp:11:18:11:18 | e |
| 1 | parents.cpp:4:19:13:5 | { ... } | parents.cpp:11:21:12:7 | { ... } |
| 1 | parents.cpp:6:11:10:7 | { ... } | parents.cpp:7:9:9:9 | for(...;...;...) ... |
| 1 | parents.cpp:6:11:10:7 | { ... } | parents.cpp:7:33:9:9 | { ... } |
| 1 | parents.cpp:7:33:9:9 | { ... } | parents.cpp:8:15:8:15 | k |
| 1 | parents.cpp:18:7:18:7 | var | parents.cpp:17:19:17:19 | T |
| 1 | parents.cpp:20:5:20:5 | g | parents.cpp:20:9:24:1 | { ... } |
| 1 | parents.cpp:20:9:24:1 | { ... } | parents.cpp:21:16:21:16 | l |

4
cpp/ql/test/query-tests/Metrics/Dependencies/dependencies.expected
View File

@@ -1,5 +1,5 @@
| file://:0:0:0:0 | declaration of 1st parameter | LibB/libb_internal.h:5:8:5:12 | thing |
| file://:0:0:0:0 | declaration of 1st parameter | LibB/libb_internal.h:5:8:5:12 | thing |
| file://:0:0:0:0 | declaration of (unnamed parameter 0) | LibB/libb_internal.h:5:8:5:12 | thing |
| file://:0:0:0:0 | declaration of (unnamed parameter 0) | LibB/libb_internal.h:5:8:5:12 | thing |
| include.h:3:25:3:33 | num | LibD/libd.h:5:12:5:14 | num |
| main.cpp:8:31:8:31 | call to container | LibC/libc.h:9:3:9:3 | container |
| main.cpp:8:31:8:31 | definition of x | LibB/libb_internal.h:5:8:5:12 | thing |

2
docs/codeql/reusables/supported-versions-compilers.rst
View File

@@ -22,7 +22,7 @@
Eclipse compiler for Java (ECJ) [6]_",``.java``
Kotlin,"Kotlin 1.5.0 to 2.1.0\ *x*","kotlinc",``.kt``
JavaScript,ECMAScript 2022 or lower,Not applicable,"``.js``, ``.jsx``, ``.mjs``, ``.es``, ``.es6``, ``.htm``, ``.html``, ``.xhtm``, ``.xhtml``, ``.vue``, ``.hbs``, ``.ejs``, ``.njk``, ``.json``, ``.yaml``, ``.yml``, ``.raml``, ``.xml`` [7]_"
Python [8]_,"2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12",Not applicable,``.py``
Python [8]_,"2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13",Not applicable,``.py``
Ruby [9]_,"up to 3.3",Not applicable,"``.rb``, ``.erb``, ``.gemspec``, ``Gemfile``"
Swift [10]_,"Swift 5.4-5.10","Swift compiler","``.swift``"
TypeScript [11]_,"2.6-5.6",Standard TypeScript compiler,"``.ts``, ``.tsx``, ``.mts``, ``.cts``"

28
go/extractor/extractor.go
View File

@@ -518,6 +518,7 @@ func extractMethod(tw *trap.Writer, meth *types.Func) trap.Label {
// For more information on objects, see:
// https://github.com/golang/example/blob/master/gotypes/README.md#objects
func extractObject(tw *trap.Writer, obj types.Object, lbl trap.Label) {
checkObjectNotSpecialized(obj)
name := obj.Name()
isBuiltin := obj.Parent() == types.Universe
var kind int
@@ -1607,7 +1608,7 @@ func extractType(tw *trap.Writer, tp types.Type) trap.Label {
case *types.Struct:
kind = dbscheme.StructType.Index()
for i := 0; i < tp.NumFields(); i++ {
field := tp.Field(i)
field := tp.Field(i).Origin()
// ensure the field is associated with a label - note that
// struct fields do not have a parent scope, so they are not
@@ -1637,7 +1638,7 @@ func extractType(tw *trap.Writer, tp types.Type) trap.Label {
// Note that methods coming from embedded interfaces can be
// accessed through `Method(i)`, so there is no need to
// deal with them separately.
meth := tp.Method(i)
meth := tp.Method(i).Origin()
// Note that methods do not have a parent scope, so they are
// not dealt with by `extractScopes`
@@ -1707,7 +1708,7 @@ func extractType(tw *trap.Writer, tp types.Type) trap.Label {
// ensure all methods have labels - note that methods do not have a
// parent scope, so they are not dealt with by `extractScopes`
for i := 0; i < origintp.NumMethods(); i++ {
meth := origintp.Method(i)
meth := origintp.Method(i).Origin()
extractMethod(tw, meth)
}
@@ -1715,7 +1716,7 @@ func extractType(tw *trap.Writer, tp types.Type) trap.Label {
// associate all methods of underlying interface with this type
if underlyingInterface, ok := underlying.(*types.Interface); ok {
for i := 0; i < underlyingInterface.NumMethods(); i++ {
methlbl := extractMethod(tw, underlyingInterface.Method(i))
methlbl := extractMethod(tw, underlyingInterface.Method(i).Origin())
dbscheme.MethodHostsTable.Emit(tw, methlbl, lbl)
}
}
@@ -1787,7 +1788,7 @@ func getTypeLabel(tw *trap.Writer, tp types.Type) (trap.Label, bool) {
case *types.Interface:
var b strings.Builder
for i := 0; i < tp.NumMethods(); i++ {
meth := tp.Method(i)
meth := tp.Method(i).Origin()
methLbl := extractType(tw, meth.Type())
if i > 0 {
b.WriteString(",")
@@ -2143,3 +2144,20 @@ func skipExtractingValueForLeftOperand(tw *trap.Writer, be *ast.BinaryExpr) bool
}
return true
}
// checkObjectNotSpecialized exits the program if `obj` is specialized. Note
// that specialization is only possible for function objects and variable
// objects.
func checkObjectNotSpecialized(obj types.Object) {
if funcObj, ok := obj.(*types.Func); ok && funcObj != funcObj.Origin() {
log.Fatalf("Encountered unexpected specialization %s of generic function object %s", funcObj.FullName(), funcObj.Origin().FullName())
}
if varObj, ok := obj.(*types.Var); ok && varObj != varObj.Origin() {
log.Fatalf("Encountered unexpected specialization %s of generic variable object %s", varObj.String(), varObj.Origin().String())
}
if typeNameObj, ok := obj.(*types.TypeName); ok {
if namedType, ok := typeNameObj.Type().(*types.Named); ok && namedType != namedType.Origin() {
log.Fatalf("Encountered type object for specialization %s of named type %s", namedType.String(), namedType.Origin().String())
}
}
}

4
go/ql/lib/change-notes/2024-10-17-ast-viewer-type-param-decl.md Normal file
View File

@@ -0,0 +1,4 @@
---
category: minorAnalysis
---
* The AST viewer now shows type parameter declarations in the correct place in the AST.

2
go/ql/lib/semmle/go/AST.qll
View File

@@ -55,6 +55,8 @@ class AstNode extends @node, Locatable {
kind = "commentgroup" and result = this.(File).getCommentGroup(i)
or
kind = "comment" and result = this.(CommentGroup).getComment(i)
or
kind = "typeparamdecl" and result = this.(TypeParamDeclParent).getTypeParameterDecl(i)
}
/**

5
go/ql/lib/semmle/go/Decls.qll
View File

@@ -212,10 +212,7 @@ class MethodDecl extends FuncDecl {
*
* is `Rectangle`.
*/
NamedType getReceiverBaseType() {
result = this.getReceiverType() or
result = this.getReceiverType().(PointerType).getBaseType()
}
NamedType getReceiverBaseType() { result = lookThroughPointerType(this.getReceiverType()) }
/**
* Gets the receiver variable of this method.

8
go/ql/lib/semmle/go/Scopes.qll
View File

@@ -519,13 +519,7 @@ class Method extends Function {
* Gets the receiver base type of this method, that is, either the base type of the receiver type
* if it is a pointer type, or the receiver type itself if it is not a pointer type.
*/
Type getReceiverBaseType() {
exists(Type recv | recv = this.getReceiverType() |
if recv instanceof PointerType
then result = recv.(PointerType).getBaseType()
else result = recv
)
}
Type getReceiverBaseType() { result = lookThroughPointerType(this.getReceiverType()) }
/** Holds if this method has name `m` and belongs to the method set of type `tp` or `*tp`. */
private predicate isIn(NamedType tp, string m) {

20
go/ql/lib/semmle/go/Types.qll
View File

@@ -446,11 +446,7 @@ class StructType extends @structtype, CompositeType {
if n = ""
then (
isEmbedded = true and
(
name = tp.(NamedType).getName()
or
name = tp.(PointerType).getBaseType().(NamedType).getName()
)
name = lookThroughPointerType(tp).(NamedType).getName()
) else (
isEmbedded = false and
name = n
@@ -518,9 +514,7 @@ class StructType extends @structtype, CompositeType {
this.hasFieldCand(_, embeddedParent, depth - 1, true) and
result.getName() = name and
(
result.getReceiverBaseType() = embeddedParent.getType()
or
result.getReceiverBaseType() = embeddedParent.getType().(PointerType).getBaseType()
result.getReceiverBaseType() = lookThroughPointerType(embeddedParent.getType())
or
methodhosts(result, embeddedParent.getType())
)
@@ -644,6 +638,16 @@ class PointerType extends @pointertype, CompositeType {
override string toString() { result = "pointer type" }
}
/**
* Gets the base type if `t` is a pointer type, otherwise `t` itself.
*/
Type lookThroughPointerType(Type t) {
not t instanceof PointerType and
result = t
or
result = t.(PointerType).getBaseType()
}
private newtype TTypeSetTerm =
MkTypeSetTerm(TypeSetLiteralType tslit, int index) { component_types(tslit, index, _, _) }

6
go/ql/lib/semmle/go/controlflow/IR.qll
View File

@@ -358,11 +358,7 @@ module IR {
override predicate reads(ValueEntity v) { v = field }
override Type getResultType() {
if field.getType() instanceof PointerType
then result = field.getType().(PointerType).getBaseType()
else result = field.getType()
}
override Type getResultType() { result = lookThroughPointerType(field.getType()) }
override ControlFlow::Root getRoot() { result.isRootOf(e) }

2
go/ql/src/InconsistentCode/LengthComparisonOffByOne.ql
View File

@@ -73,7 +73,7 @@ predicate isRegexpMethodCall(DataFlow::MethodCallNode c) {
exists(NamedType regexp, Type recvtp |
regexp.getName() = "Regexp" and recvtp = c.getReceiver().getType()
|
recvtp = regexp or recvtp.(PointerType).getBaseType() = regexp
lookThroughPointerType(recvtp) = regexp
)
}

34
go/ql/test/library-tests/semmle/go/PrintAst/PrintAst.expected
View File

@@ -639,6 +639,11 @@ other.go:
# 11| Type = int
# 11| 0: [Ident, VariableName] myNested
# 11| Type = func() int
# 8| 3: [TypeParamDecl] type parameter declaration
# 8| 0: [Ident, TypeName] int
# 8| Type = int
# 8| 1: [Ident, TypeName] U
# 8| Type = U
# 15| 5: [VarDecl] variable declaration
# 15| 0: [ValueSpec] value declaration specifier
# 15| 0: [Ident, VariableName] x
@@ -648,3 +653,32 @@ other.go:
# 15| 2: [IntLit] 0
# 15| Type = int
# 15| Value = [IntLit] 0
# 17| 6: [TypeDecl] type declaration
# 17| 0: [TypeSpec] type declaration specifier
# 17| 0: [Ident, TypeName] myType
# 17| Type = myType
# 17| 1: [ArrayTypeExpr] array type
# 17| Type = []T
# 17| 0: [Ident, TypeName] T
# 17| Type = T
# 17| 2: [TypeParamDecl] type parameter declaration
# 17| 0: [TypeSetLiteralExpr] type set literal
# 17| Type = ~string
# 17| 0: [Ident, TypeName] string
# 17| Type = string
# 17| 1: [Ident, TypeName] T
# 17| Type = T
# 19| 7: [MethodDecl] function declaration
# 19| 0: [FunctionName, Ident] f
# 19| Type = func()
# 19| 1: [FuncTypeExpr] function type
# 19| 2: [ReceiverDecl] receiver declaration
# 19| 0: [GenericTypeInstantiationExpr] generic type instantiation expression
# 19| Type = myType
# 19| 0: [Ident, TypeName] myType
# 19| Type = myType
# 19| 1: [Ident, TypeName] U
# 19| Type = U
# 19| 1: [Ident, VariableName] m
# 19| Type = myType
# 19| 3: [BlockStmt] block statement

34
go/ql/test/library-tests/semmle/go/PrintAst/PrintAstExcludeComments.expected
View File

@@ -619,6 +619,11 @@ other.go:
# 11| Type = int
# 11| 0: [Ident, VariableName] myNested
# 11| Type = func() int
# 8| 3: [TypeParamDecl] type parameter declaration
# 8| 0: [Ident, TypeName] int
# 8| Type = int
# 8| 1: [Ident, TypeName] U
# 8| Type = U
# 15| 5: [VarDecl] variable declaration
# 15| 0: [ValueSpec] value declaration specifier
# 15| 0: [Ident, VariableName] x
@@ -628,3 +633,32 @@ other.go:
# 15| 2: [IntLit] 0
# 15| Type = int
# 15| Value = [IntLit] 0
# 17| 6: [TypeDecl] type declaration
# 17| 0: [TypeSpec] type declaration specifier
# 17| 0: [Ident, TypeName] myType
# 17| Type = myType
# 17| 1: [ArrayTypeExpr] array type
# 17| Type = []T
# 17| 0: [Ident, TypeName] T
# 17| Type = T
# 17| 2: [TypeParamDecl] type parameter declaration
# 17| 0: [TypeSetLiteralExpr] type set literal
# 17| Type = ~string
# 17| 0: [Ident, TypeName] string
# 17| Type = string
# 17| 1: [Ident, TypeName] T
# 17| Type = T
# 19| 7: [MethodDecl] function declaration
# 19| 0: [FunctionName, Ident] f
# 19| Type = func()
# 19| 1: [FuncTypeExpr] function type
# 19| 2: [ReceiverDecl] receiver declaration
# 19| 0: [GenericTypeInstantiationExpr] generic type instantiation expression
# 19| Type = myType
# 19| 0: [Ident, TypeName] myType
# 19| Type = myType
# 19| 1: [Ident, TypeName] U
# 19| Type = U
# 19| 1: [Ident, VariableName] m
# 19| Type = myType
# 19| 3: [BlockStmt] block statement

20
go/ql/test/library-tests/semmle/go/PrintAst/PrintAstNestedFunction.expected
View File

@@ -56,6 +56,11 @@ other.go:
# 11| Type = int
# 11| 0: [Ident, VariableName] myNested
# 11| Type = func() int
# 8| 3: [TypeParamDecl] type parameter declaration
# 8| 0: [Ident, TypeName] int
# 8| Type = int
# 8| 1: [Ident, TypeName] U
# 8| Type = U
# 15| 2: [VarDecl] variable declaration
# 15| 0: [ValueSpec] value declaration specifier
# 15| 0: [Ident, VariableName] x
@@ -65,3 +70,18 @@ other.go:
# 15| 2: [IntLit] 0
# 15| Type = int
# 15| Value = [IntLit] 0
# 17| 3: [TypeDecl] type declaration
# 17| 0: [TypeSpec] type declaration specifier
# 17| 0: [Ident, TypeName] myType
# 17| Type = myType
# 17| 1: [ArrayTypeExpr] array type
# 17| Type = []T
# 17| 0: [Ident, TypeName] T
# 17| Type = T
# 17| 2: [TypeParamDecl] type parameter declaration
# 17| 0: [TypeSetLiteralExpr] type set literal
# 17| Type = ~string
# 17| 0: [Ident, TypeName] string
# 17| Type = string
# 17| 1: [Ident, TypeName] T
# 17| Type = T

34
go/ql/test/library-tests/semmle/go/PrintAst/PrintAstRestrictFile.expected
View File

@@ -41,6 +41,11 @@ other.go:
# 11| Type = int
# 11| 0: [Ident, VariableName] myNested
# 11| Type = func() int
# 8| 3: [TypeParamDecl] type parameter declaration
# 8| 0: [Ident, TypeName] int
# 8| Type = int
# 8| 1: [Ident, TypeName] U
# 8| Type = U
# 15| 5: [VarDecl] variable declaration
# 15| 0: [ValueSpec] value declaration specifier
# 15| 0: [Ident, VariableName] x
@@ -50,3 +55,32 @@ other.go:
# 15| 2: [IntLit] 0
# 15| Type = int
# 15| Value = [IntLit] 0
# 17| 6: [TypeDecl] type declaration
# 17| 0: [TypeSpec] type declaration specifier
# 17| 0: [Ident, TypeName] myType
# 17| Type = myType
# 17| 1: [ArrayTypeExpr] array type
# 17| Type = []T
# 17| 0: [Ident, TypeName] T
# 17| Type = T
# 17| 2: [TypeParamDecl] type parameter declaration
# 17| 0: [TypeSetLiteralExpr] type set literal
# 17| Type = ~string
# 17| 0: [Ident, TypeName] string
# 17| Type = string
# 17| 1: [Ident, TypeName] T
# 17| Type = T
# 19| 7: [MethodDecl] function declaration
# 19| 0: [FunctionName, Ident] f
# 19| Type = func()
# 19| 1: [FuncTypeExpr] function type
# 19| 2: [ReceiverDecl] receiver declaration
# 19| 0: [GenericTypeInstantiationExpr] generic type instantiation expression
# 19| Type = myType
# 19| 0: [Ident, TypeName] myType
# 19| Type = myType
# 19| 1: [Ident, TypeName] U
# 19| Type = U
# 19| 1: [Ident, VariableName] m
# 19| Type = myType
# 19| 3: [BlockStmt] block statement

21
go/ql/test/library-tests/semmle/go/PrintAst/PrintAstRestrictFunction.expected
View File

@@ -1,3 +1,9 @@
other.go:
# 8| [TypeParamDecl] type parameter declaration
# 8| 0: [Ident, TypeName] int
# 8| Type = int
# 8| 1: [Ident, TypeName] U
# 8| Type = U
go.mod:
# 0| [GoModFile] go.mod
# 1| 0: [GoModModuleLine] go.mod module line
@@ -45,3 +51,18 @@ other.go:
# 15| 2: [IntLit] 0
# 15| Type = int
# 15| Value = [IntLit] 0
# 17| 3: [TypeDecl] type declaration
# 17| 0: [TypeSpec] type declaration specifier
# 17| 0: [Ident, TypeName] myType
# 17| Type = myType
# 17| 1: [ArrayTypeExpr] array type
# 17| Type = []T
# 17| 0: [Ident, TypeName] T
# 17| Type = T
# 17| 2: [TypeParamDecl] type parameter declaration
# 17| 0: [TypeSetLiteralExpr] type set literal
# 17| Type = ~string
# 17| 0: [Ident, TypeName] string
# 17| Type = string
# 17| 1: [Ident, TypeName] T
# 17| Type = T

3
go/ql/test/library-tests/semmle/go/PrintAst/go.mod
View File

@@ -1,4 +1,3 @@
module codeql-go-tests/printast
go 1.14
go 1.18

6
go/ql/test/library-tests/semmle/go/PrintAst/other.go
View File

@@ -5,7 +5,7 @@ func main() {}
func f() {}
func g() {}
func hasNested() {
func hasNested[U int]() {
myNested := func() int { return 1 }
myNested()
@@ -13,3 +13,7 @@ func hasNested() {
}
var x int = 0
type myType[T ~string] []T
func (m myType[U]) f() {}

10
java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt
View File

@@ -169,11 +169,17 @@ open class KotlinFileExtractor(
}
private fun FunctionDescriptor.tryIsHiddenToOvercomeSignatureClash(d: IrFunction): Boolean {
// `org.jetbrains.kotlin.ir.descriptors.IrBasedClassConstructorDescriptor.isHiddenToOvercomeSignatureClash`
// throws one exception or other in Kotlin 2, depending on the version.
// TODO: We need a replacement for this for Kotlin 2
try {
return this.isHiddenToOvercomeSignatureClash
} catch (e: NotImplementedError) {
// `org.jetbrains.kotlin.ir.descriptors.IrBasedClassConstructorDescriptor.isHiddenToOvercomeSignatureClash` throws the exception
// TODO: We need a replacement for this for Kotlin 2
if (!usesK2) {
logger.warnElement("Couldn't query if element is fake, deciding it's not.", d, e)
}
return false
} catch (e: IllegalStateException) {
if (!usesK2) {
logger.warnElement("Couldn't query if element is fake, deciding it's not.", d, e)
}

24
misc/bazel/internal/git_lfs_probe.py
View File

@@ -179,15 +179,18 @@ def get_locations(objects):
try:
with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
data = json.load(resp)
assert len(data["objects"]) == len(
indexes
), f"received {len(data)} objects, expected {len(indexes)}"
for i, resp in zip(indexes, data["objects"]):
ret[i] = f'{resp["oid"]} {resp["actions"]["download"]["href"]}'
return ret
except urllib.error.URLError as e:
warn(f"encountered {type(e).__name__} {e}, ignoring endpoint {endpoint.name}")
continue
assert len(data["objects"]) == len(
indexes
), f"received {len(data)} objects, expected {len(indexes)}"
for i, resp in zip(indexes, data["objects"]):
ret[i] = f'{resp["oid"]} {resp["actions"]["download"]["href"]}'
return ret
except KeyError:
warn(f"encountered malformed response, ignoring endpoint {endpoint.name}:\n{json.dumps(data, indent=2)}")
continue
raise NoEndpointsFound
@@ -210,5 +213,12 @@ try:
for resp in get_locations(objects):
print(resp)
except NoEndpointsFound as e:
print(f"ERROR: no valid endpoints found", file=sys.stderr)
print("""\
ERROR: no valid endpoints found, your git authentication method might be currently unsupported by this script.
You can bypass this error by running from semmle-code (this might take a while):
git config lfs.fetchexclude ""
git -C ql config lfs.fetchinclude \\*
git lfs fetch && git lfs checkout
cd ql
git lfs fetch && git lfs checkout""", file=sys.stderr)
sys.exit(1)

1236
python/downgrades/5af903da088e3746aa283700a43a779302453523/old.dbscheme Normal file
View File

File diff suppressed because it is too large Load Diff

42
python/downgrades/5af903da088e3746aa283700a43a779302453523/py_exprs.ql Normal file
View File

@@ -0,0 +1,42 @@
// We must wrap the DB types, as these cannot appear in argument lists
class TypeParameter_ extends @py_type_parameter {
string toString() { result = "TypeParameter" }
}
class Expr_ extends @py_expr {
string toString() { result = "Expr" }
}
class ExprParent_ extends @py_expr_parent {
string toString() { result = "ExprParent" }
}
class TypeVar_ extends @py_TypeVar, TypeParameter_ {
override string toString() { result = "TypeVar" }
}
class TypeVarTuple_ extends @py_TypeVarTuple, TypeParameter_ {
override string toString() { result = "TypeVarTuple" }
}
class ParamSpec_ extends @py_ParamSpec, TypeParameter_ {
override string toString() { result = "ParamSpec" }
}
// From the dbscheme:
// py_exprs(unique int id : @py_expr,
// int kind: int ref,
// int parent : @py_expr_parent ref,
// int idx : int ref);
query predicate py_exprs_without_type_parameter_defaults(
Expr_ id, int kind, ExprParent_ parent, int idx
) {
py_exprs(id, kind, parent, idx) and
// From the dbscheme
// /* <Field> ParamSpec.default = 2, expr */
// /* <Field> TypeVar.default = 3, expr */
// /* <Field> TypeVarTuple.default = 2, expr */
(parent instanceof ParamSpec_ implies idx != 2) and
(parent instanceof TypeVar_ implies idx != 3) and
(parent instanceof TypeVarTuple_ implies idx != 2)
}

1233
python/downgrades/5af903da088e3746aa283700a43a779302453523/semmlecode.python.dbscheme Normal file
View File

File diff suppressed because it is too large Load Diff

3
python/downgrades/5af903da088e3746aa283700a43a779302453523/upgrade.properties Normal file
View File

@@ -0,0 +1,3 @@
description: Remove support for type parameter defaults.
compatibility: backwards
py_exprs.rel: run py_exprs.qlo py_exprs_without_type_parameter_defaults

6
python/extractor/Makefile
View File

@@ -21,19 +21,19 @@ $(TOKENIZER_FILE): $(TOKENIZER_DEPS)
MASTER_FILE = semmle/python/master.py
DBSCHEME_FILE = $(GIT_ROOT)/ql/python/ql/lib/semmlecode.python.dbscheme
DBSCHEME_FILE = $(GIT_ROOT)/python/ql/lib/semmlecode.python.dbscheme
.PHONY: dbscheme
dbscheme: $(MASTER_FILE)
python3 -m semmle.dbscheme_gen $(DBSCHEME_FILE)
AST_GENERATED_DIR = $(GIT_ROOT)/ql/python/ql/lib/semmle/python/
AST_GENERATED_DIR = $(GIT_ROOT)/python/ql/lib/semmle/python/
AST_GENERATED_FILE = $(AST_GENERATED_DIR)AstGenerated.qll
.PHONY: ast
ast: $(MASTER_FILE)
python3 -m semmle.query_gen $(AST_GENERATED_DIR)
$(GIT_ROOT)/target/intree/codeql/codeql query format --in-place $(AST_GENERATED_FILE)
codeql query format --in-place $(AST_GENERATED_FILE)
################################################################################
# Tests

15
python/extractor/semmle/python/ast.py
View File

@@ -500,10 +500,11 @@ class Num(expr):
self.text = text
class ParamSpec(type_parameter):
__slots__ = "name",
__slots__ = "name", "default",
def __init__(self, name):
def __init__(self, name, default):
self.name = name
self.default = default
@@ -607,17 +608,19 @@ class TypeAlias(stmt):
self.value = value
class TypeVar(type_parameter):
__slots__ = "name", "bound",
__slots__ = "name", "bound", "default"
def __init__(self, name, bound):
def __init__(self, name, bound, default):
self.name = name
self.bound = bound
self.default = default
class TypeVarTuple(type_parameter):
__slots__ = "name",
__slots__ = "name", "default",
def __init__(self, name):
def __init__(self, name, default):
self.name = name
self.default = default
class UnaryOp(expr):
__slots__ = "op", "operand",

3
python/extractor/semmle/python/master.py
View File

@@ -397,6 +397,7 @@ Num.field('n', number, 'value')
Num.field('text', number)
ParamSpec.field('name', expr)
ParamSpec.field('default', expr)
Print.field('dest', expr, 'destination')
Print.field('values', expr_list)
@@ -448,8 +449,10 @@ TypeAlias.field('value', expr)
TypeVar.field('name', expr)
TypeVar.field('bound', expr)
TypeVar.field('default', expr)
TypeVarTuple.field('name', expr)
TypeVarTuple.field('default', expr)
UnaryOp.field('op', unaryop, 'operator')
UnaryOp.field('operand', expr)

2
python/extractor/semmle/util.py
View File

@@ -10,7 +10,7 @@ from io import BytesIO
#Semantic version of extractor.
#Update this if any changes are made
VERSION = "7.0.0"
VERSION = "7.1.0"
PY_EXTENSIONS = ".py", ".pyw"

294
python/extractor/tests/parser/types_new.expected
View File

@@ -1,4 +1,4 @@
Module: [1, 0] - [6, 0]
Module: [1, 0] - [23, 0]
body: [
TypeAlias: [1, 0] - [1, 34]
name:
@@ -12,6 +12,7 @@ Module: [1, 0] - [6, 0]
variable: Variable('T1', None)
ctx: Store
bound: None
default: None
TypeVar: [1, 11] - [1, 17]
name:
Name: [1, 11] - [1, 13]
@@ -21,16 +22,19 @@ Module: [1, 0] - [6, 0]
Name: [1, 15] - [1, 17]
variable: Variable('E1', None)
ctx: Load
default: None
TypeVarTuple: [1, 19] - [1, 22]
name:
Name: [1, 20] - [1, 22]
variable: Variable('T3', None)
ctx: Store
default: None
ParamSpec: [1, 24] - [1, 28]
name:
Name: [1, 26] - [1, 28]
variable: Variable('T4', None)
ctx: Store
default: None
]
value:
Name: [1, 32] - [1, 34]
@@ -64,6 +68,7 @@ Module: [1, 0] - [6, 0]
variable: Variable('T6', None)
ctx: Store
bound: None
default: None
TypeVar: [3, 10] - [3, 16]
name:
Name: [3, 10] - [3, 12]
@@ -73,16 +78,19 @@ Module: [1, 0] - [6, 0]
Name: [3, 14] - [3, 16]
variable: Variable('E2', None)
ctx: Load
default: None
TypeVarTuple: [3, 18] - [3, 21]
name:
Name: [3, 19] - [3, 21]
variable: Variable('T8', None)
ctx: Store
default: None
ParamSpec: [3, 23] - [3, 27]
name:
Name: [3, 25] - [3, 27]
variable: Variable('T9', None)
ctx: Store
default: None
]
args: []
vararg: None
@@ -109,6 +117,7 @@ Module: [1, 0] - [6, 0]
variable: Variable('T10', None)
ctx: Store
bound: None
default: None
TypeVar: [5, 13] - [5, 20]
name:
Name: [5, 13] - [5, 16]
@@ -118,16 +127,19 @@ Module: [1, 0] - [6, 0]
Name: [5, 18] - [5, 20]
variable: Variable('E3', None)
ctx: Load
default: None
TypeVarTuple: [5, 22] - [5, 26]
name:
Name: [5, 23] - [5, 26]
variable: Variable('T12', None)
ctx: Store
default: None
ParamSpec: [5, 28] - [5, 33]
name:
Name: [5, 30] - [5, 33]
variable: Variable('T13', None)
ctx: Store
default: None
]
bases: []
keywords: []
@@ -139,4 +151,284 @@ Module: [1, 0] - [6, 0]
value:
Ellipsis: [5, 36] - [5, 39]
]
Assign: [10, 0] - [10, 22]
targets: [
Name: [10, 6] - [10, 10]
variable: Variable('Foo1', None)
ctx: Store
]
value:
ClassExpr: [10, 0] - [10, 22]
name: 'Foo1'
type_parameters: [
TypeVar: [10, 11] - [10, 20]
name:
Name: [10, 11] - [10, 14]
variable: Variable('T14', None)
ctx: Store
bound: None
default:
Name: [10, 17] - [10, 20]
variable: Variable('str', None)
ctx: Load
]
bases: []
keywords: []
inner_scope:
Class: [10, 0] - [10, 22]
name: 'Foo1'
body: [
Expr: [10, 23] - [10, 26]
value:
Ellipsis: [10, 23] - [10, 26]
]
Assign: [13, 0] - [13, 30]
targets: [
Name: [13, 6] - [13, 10]
variable: Variable('Baz1', None)
ctx: Store
]
value:
ClassExpr: [13, 0] - [13, 30]
name: 'Baz1'
type_parameters: [
ParamSpec: [13, 11] - [13, 28]
name:
Name: [13, 13] - [13, 15]
variable: Variable('P1', None)
ctx: Store
default:
List: [13, 18] - [13, 28]
elts: [
Name: [13, 19] - [13, 22]
variable: Variable('int', None)
ctx: Load
Name: [13, 24] - [13, 27]
variable: Variable('str', None)
ctx: Load
]
ctx: Load
]
bases: []
keywords: []
inner_scope:
Class: [13, 0] - [13, 30]
name: 'Baz1'
body: [
Expr: [13, 31] - [13, 34]
value:
Ellipsis: [13, 31] - [13, 34]
]
Assign: [16, 0] - [16, 37]
targets: [
Name: [16, 6] - [16, 10]
variable: Variable('Qux1', None)
ctx: Store
]
value:
ClassExpr: [16, 0] - [16, 37]
name: 'Qux1'
type_parameters: [
TypeVarTuple: [16, 11] - [16, 35]
name:
Name: [16, 12] - [16, 15]
variable: Variable('Ts1', None)
ctx: Store
default:
Starred: [16, 18] - [16, 35]
value:
Subscript: [16, 19] - [16, 35]
value:
Name: [16, 19] - [16, 24]
variable: Variable('tuple', None)
ctx: Load
index:
Tuple: [16, 25] - [16, 34]
elts: [
Name: [16, 25] - [16, 28]
variable: Variable('int', None)
ctx: Load
Name: [16, 30] - [16, 34]
variable: Variable('bool', None)
ctx: Load
]
ctx: Load
ctx: Load
ctx: Load
]
bases: []
keywords: []
inner_scope:
Class: [16, 0] - [16, 37]
name: 'Qux1'
body: [
Expr: [16, 38] - [16, 41]
value:
Ellipsis: [16, 38] - [16, 41]
]
TypeAlias: [19, 0] - [19, 40]
name:
Name: [19, 5] - [19, 9]
variable: Variable('Foo2', None)
ctx: Store
type_parameters: [
TypeVar: [19, 10] - [19, 13]
name:
Name: [19, 10] - [19, 13]
variable: Variable('T15', None)
ctx: Store
bound: None
default: None
TypeVar: [19, 15] - [19, 23]
name:
Name: [19, 15] - [19, 17]
variable: Variable('U1', None)
ctx: Store
bound: None
default:
Name: [19, 20] - [19, 23]
variable: Variable('str', None)
ctx: Load
]
value:
Subscript: [19, 27] - [19, 40]
value:
Name: [19, 27] - [19, 31]
variable: Variable('Bar1', None)
ctx: Load
index:
Tuple: [19, 32] - [19, 39]
elts: [
Name: [19, 32] - [19, 35]
variable: Variable('T15', None)
ctx: Load
Name: [19, 37] - [19, 39]
variable: Variable('U1', None)
ctx: Load
]
ctx: Load
ctx: Load
TypeAlias: [20, 0] - [20, 41]
name:
Name: [20, 5] - [20, 9]
variable: Variable('Baz2', None)
ctx: Store
type_parameters: [
ParamSpec: [20, 10] - [20, 27]
name:
Name: [20, 12] - [20, 14]
variable: Variable('P2', None)
ctx: Store
default:
List: [20, 17] - [20, 27]
elts: [
Name: [20, 18] - [20, 21]
variable: Variable('int', None)
ctx: Load
Name: [20, 23] - [20, 26]
variable: Variable('str', None)
ctx: Load
]
ctx: Load
]
value:
Subscript: [20, 31] - [20, 41]
value:
Name: [20, 31] - [20, 35]
variable: Variable('Spam', None)
ctx: Load
index:
BinOp: [20, 36] - [20, 40]
left:
Name: [20, 36] - [20, 36]
variable: Variable('', None)
ctx: Load
op: Pow
right:
Name: [20, 38] - [20, 40]
variable: Variable('P2', None)
ctx: Load
ctx: Load
TypeAlias: [21, 0] - [21, 41]
name:
Name: [21, 5] - [21, 9]
variable: Variable('Qux2', None)
ctx: Store
type_parameters: [
TypeVarTuple: [21, 10] - [21, 28]
name:
Name: [21, 11] - [21, 14]
variable: Variable('Ts2', None)
ctx: Store
default:
Starred: [21, 17] - [21, 28]
value:
Subscript: [21, 18] - [21, 28]
value:
Name: [21, 18] - [21, 23]
variable: Variable('tuple', None)
ctx: Load
index:
Name: [21, 24] - [21, 27]
variable: Variable('str', None)
ctx: Load
ctx: Load
ctx: Load
]
value:
Subscript: [21, 32] - [21, 41]
value:
Name: [21, 32] - [21, 35]
variable: Variable('Ham', None)
ctx: Load
index:
Starred: [21, 36] - [21, 40]
value:
Name: [21, 37] - [21, 40]
variable: Variable('Ts2', None)
ctx: Load
ctx: Load
ctx: Load
TypeAlias: [22, 0] - [22, 39]
name:
Name: [22, 5] - [22, 8]
variable: Variable('Rab', None)
ctx: Store
type_parameters: [
TypeVar: [22, 9] - [22, 11]
name:
Name: [22, 9] - [22, 11]
variable: Variable('U2', None)
ctx: Store
bound: None
default: None
TypeVar: [22, 13] - [22, 22]
name:
Name: [22, 13] - [22, 16]
variable: Variable('T15', None)
ctx: Store
bound: None
default:
Name: [22, 19] - [22, 22]
variable: Variable('str', None)
ctx: Load
]
value:
Subscript: [22, 26] - [22, 39]
value:
Name: [22, 26] - [22, 30]
variable: Variable('Bar2', None)
ctx: Load
index:
Tuple: [22, 31] - [22, 38]
elts: [
Name: [22, 31] - [22, 34]
variable: Variable('T15', None)
ctx: Load
Name: [22, 36] - [22, 38]
variable: Variable('U2', None)
ctx: Load
]
ctx: Load
ctx: Load
]

17
python/extractor/tests/parser/types_new.py
View File

@@ -3,3 +3,20 @@ type T[T1, T2: E1, *T3, **T4] = T5
def f[T6, T7: E2, *T8, **T9](): ...
class C[T10, T11: E3, *T12, **T13]: ...
# From PEP-696 (https://peps.python.org/pep-0696/#grammar-changes):
# TypeVars
class Foo1[T14 = str]: ...
# ParamSpecs
class Baz1[**P1 = [int, str]]: ...
# TypeVarTuples
class Qux1[*Ts1 = *tuple[int, bool]]: ...
# TypeAliases
type Foo2[T15, U1 = str] = Bar1[T15, U1]
type Baz2[**P2 = [int, str]] = Spam[**P2]
type Qux2[*Ts2 = *tuple[str]] = Ham[*Ts2]
type Rab[U2, T15 = str] = Bar2[T15, U2]

15
python/extractor/tsg-python/python.tsg
View File

@@ -3388,6 +3388,7 @@
(typevar_parameter
name: (_) @name
bound: (_)? @bound
default: (_)? @default
) @typevar
{
attr (@name.node) ctx = "store"
@@ -3396,22 +3397,36 @@
attr (@bound.node) ctx = "load"
attr (@typevar.node) bound = @bound.node
}
if some @default {
attr (@default.node) ctx = "load"
attr (@typevar.node) default = @default.node
}
}
(typevartuple_parameter
name: (_) @name
default: (_)? @default
) @typevartuple
{
attr (@name.node) ctx = "store"
attr (@typevartuple.node) name = @name.node
if some @default {
attr (@default.node) ctx = "load"
attr (@typevartuple.node) default = @default.node
}
}
(paramspec_parameter
name: (_) @name
default: (_)? @default
) @paramspec
{
attr (@name.node) ctx = "store"
attr (@paramspec.node) name = @name.node
if some @default {
attr (@default.node) ctx = "load"
attr (@paramspec.node) default = @default.node
}
}
;;;;;; End of Type parameters (`T: ..., *T, **T`)

12
python/extractor/tsg-python/tsp/grammar.js
View File

@@ -589,17 +589,20 @@ module.exports = grammar({
typevar_parameter: $ => seq(
field('name', $.identifier),
optional($._type_bound)
optional($._type_bound),
optional($._type_param_default)
),
typevartuple_parameter: $ => seq(
'*',
field('name', $.identifier),
optional($._type_param_default)
),
paramspec_parameter: $ => seq(
'**',
field('name', $.identifier),
optional($._type_param_default),
),
_type_parameter: $ => choice(
@@ -608,6 +611,11 @@ module.exports = grammar({
$.paramspec_parameter,
),
_type_param_default: $ => seq(
'=',
field('default', choice($.list_splat, $.expression))
),
parenthesized_list_splat: $ => prec(PREC.parenthesized_list_splat, seq(
'(',
choice(
@@ -923,7 +931,7 @@ module.exports = grammar({
subscript: $ => prec(PREC.call, seq(
field('value', $.primary_expression),
'[',
commaSep1(field('subscript', choice($.expression, $.slice))),
commaSep1(field('subscript', choice($.list_splat, $.expression, $.slice))),
optional(','),
']'
)),

72
python/extractor/tsg-python/tsp/src/grammar.json
View File

@@ -1,4 +1,5 @@
{
"$schema": "https://tree-sitter.github.io/tree-sitter/assets/schemas/grammar.schema.json",
"name": "python",
"word": "identifier",
"rules": {
@@ -2970,6 +2971,18 @@
"type": "BLANK"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_type_param_default"
},
{
"type": "BLANK"
}
]
}
]
},
@@ -2987,6 +3000,18 @@
"type": "SYMBOL",
"name": "identifier"
}
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_type_param_default"
},
{
"type": "BLANK"
}
]
}
]
},
@@ -3004,6 +3029,18 @@
"type": "SYMBOL",
"name": "identifier"
}
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_type_param_default"
},
{
"type": "BLANK"
}
]
}
]
},
@@ -3024,6 +3061,32 @@
}
]
},
"_type_param_default": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "="
},
{
"type": "FIELD",
"name": "default",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "list_splat"
},
{
"type": "SYMBOL",
"name": "expression"
}
]
}
}
]
},
"parenthesized_list_splat": {
"type": "PREC",
"value": 1,
@@ -5006,6 +5069,10 @@
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "list_splat"
},
{
"type": "SYMBOL",
"name": "expression"
@@ -5032,6 +5099,10 @@
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "list_splat"
},
{
"type": "SYMBOL",
"name": "expression"
@@ -6612,4 +6683,3 @@
"parameter"
]
}

55
python/extractor/tsg-python/tsp/src/node-types.json
View File

@@ -2691,6 +2691,7 @@
{
"type": "module",
"named": true,
"root": true,
"fields": {},
"children": {
"multiple": true,
@@ -2809,6 +2810,20 @@
"type": "paramspec_parameter",
"named": true,
"fields": {
"default": {
"multiple": false,
"required": false,
"types": [
{
"type": "expression",
"named": true
},
{
"type": "list_splat",
"named": true
}
]
},
"name": {
"multiple": false,
"required": true,
@@ -3193,6 +3208,10 @@
"type": "expression",
"named": true
},
{
"type": "list_splat",
"named": true
},
{
"type": "slice",
"named": true
@@ -3476,6 +3495,20 @@
}
]
},
"default": {
"multiple": false,
"required": false,
"types": [
{
"type": "expression",
"named": true
},
{
"type": "list_splat",
"named": true
}
]
},
"name": {
"multiple": false,
"required": true,
@@ -3492,6 +3525,20 @@
"type": "typevartuple_parameter",
"named": true,
"fields": {
"default": {
"multiple": false,
"required": false,
"types": [
{
"type": "expression",
"named": true
},
{
"type": "list_splat",
"named": true
}
]
},
"name": {
"multiple": false,
"required": true,
@@ -3765,6 +3812,10 @@
"type": ":=",
"named": false
},
{
"type": ";",
"named": false
},
{
"type": "<",
"named": false
@@ -3821,6 +3872,10 @@
"type": "[",
"named": false
},
{
"type": "\\",
"named": false
},
{
"type": "]",
"named": false

100738
python/extractor/tsg-python/tsp/src/parser.c
View File

File diff suppressed because it is too large Load Diff

54
python/extractor/tsg-python/tsp/src/tree_sitter/alloc.h Normal file
View File

@@ -0,0 +1,54 @@
#ifndef TREE_SITTER_ALLOC_H_
#define TREE_SITTER_ALLOC_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
// Allow clients to override allocation functions
#ifdef TREE_SITTER_REUSE_ALLOCATOR
extern void *(*ts_current_malloc)(size_t size);
extern void *(*ts_current_calloc)(size_t count, size_t size);
extern void *(*ts_current_realloc)(void *ptr, size_t size);
extern void (*ts_current_free)(void *ptr);
#ifndef ts_malloc
#define ts_malloc ts_current_malloc
#endif
#ifndef ts_calloc
#define ts_calloc ts_current_calloc
#endif
#ifndef ts_realloc
#define ts_realloc ts_current_realloc
#endif
#ifndef ts_free
#define ts_free ts_current_free
#endif
#else
#ifndef ts_malloc
#define ts_malloc malloc
#endif
#ifndef ts_calloc
#define ts_calloc calloc
#endif
#ifndef ts_realloc
#define ts_realloc realloc
#endif
#ifndef ts_free
#define ts_free free
#endif
#endif
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_ALLOC_H_

290
python/extractor/tsg-python/tsp/src/tree_sitter/array.h Normal file
View File

@@ -0,0 +1,290 @@
#ifndef TREE_SITTER_ARRAY_H_
#define TREE_SITTER_ARRAY_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "./alloc.h"
#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#ifdef _MSC_VER
#pragma warning(disable : 4101)
#elif defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
#endif
#define Array(T) \
struct { \
T *contents; \
uint32_t size; \
uint32_t capacity; \
}
/// Initialize an array.
#define array_init(self) \
((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
/// Create an empty array.
#define array_new() \
{ NULL, 0, 0 }
/// Get a pointer to the element at a given `index` in the array.
#define array_get(self, _index) \
(assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
/// Get a pointer to the first element in the array.
#define array_front(self) array_get(self, 0)
/// Get a pointer to the last element in the array.
#define array_back(self) array_get(self, (self)->size - 1)
/// Clear the array, setting its size to zero. Note that this does not free any
/// memory allocated for the array's contents.
#define array_clear(self) ((self)->size = 0)
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
/// less than the array's current capacity, this function has no effect.
#define array_reserve(self, new_capacity) \
_array__reserve((Array *)(self), array_elem_size(self), new_capacity)
/// Free any memory allocated for this array. Note that this does not free any
/// memory allocated for the array's contents.
#define array_delete(self) _array__delete((Array *)(self))
/// Push a new `element` onto the end of the array.
#define array_push(self, element) \
(_array__grow((Array *)(self), 1, array_elem_size(self)), \
(self)->contents[(self)->size++] = (element))
/// Increase the array's size by `count` elements.
/// New elements are zero-initialized.
#define array_grow_by(self, count) \
do { \
if ((count) == 0) break; \
_array__grow((Array *)(self), count, array_elem_size(self)); \
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
(self)->size += (count); \
} while (0)
/// Append all elements from one array to the end of another.
#define array_push_all(self, other) \
array_extend((self), (other)->size, (other)->contents)
/// Append `count` elements to the end of the array, reading their values from the
/// `contents` pointer.
#define array_extend(self, count, contents) \
_array__splice( \
(Array *)(self), array_elem_size(self), (self)->size, \
0, count, contents \
)
/// Remove `old_count` elements from the array starting at the given `index`. At
/// the same index, insert `new_count` new elements, reading their values from the
/// `new_contents` pointer.
#define array_splice(self, _index, old_count, new_count, new_contents) \
_array__splice( \
(Array *)(self), array_elem_size(self), _index, \
old_count, new_count, new_contents \
)
/// Insert one `element` into the array at the given `index`.
#define array_insert(self, _index, element) \
_array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
/// Remove one element from the array at the given `index`.
#define array_erase(self, _index) \
_array__erase((Array *)(self), array_elem_size(self), _index)
/// Pop the last element off the array, returning the element by value.
#define array_pop(self) ((self)->contents[--(self)->size])
/// Assign the contents of one array to another, reallocating if necessary.
#define array_assign(self, other) \
_array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
/// Swap one array with another
#define array_swap(self, other) \
_array__swap((Array *)(self), (Array *)(other))
/// Get the size of the array contents
#define array_elem_size(self) (sizeof *(self)->contents)
/// Search a sorted array for a given `needle` value, using the given `compare`
/// callback to determine the order.
///
/// If an existing element is found to be equal to `needle`, then the `index`
/// out-parameter is set to the existing value's index, and the `exists`
/// out-parameter is set to true. Otherwise, `index` is set to an index where
/// `needle` should be inserted in order to preserve the sorting, and `exists`
/// is set to false.
#define array_search_sorted_with(self, compare, needle, _index, _exists) \
_array__search_sorted(self, 0, compare, , needle, _index, _exists)
/// Search a sorted array for a given `needle` value, using integer comparisons
/// of a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_with`.
#define array_search_sorted_by(self, field, needle, _index, _exists) \
_array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
/// Insert a given `value` into a sorted array, using the given `compare`
/// callback to determine the order.
#define array_insert_sorted_with(self, compare, value) \
do { \
unsigned _index, _exists; \
array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
if (!_exists) array_insert(self, _index, value); \
} while (0)
/// Insert a given `value` into a sorted array, using integer comparisons of
/// a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_by`.
#define array_insert_sorted_by(self, field, value) \
do { \
unsigned _index, _exists; \
array_search_sorted_by(self, field, (value) field, &_index, &_exists); \
if (!_exists) array_insert(self, _index, value); \
} while (0)
// Private
typedef Array(void) Array;
/// This is not what you're looking for, see `array_delete`.
static inline void _array__delete(Array *self) {
if (self->contents) {
ts_free(self->contents);
self->contents = NULL;
self->size = 0;
self->capacity = 0;
}
}
/// This is not what you're looking for, see `array_erase`.
static inline void _array__erase(Array *self, size_t element_size,
uint32_t index) {
assert(index < self->size);
char *contents = (char *)self->contents;
memmove(contents + index * element_size, contents + (index + 1) * element_size,
(self->size - index - 1) * element_size);
self->size--;
}
/// This is not what you're looking for, see `array_reserve`.
static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
if (new_capacity > self->capacity) {
if (self->contents) {
self->contents = ts_realloc(self->contents, new_capacity * element_size);
} else {
self->contents = ts_malloc(new_capacity * element_size);
}
self->capacity = new_capacity;
}
}
/// This is not what you're looking for, see `array_assign`.
static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
_array__reserve(self, element_size, other->size);
self->size = other->size;
memcpy(self->contents, other->contents, self->size * element_size);
}
/// This is not what you're looking for, see `array_swap`.
static inline void _array__swap(Array *self, Array *other) {
Array swap = *other;
*other = *self;
*self = swap;
}
/// This is not what you're looking for, see `array_push` or `array_grow_by`.
static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
uint32_t new_size = self->size + count;
if (new_size > self->capacity) {
uint32_t new_capacity = self->capacity * 2;
if (new_capacity < 8) new_capacity = 8;
if (new_capacity < new_size) new_capacity = new_size;
_array__reserve(self, element_size, new_capacity);
}
}
/// This is not what you're looking for, see `array_splice`.
static inline void _array__splice(Array *self, size_t element_size,
uint32_t index, uint32_t old_count,
uint32_t new_count, const void *elements) {
uint32_t new_size = self->size + new_count - old_count;
uint32_t old_end = index + old_count;
uint32_t new_end = index + new_count;
assert(old_end <= self->size);
_array__reserve(self, element_size, new_size);
char *contents = (char *)self->contents;
if (self->size > old_end) {
memmove(
contents + new_end * element_size,
contents + old_end * element_size,
(self->size - old_end) * element_size
);
}
if (new_count > 0) {
if (elements) {
memcpy(
(contents + index * element_size),
elements,
new_count * element_size
);
} else {
memset(
(contents + index * element_size),
0,
new_count * element_size
);
}
}
self->size += new_count - old_count;
}
/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`.
#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
do { \
*(_index) = start; \
*(_exists) = false; \
uint32_t size = (self)->size - *(_index); \
if (size == 0) break; \
int comparison; \
while (size > 1) { \
uint32_t half_size = size / 2; \
uint32_t mid_index = *(_index) + half_size; \
comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
if (comparison <= 0) *(_index) = mid_index; \
size -= half_size; \
} \
comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
if (comparison == 0) *(_exists) = true; \
else if (comparison < 0) *(_index) += 1; \
} while (0)
/// Helper macro for the `_sorted_by` routines below. This takes the left (existing)
/// parameter by reference in order to work with the generic sorting function above.
#define _compare_int(a, b) ((int)*(a) - (int)(b))
#ifdef _MSC_VER
#pragma warning(default : 4101)
#elif defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic pop
#endif
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_ARRAY_H_

68
python/extractor/tsg-python/tsp/src/tree_sitter/parser.h
View File

@@ -13,9 +13,8 @@ extern "C" {
#define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
typedef uint16_t TSStateId;
#ifndef TREE_SITTER_API_H_
typedef uint16_t TSStateId;
typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage;
@@ -48,6 +47,7 @@ struct TSLexer {
uint32_t (*get_column)(TSLexer *);
bool (*is_at_included_range_start)(const TSLexer *);
bool (*eof)(const TSLexer *);
void (*log)(const TSLexer *, const char *, ...);
};
typedef enum {
@@ -87,6 +87,11 @@ typedef union {
} entry;
} TSParseActionEntry;
typedef struct {
int32_t start;
int32_t end;
} TSCharacterRange;
struct TSLanguage {
uint32_t version;
uint32_t symbol_count;
@@ -126,13 +131,38 @@ struct TSLanguage {
const TSStateId *primary_state_ids;
};
static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
uint32_t index = 0;
uint32_t size = len - index;
while (size > 1) {
uint32_t half_size = size / 2;
uint32_t mid_index = index + half_size;
TSCharacterRange *range = &ranges[mid_index];
if (lookahead >= range->start && lookahead <= range->end) {
return true;
} else if (lookahead > range->end) {
index = mid_index;
}
size -= half_size;
}
TSCharacterRange *range = &ranges[index];
return (lookahead >= range->start && lookahead <= range->end);
}
/*
* Lexer Macros
*/
#ifdef _MSC_VER
#define UNUSED __pragma(warning(suppress : 4101))
#else
#define UNUSED __attribute__((unused))
#endif
#define START_LEXER() \
bool result = false; \
bool skip = false; \
UNUSED \
bool eof = false; \
int32_t lookahead; \
goto start; \
@@ -148,6 +178,17 @@ struct TSLanguage {
goto next_state; \
}
#define ADVANCE_MAP(...) \
{ \
static const uint16_t map[] = { __VA_ARGS__ }; \
for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \
if (map[i] == lookahead) { \
state = map[i + 1]; \
goto next_state; \
} \
} \
}
#define SKIP(state_value) \
{ \
skip = true; \
@@ -166,7 +207,7 @@ struct TSLanguage {
* Parse Table Macros
*/
#define SMALL_STATE(id) id - LARGE_STATE_COUNT
#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
#define STATE(id) id
@@ -176,7 +217,7 @@ struct TSLanguage {
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = state_value \
.state = (state_value) \
} \
}}
@@ -184,7 +225,7 @@ struct TSLanguage {
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = state_value, \
.state = (state_value), \
.repetition = true \
} \
}}
@@ -197,14 +238,15 @@ struct TSLanguage {
} \
}}
#define REDUCE(symbol_val, child_count_val, ...) \
{{ \
.reduce = { \
.type = TSParseActionTypeReduce, \
.symbol = symbol_val, \
.child_count = child_count_val, \
__VA_ARGS__ \
}, \
#define REDUCE(symbol_name, children, precedence, prod_id) \
{{ \
.reduce = { \
.type = TSParseActionTypeReduce, \
.symbol = symbol_name, \
.child_count = children, \
.dynamic_precedence = precedence, \
.production_id = prod_id \
}, \
}}
#define RECOVER() \

5
python/ql/lib/change-notes/2024-10-09-type-parameter-defaults.md Normal file
View File

@@ -0,0 +1,5 @@
---
category: minorAnalysis
---
- Added support for type parameter defaults, as specified in [PEP-696](https://peps.python.org/pep-0696/).

5
python/ql/lib/change-notes/2024-10-15-models-for-copy-replace.md Normal file
View File

@@ -0,0 +1,5 @@
---
category: minorAnalysis
---
- Added partial support for the `copy.replace` method, [added](https://docs.python.org/3.13/library/copy.html#copy.replace) in Python 3.13.

9
python/ql/lib/semmle/python/AstGenerated.qll
View File

@@ -1126,6 +1126,9 @@ class ParamSpec_ extends @py_ParamSpec, TypeParameter {
/** Gets the name of this parameter spec. */
Expr getName() { py_exprs(result, _, this, 1) }
/** Gets the default of this parameter spec. */
Expr getDefault() { py_exprs(result, _, this, 2) }
override string toString() { result = "ParamSpec" }
}
@@ -1466,6 +1469,9 @@ class TypeVar_ extends @py_TypeVar, TypeParameter {
/** Gets the bound of this type variable. */
Expr getBound() { py_exprs(result, _, this, 2) }
/** Gets the default of this type variable. */
Expr getDefault() { py_exprs(result, _, this, 3) }
override string toString() { result = "TypeVar" }
}
@@ -1474,6 +1480,9 @@ class TypeVarTuple_ extends @py_TypeVarTuple, TypeParameter {
/** Gets the name of this type variable tuple. */
Expr getName() { py_exprs(result, _, this, 1) }
/** Gets the default of this type variable tuple. */
Expr getDefault() { py_exprs(result, _, this, 2) }
override string toString() { result = "TypeVarTuple" }
}

17
python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll
View File

@@ -687,16 +687,23 @@ newtype TContent =
class Content extends TContent {
/** Gets a textual representation of this element. */
string toString() { result = "Content" }
/** Gets the Models-as-Data representation of this content (if any). */
string getMaDRepresentation() { none() }
}
/** An element of a list. */
class ListElementContent extends TListElementContent, Content {
override string toString() { result = "List element" }
override string getMaDRepresentation() { result = "ListElement" }
}
/** An element of a set. */
class SetElementContent extends TSetElementContent, Content {
override string toString() { result = "Set element" }
override string getMaDRepresentation() { result = "SetElement" }
}
/** An element of a tuple at a specific index. */
@@ -709,6 +716,8 @@ class TupleElementContent extends TTupleElementContent, Content {
int getIndex() { result = index }
override string toString() { result = "Tuple element at index " + index.toString() }
override string getMaDRepresentation() { result = "TupleElement[" + index + "]" }
}
/** An element of a dictionary under a specific key. */
@@ -721,11 +730,15 @@ class DictionaryElementContent extends TDictionaryElementContent, Content {
string getKey() { result = key }
override string toString() { result = "Dictionary element at key " + key }
override string getMaDRepresentation() { result = "DictionaryElement[" + key + "]" }
}
/** An element of a dictionary under any key. */
class DictionaryElementAnyContent extends TDictionaryElementAnyContent, Content {
override string toString() { result = "Any dictionary element" }
override string getMaDRepresentation() { result = "DictionaryElementAny" }
}
/** An object attribute. */
@@ -738,6 +751,8 @@ class AttributeContent extends TAttributeContent, Content {
string getAttribute() { result = attr }
override string toString() { result = "Attribute " + attr }
override string getMaDRepresentation() { result = "Attribute[" + attr + "]" }
}
/** A captured variable. */
@@ -750,6 +765,8 @@ class CapturedVariableContent extends Content, TCapturedVariableContent {
VariableCapture::CapturedVariable getVariable() { result = v }
override string toString() { result = "captured " + v }
override string getMaDRepresentation() { none() }
}
/**

2
python/ql/lib/semmle/python/frameworks/Stdlib.model.yml
View File

@@ -45,7 +45,7 @@ extensions:
# See https://docs.python.org/3/library/contextlib.html#contextlib.ExitStack
- ["contextlib.ExitStack", "Member[enter_context]", "Argument[0,cm:]", "ReturnValue", "taint"]
# See https://docs.python.org/3/library/copy.html#copy.deepcopy
- ["copy", "Member[copy,deepcopy]", "Argument[0,x:]", "ReturnValue", "value"]
- ["copy", "Member[copy,deepcopy,replace]", "Argument[0,x:]", "ReturnValue", "value"]
# See
# - https://docs.python.org/3/library/ctypes.html#ctypes.create_string_buffer
# - https://docs.python.org/3/library/ctypes.html#ctypes.create_unicode_buffer

44
python/ql/lib/semmle/python/frameworks/Stdlib.qll
View File

@@ -4537,21 +4537,9 @@ module StdlibPrivate {
override DataFlow::ArgumentNode getACallback() { none() }
override predicate propagatesFlow(string input, string output, boolean preservesValue) {
exists(string content |
content = "ListElement"
or
content = "SetElement"
or
exists(DataFlow::TupleElementContent tc, int i | i = tc.getIndex() |
content = "TupleElement[" + i.toString() + "]"
)
or
exists(DataFlow::DictionaryElementContent dc, string key | key = dc.getKey() |
content = "DictionaryElement[" + key + "]"
)
|
input = "Argument[self]." + content and
output = "ReturnValue." + content and
exists(DataFlow::Content c |
input = "Argument[self]." + c.getMaDRepresentation() and
output = "ReturnValue." + c.getMaDRepresentation() and
preservesValue = true
)
or
@@ -4561,6 +4549,32 @@ module StdlibPrivate {
}
}
/** A flow summary for `copy.replace`. */
class ReplaceSummary extends SummarizedCallable {
ReplaceSummary() { this = "copy.replace" }
override DataFlow::CallCfgNode getACall() {
result = API::moduleImport("copy").getMember("replace").getACall()
}
override DataFlow::ArgumentNode getACallback() {
result = API::moduleImport("copy").getMember("replace").getAValueReachableFromSource()
}
override predicate propagatesFlow(string input, string output, boolean preservesValue) {
exists(CallNode c, string name, ControlFlowNode n, DataFlow::AttributeContent ac |
c.getFunction().(NameNode).getId() = "replace" or
c.getFunction().(AttrNode).getName() = "replace"
|
n = c.getArgByName(name) and
ac.getAttribute() = name and
input = "Argument[" + name + ":]" and
output = "ReturnValue." + ac.getMaDRepresentation() and
preservesValue = true
)
}
}
/**
* A flow summary for `pop` either for list or set.
* This ignores the index if given, since content is

3
python/ql/lib/semmlecode.python.dbscheme
View File

@@ -617,6 +617,7 @@ py_extracted_version(int module : @py_Module ref,
/* <Field> ParamSpec.location = 0, location */
/* <Field> ParamSpec.name = 1, expr */
/* <Field> ParamSpec.default = 2, expr */
/* <Field> Pass.location = 0, location */
@@ -715,9 +716,11 @@ py_extracted_version(int module : @py_Module ref,
/* <Field> TypeVar.location = 0, location */
/* <Field> TypeVar.name = 1, expr */
/* <Field> TypeVar.bound = 2, expr */
/* <Field> TypeVar.default = 3, expr */
/* <Field> TypeVarTuple.location = 0, location */
/* <Field> TypeVarTuple.name = 1, expr */
/* <Field> TypeVarTuple.default = 2, expr */
/* <Field> UnaryExpr.location = 0, location */
/* <Field> UnaryExpr.parenthesised = 1, bool */

1233
python/ql/lib/upgrades/728c6d65e61d808ae276013ebc15abc3a97aaef1/old.dbscheme Normal file
View File

File diff suppressed because it is too large Load Diff

1236
python/ql/lib/upgrades/728c6d65e61d808ae276013ebc15abc3a97aaef1/semmlecode.python.dbscheme Normal file
View File

File diff suppressed because it is too large Load Diff

2
python/ql/lib/upgrades/728c6d65e61d808ae276013ebc15abc3a97aaef1/upgrade.properties Normal file
View File

@@ -0,0 +1,2 @@
description: Add support for type parameter defaults.
compatibility: backwards

29
python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_collections.py
View File

@@ -166,6 +166,34 @@ def test_copy_2():
copy.deepcopy(TAINTED_LIST), # $ tainted
)
def test_replace():
from copy import replace
class C:
def __init__(self, always_tainted, tainted_to_safe, safe_to_tainted, always_safe):
self.always_tainted = always_tainted
self.tainted_to_safe = tainted_to_safe
self.safe_to_tainted = safe_to_tainted
self.always_safe = always_safe
c = C(always_tainted=TAINTED_STRING,
tainted_to_safe=TAINTED_STRING,
safe_to_tainted=NOT_TAINTED,
always_safe=NOT_TAINTED)
d = replace(c, tainted_to_safe=NOT_TAINTED, safe_to_tainted=TAINTED_STRING)
ensure_tainted(d.always_tainted) # $ tainted
ensure_tainted(d.safe_to_tainted) # $ tainted
ensure_not_tainted(d.always_safe)
# Currently, we have no way of stopping the value in the tainted_to_safe field (which gets
# overwritten) from flowing through the replace call, which means we get a spurious result.
ensure_not_tainted(d.tainted_to_safe) # $ SPURIOUS: tainted
def list_index_assign():
tainted_string = TAINTED_STRING
@@ -274,6 +302,7 @@ test_named_tuple()
test_defaultdict("key", "key")
test_copy_1()
test_copy_2()
test_replace()
list_index_assign()
list_index_aug_assign()

15
rust/codegen/BUILD.bazel
View File

@@ -1,3 +1,5 @@
load("@bazel_skylib//rules:native_binary.bzl", "native_binary")
_args = [
"//rust/ast-generator",
"//rust/ast-generator:manifest",
@@ -15,3 +17,16 @@ sh_binary(
"//misc/bazel:sh_runfiles",
],
)
native_binary(
name = "py",
src = "//misc/codegen",
out = "codegen",
args = [
"--configuration-file=$(location //rust:codegen-conf)",
],
data = [
"//rust:codegen-conf",
],
visibility = ["//rust:__subpackages__"],
)

3
rust/codegen/codegen.sh
View File

@@ -8,6 +8,7 @@ ast_generator="$(rlocation "$1")"
ast_generator_manifest="$(rlocation "$2")"
codegen="$(rlocation "$3")"
codegen_conf="$(rlocation "$4")"
shift 4
CARGO_MANIFEST_DIR="$(dirname "$ast_generator_manifest")" "$ast_generator"
"$codegen" --configuration-file="$codegen_conf"
"$codegen" --configuration-file="$codegen_conf" "$@"

27
rust/ql/consistency-queries/AstConsistency.ql
View File

@@ -1,21 +1,8 @@
import rust
import codeql.rust.elements.internal.generated.ParentChild
/**
* @name Abstract syntax tree inconsistencies
* @description Lists the abstract syntax tree inconsistencies in the database. This query is intended for internal use.
* @kind table
* @id rust/diagnostics/ast-consistency
*/
query predicate multipleToString(Element e, string s) {
s = strictconcat(e.toString(), ",") and
strictcount(e.toString()) > 1
}
query predicate multipleLocations(Locatable e) { strictcount(e.getLocation()) > 1 }
query predicate multiplePrimaryQlClasses(Element e, string s) {
s = e.getPrimaryQlClasses() and
strictcount(e.getAPrimaryQlClass()) > 1
}
private Element getParent(Element child) { child = getChildAndAccessor(result, _, _) }
query predicate multipleParents(Element child, Element parent) {
parent = getParent(child) and
strictcount(getParent(child)) > 1
}
import codeql.rust.AstConsistency

39
rust/ql/consistency-queries/CfgConsistency.ql
View File

@@ -1,37 +1,8 @@
import rust
import codeql.rust.controlflow.internal.ControlFlowGraphImpl::Consistency as Consistency
import Consistency
import codeql.rust.controlflow.ControlFlowGraph
import codeql.rust.controlflow.internal.ControlFlowGraphImpl as CfgImpl
import codeql.rust.controlflow.internal.Completion
/**
* All `Expr` nodes are `PostOrderTree`s
* @name Control flow graph inconsistencies
* @description Lists the control flow graph inconsistencies in the database. This query is intended for internal use.
* @kind table
* @id rust/diagnostics/cfg-consistency
*/
query predicate nonPostOrderExpr(Expr e, string cls) {
cls = e.getPrimaryQlClasses() and
not e instanceof LetExpr and
not e instanceof ParenExpr and
exists(AstNode last, Completion c |
CfgImpl::last(e, last, c) and
last != e and
c instanceof NormalCompletion
)
}
query predicate scopeNoFirst(CfgScope scope) {
Consistency::scopeNoFirst(scope) and
not scope = any(Function f | not exists(f.getBody())) and
not scope = any(ClosureExpr c | not exists(c.getBody()))
}
/** Holds if `be` is the `else` branch of a `let` statement that results in a panic. */
private predicate letElsePanic(BlockExpr be) {
be = any(LetStmt let).getLetElse().getBlockExpr() and
exists(Completion c | CfgImpl::last(be, _, c) | completionIsNormal(c))
}
query predicate deadEnd(CfgImpl::Node node) {
Consistency::deadEnd(node) and
not letElsePanic(node.getAstNode())
}
import codeql.rust.controlflow.internal.CfgConsistency

7
rust/ql/consistency-queries/ExtractionConsistency.ql
View File

@@ -1,3 +1,10 @@
/**
* @name Extraction consistency
* @description Lists the extraction inconsistencies (errors) in the database. This query is intended for internal use.
* @kind table
* @id rust/diagnostics/extraction-consistency
*/
import codeql.rust.Diagnostics
query predicate extractionError(ExtractionError ee) { any() }

55
rust/ql/lib/codeql/rust/AstConsistency.qll Normal file
View File

@@ -0,0 +1,55 @@
/**
* Provides classes for recognizing control flow graph inconsistencies.
*/
private import rust
private import codeql.rust.elements.internal.generated.ParentChild
/**
* Holds if `e` has more than one `toString()` result.
*/
query predicate multipleToStrings(Element e, string s) {
s = strictconcat(e.toString(), ", ") and
strictcount(e.toString()) > 1
}
/**
* Holds if `e` has more than one `Location`.
*/
query predicate multipleLocations(Locatable e) { strictcount(e.getLocation()) > 1 }
/**
* Holds if `e` has more than one `getPrimaryQlClasses()` result.
*/
query predicate multiplePrimaryQlClasses(Element e, string s) {
s = strictconcat(e.getPrimaryQlClasses(), ", ") and
strictcount(e.getAPrimaryQlClass()) > 1
}
private Element getParent(Element child) { child = getChildAndAccessor(result, _, _) }
/**
* Holds if `child` has more than one AST parent.
*/
query predicate multipleParents(Element child, Element parent) {
parent = getParent(child) and
strictcount(getParent(child)) > 1
}
/**
* Gets counts of abstract syntax tree inconsistencies of each type.
*/
int getAstInconsistencyCounts(string type) {
// total results from all the AST consistency query predicates.
type = "Multiple toStrings" and
result = count(Element e | multipleToStrings(e, _) | e)
or
type = "Multiple locations" and
result = count(Element e | multipleLocations(e) | e)
or
type = "Multiple primary QL classes" and
result = count(Element e | multiplePrimaryQlClasses(e, _) | e)
or
type = "Multiple parents" and
result = count(Element e | multipleParents(e, _) | e)
}

94
rust/ql/lib/codeql/rust/controlflow/internal/CfgConsistency.qll Normal file
View File

@@ -0,0 +1,94 @@
/**
* Provides classes for recognizing control flow graph inconsistencies.
*/
private import rust
private import codeql.rust.controlflow.internal.ControlFlowGraphImpl::Consistency as Consistency
import Consistency
private import codeql.rust.controlflow.ControlFlowGraph
private import codeql.rust.controlflow.internal.ControlFlowGraphImpl as CfgImpl
private import codeql.rust.controlflow.internal.Completion
/**
* All `Expr` nodes are `PostOrderTree`s
*/
query predicate nonPostOrderExpr(Expr e, string cls) {
cls = e.getPrimaryQlClasses() and
not e instanceof LetExpr and
not e instanceof ParenExpr and
exists(AstNode last, Completion c |
CfgImpl::last(e, last, c) and
last != e and
c instanceof NormalCompletion
)
}
/**
* Holds if CFG scope `scope` lacks an initial AST node. Overrides shared consistency predicate.
*/
query predicate scopeNoFirst(CfgScope scope) {
Consistency::scopeNoFirst(scope) and
not scope = any(Function f | not exists(f.getBody())) and
not scope = any(ClosureExpr c | not exists(c.getBody()))
}
/** Holds if `be` is the `else` branch of a `let` statement that results in a panic. */
private predicate letElsePanic(BlockExpr be) {
be = any(LetStmt let).getLetElse().getBlockExpr() and
exists(Completion c | CfgImpl::last(be, _, c) | completionIsNormal(c))
}
/**
* Holds if `node` is lacking a successor. Overrides shared consistency predicate.
*/
query predicate deadEnd(CfgImpl::Node node) {
Consistency::deadEnd(node) and
not letElsePanic(node.getAstNode())
}
/**
* Gets counts of control flow graph inconsistencies of each type.
*/
int getCfgInconsistencyCounts(string type) {
// total results from all the CFG consistency query predicates in:
// - `codeql.rust.controlflow.internal.CfgConsistency` (this file)
// - `shared.controlflow.codeql.controlflow.Cfg`
type = "Non-unique set representation" and
result = count(CfgImpl::Splits ss | nonUniqueSetRepresentation(ss, _) | ss)
or
type = "Splitting invariant 2" and
result = count(AstNode n | breakInvariant2(n, _, _, _, _, _) | n)
or
type = "Splitting invariant 3" and
result = count(AstNode n | breakInvariant3(n, _, _, _, _, _) | n)
or
type = "Splitting invariant 4" and
result = count(AstNode n | breakInvariant4(n, _, _, _, _, _) | n)
or
type = "Splitting invariant 5" and
result = count(AstNode n | breakInvariant5(n, _, _, _, _, _) | n)
or
type = "Multiple successors of the same type" and
result = count(CfgNode n | multipleSuccessors(n, _, _) | n)
or
type = "Simple and normal successors" and
result = count(CfgNode n | simpleAndNormalSuccessors(n, _, _, _, _) | n)
or
type = "Dead end" and
result = count(CfgNode n | deadEnd(n) | n)
or
type = "Non-unique split kind" and
result = count(CfgImpl::SplitImpl si | nonUniqueSplitKind(si, _) | si)
or
type = "Non-unique list order" and
result = count(CfgImpl::SplitKind sk | nonUniqueListOrder(sk, _) | sk)
or
type = "Multiple toStrings" and
result = count(CfgNode n | multipleToString(n, _) | n)
or
type = "CFG scope lacks initial AST node" and
result = count(CfgScope s | scopeNoFirst(s) | s)
or
type = "Non-PostOrderTree Expr node" and
result = count(Expr e | nonPostOrderExpr(e, _) | e)
}

15
rust/ql/src/queries/diagnostics/AstConsistencyCounts.ql Normal file
View File

@@ -0,0 +1,15 @@
/**
* @name Abstract syntax tree inconsistency counts
* @description Counts the number of abstract syntax tree inconsistencies of each type. This query is intended for internal use.
* @kind diagnostic
* @id rust/diagnostics/ast-consistency-counts
*/
import rust
import codeql.rust.AstConsistency as Consistency
// see also `rust/diagnostics/ast-consistency`, which lists the
// individual inconsistency results.
from string type, int num
where num = Consistency::getAstInconsistencyCounts(type)
select type, num

15
rust/ql/src/queries/diagnostics/CfgConsistencyCounts.ql Normal file
View File

@@ -0,0 +1,15 @@
/**
* @name Control flow graph inconsistency counts
* @description Counts the number of control flow graph inconsistencies of each type. This query is intended for internal use.
* @kind diagnostic
* @id rust/diagnostics/cfg-consistency-counts
*/
import rust
import codeql.rust.controlflow.internal.CfgConsistency as Consistency
// see also `rust/diagnostics/cfg-consistency`, which lists the
// individual inconsistency results.
from string type, int num
where num = Consistency::getCfgInconsistencyCounts(type)
select type, num

22
rust/ql/src/queries/summary/Stats.qll
View File

@@ -3,9 +3,31 @@
*/
import rust
import codeql.rust.AstConsistency as AstConsistency
private import codeql.rust.controlflow.internal.CfgConsistency as CfgConsistency
/**
* Gets a count of the total number of lines of code in the database.
*/
int getLinesOfCode() { result = sum(File f | | f.getNumberOfLinesOfCode()) }
/**
* Gets a count of the total number of lines of code from the source code directory in the database.
*/
int getLinesOfUserCode() {
result = sum(File f | exists(f.getRelativePath()) | f.getNumberOfLinesOfCode())
}
/**
* Gets a count of the total number of abstract syntax tree inconsistencies in the database.
*/
int getTotalAstInconsistencies() {
result = sum(string type | | AstConsistency::getAstInconsistencyCounts(type))
}
/**
* Gets a count of the total number of control flow graph inconsistencies in the database.
*/
int getTotalCfgInconsistencies() {
result = sum(string type | | CfgConsistency::getCfgInconsistencyCounts(type))
}

28
rust/ql/src/queries/summary/SummaryStats.ql
View File

@@ -1,7 +1,7 @@
/**
* @name Summary Statistics
* @description A table of summary statistics about a database.
* @kind table
* @kind metric
* @id rust/summary/summary-statistics
* @tags summary
*/
@@ -10,27 +10,29 @@ import rust
import codeql.rust.Diagnostics
import Stats
from string key, string value
from string key, int value
where
key = "Elements extracted" and value = count(Element e | not e instanceof Unextracted).toString()
key = "Elements extracted" and value = count(Element e | not e instanceof Unextracted)
or
key = "Elements unextracted" and value = count(Unextracted e).toString()
key = "Elements unextracted" and value = count(Unextracted e)
or
key = "Extraction errors" and value = count(ExtractionError e).toString()
key = "Extraction errors" and value = count(ExtractionError e)
or
key = "Extraction warnings" and value = count(ExtractionWarning w).toString()
key = "Extraction warnings" and value = count(ExtractionWarning w)
or
key = "Files extracted - total" and value = count(File f | exists(f.getRelativePath())).toString()
key = "Files extracted - total" and value = count(File f | exists(f.getRelativePath()))
or
key = "Files extracted - with errors" and
value =
count(File f | exists(f.getRelativePath()) and not f instanceof SuccessfullyExtractedFile)
.toString()
value = count(File f | exists(f.getRelativePath()) and not f instanceof SuccessfullyExtractedFile)
or
key = "Files extracted - without errors" and
value = count(SuccessfullyExtractedFile f | exists(f.getRelativePath())).toString()
value = count(SuccessfullyExtractedFile f | exists(f.getRelativePath()))
or
key = "Lines of code extracted" and value = getLinesOfCode().toString()
key = "Lines of code extracted" and value = getLinesOfCode()
or
key = "Lines of user code extracted" and value = getLinesOfUserCode().toString()
key = "Lines of user code extracted" and value = getLinesOfUserCode()
or
key = "Inconsistencies - AST" and value = getTotalAstInconsistencies()
or
key = "Inconsistencies - CFG" and value = getTotalCfgInconsistencies()
select key, value

15
rust/ql/src/queries/unusedentities/UnusedValue.ql
View File

@@ -9,7 +9,16 @@
*/
import rust
import codeql.rust.dataflow.Ssa
import codeql.rust.dataflow.internal.SsaImpl
import UnusedVariable
from Locatable e
where none() // TODO: implement query
select e, "Variable is assigned a value that is never used."
from AstNode write, Ssa::Variable v
where
variableWrite(write, v) and
// SSA definitions are only created for live writes
not write = any(Ssa::WriteDefinition def).getWriteAccess().getAstNode() and
// avoid overlap with the unused variable query
not isUnused(v) and
not v instanceof DiscardVariable
select write, "Variable is assigned a value that is never used."

7
rust/ql/src/queries/unusedentities/UnusedVariable.ql
View File

@@ -9,11 +9,8 @@
*/
import rust
import UnusedVariable
from Variable v
where
not exists(v.getAnAccess()) and
not exists(v.getInitializer()) and
not v.getName().charAt(0) = "_" and
exists(File f | f.getBaseName() = "main.rs" | v.getLocation().getFile() = f) // temporarily severely limit results
where isUnused(v)
select v, "Variable is not used."

14
rust/ql/src/queries/unusedentities/UnusedVariable.qll Normal file
View File

@@ -0,0 +1,14 @@
import rust
/** A deliberately unused variable. */
class DiscardVariable extends Variable {
DiscardVariable() { this.getName().charAt(0) = "_" }
}
/** Holds if variable `v` is unused. */
predicate isUnused(Variable v) {
not exists(v.getAnAccess()) and
not exists(v.getInitializer()) and
not v instanceof DiscardVariable and
exists(File f | f.getBaseName() = "main.rs" | v.getLocation().getFile() = f) // temporarily severely limit results
}

4
rust/ql/test/query-tests/diagnostics/AstConsistencyCounts.expected Normal file
View File

@@ -0,0 +1,4 @@
| Multiple locations | 0 |
| Multiple parents | 0 |
| Multiple primary QL classes | 0 |
| Multiple toStrings | 0 |

1
rust/ql/test/query-tests/diagnostics/AstConsistencyCounts.qlref Normal file
View File

@@ -0,0 +1 @@
queries/diagnostics/AstConsistencyCounts.ql

13
rust/ql/test/query-tests/diagnostics/CfgConsistencyCounts.expected Normal file
View File

@@ -0,0 +1,13 @@
| CFG scope lacks initial AST node | 0 |
| Dead end | 0 |
| Multiple successors of the same type | 0 |
| Multiple toStrings | 0 |
| Non-PostOrderTree Expr node | 0 |
| Non-unique list order | 0 |
| Non-unique set representation | 0 |
| Non-unique split kind | 0 |
| Simple and normal successors | 0 |
| Splitting invariant 2 | 0 |
| Splitting invariant 3 | 0 |
| Splitting invariant 4 | 0 |
| Splitting invariant 5 | 0 |

1
rust/ql/test/query-tests/diagnostics/CfgConsistencyCounts.qlref Normal file
View File

@@ -0,0 +1 @@
queries/diagnostics/CfgConsistencyCounts.ql

2
rust/ql/test/query-tests/diagnostics/SummaryStats.expected
View File

@@ -5,5 +5,7 @@
| Files extracted - total | 7 |
| Files extracted - with errors | 2 |
| Files extracted - without errors | 5 |
| Inconsistencies - AST | 0 |
| Inconsistencies - CFG | 0 |
| Lines of code extracted | 59 |
| Lines of user code extracted | 59 |

26
rust/ql/test/query-tests/unusedentities/CONSISTENCY/CfgConsistency.expected
View File

@@ -1,11 +1,17 @@
multipleSuccessors
| main.rs:428:17:428:17 | 0 | successor | main.rs:428:9:428:10 | a2 |
| main.rs:428:17:428:17 | 0 | successor | main.rs:428:20:428:62 | ClosureExpr |
| main.rs:431:17:431:17 | 0 | successor | main.rs:431:9:431:10 | a3 |
| main.rs:431:17:431:17 | 0 | successor | main.rs:431:20:431:41 | ClosureExpr |
| main.rs:434:17:434:17 | 0 | successor | main.rs:434:9:434:10 | a4 |
| main.rs:434:17:434:17 | 0 | successor | main.rs:434:20:434:35 | ClosureExpr |
| main.rs:437:17:437:17 | 0 | successor | main.rs:437:9:437:10 | a5 |
| main.rs:437:17:437:17 | 0 | successor | main.rs:437:20:437:35 | ClosureExpr |
| main.rs:441:17:441:17 | 0 | successor | main.rs:441:9:441:10 | a6 |
| main.rs:441:17:441:17 | 0 | successor | main.rs:441:20:441:46 | ClosureExpr |
| main.rs:421:17:421:17 | 0 | successor | main.rs:421:9:421:10 | a2 |
| main.rs:421:17:421:17 | 0 | successor | main.rs:421:20:421:62 | ClosureExpr |
| main.rs:424:17:424:17 | 0 | successor | main.rs:424:9:424:10 | a3 |
| main.rs:424:17:424:17 | 0 | successor | main.rs:424:20:424:41 | ClosureExpr |
| main.rs:427:17:427:17 | 0 | successor | main.rs:427:9:427:10 | a4 |
| main.rs:427:17:427:17 | 0 | successor | main.rs:427:20:427:35 | ClosureExpr |
| main.rs:430:17:430:17 | 0 | successor | main.rs:430:9:430:10 | a5 |
| main.rs:430:17:430:17 | 0 | successor | main.rs:430:20:430:35 | ClosureExpr |
| main.rs:434:17:434:17 | 0 | successor | main.rs:434:9:434:10 | a6 |
| main.rs:434:17:434:17 | 0 | successor | main.rs:434:20:434:46 | ClosureExpr |
| more.rs:32:17:32:17 | a | successor | more.rs:32:5:32:5 | i |
| more.rs:32:17:32:17 | a | successor | more.rs:32:20:32:20 | b |
deadEnd
| more.rs:9:9:9:19 | Param |
| more.rs:38:23:38:28 | Param |
| more.rs:42:19:42:24 | Param |

30
rust/ql/test/query-tests/unusedentities/UnusedValue.expected
View File

@@ -0,0 +1,30 @@
| main.rs:6:9:6:9 | a | Variable is assigned a value that is never used. |
| main.rs:9:9:9:9 | d | Variable is assigned a value that is never used. |
| main.rs:35:5:35:5 | b | Variable is assigned a value that is never used. |
| main.rs:37:5:37:5 | c | Variable is assigned a value that is never used. |
| main.rs:40:5:40:5 | c | Variable is assigned a value that is never used. |
| main.rs:44:9:44:9 | d | Variable is assigned a value that is never used. |
| main.rs:50:5:50:5 | e | Variable is assigned a value that is never used. |
| main.rs:61:5:61:5 | f | Variable is assigned a value that is never used. |
| main.rs:63:5:63:5 | f | Variable is assigned a value that is never used. |
| main.rs:65:5:65:5 | g | Variable is assigned a value that is never used. |
| main.rs:87:9:87:9 | a | Variable is assigned a value that is never used. |
| main.rs:108:9:108:10 | is | Variable is assigned a value that is never used. |
| main.rs:133:13:133:17 | total | Variable is assigned a value that is never used. |
| main.rs:203:13:203:31 | res | Variable is assigned a value that is never used. |
| main.rs:218:9:218:24 | kind | Variable is assigned a value that is never used. |
| main.rs:223:9:223:32 | kind | Variable is assigned a value that is never used. |
| main.rs:280:13:280:17 | total | Variable is assigned a value that is never used. |
| main.rs:348:5:348:39 | kind | Variable is assigned a value that is never used. |
| main.rs:370:9:370:9 | x | Variable is assigned a value that is never used. |
| main.rs:378:17:378:17 | x | Variable is assigned a value that is never used. |
| main.rs:432:9:432:10 | i6 | Variable is assigned a value that is never used. |
| more.rs:8:9:8:13 | times | Variable is assigned a value that is never used. |
| more.rs:9:9:9:14 | unused | Variable is assigned a value that is never used. |
| more.rs:21:9:21:14 | unused | Variable is assigned a value that is never used. |
| more.rs:38:23:38:25 | val | Variable is assigned a value that is never used. |
| more.rs:42:19:42:21 | val | Variable is assigned a value that is never used. |
| more.rs:58:9:58:11 | val | Variable is assigned a value that is never used. |
| more.rs:80:9:80:14 | a_ptr4 | Variable is assigned a value that is never used. |
| more.rs:95:9:95:13 | d_ptr | Variable is assigned a value that is never used. |
| more.rs:101:9:101:17 | f_ptr | Variable is assigned a value that is never used. |

28
rust/ql/test/query-tests/unusedentities/UnusedVariable.expected
View File

@@ -6,17 +6,17 @@
| main.rs:201:9:201:9 | x | Variable is not used. |
| main.rs:250:17:250:17 | a | Variable is not used. |
| main.rs:258:20:258:22 | val | Variable is not used. |
| main.rs:271:14:271:16 | val | Variable is not used. |
| main.rs:288:22:288:24 | val | Variable is not used. |
| main.rs:296:24:296:26 | val | Variable is not used. |
| main.rs:305:13:305:15 | num | Variable is not used. |
| main.rs:320:12:320:12 | j | Variable is not used. |
| main.rs:342:25:342:25 | y | Variable is not used. |
| main.rs:346:28:346:28 | a | Variable is not used. |
| main.rs:350:9:350:9 | p | Variable is not used. |
| main.rs:365:9:365:13 | right | Variable is not used. |
| main.rs:371:9:371:14 | right2 | Variable is not used. |
| main.rs:378:13:378:13 | y | Variable is not used. |
| main.rs:386:21:386:21 | y | Variable is not used. |
| main.rs:434:27:434:29 | val | Variable is not used. |
| main.rs:437:22:437:24 | acc | Variable is not used. |
| main.rs:272:14:272:16 | val | Variable is not used. |
| main.rs:287:22:287:24 | val | Variable is not used. |
| main.rs:294:24:294:26 | val | Variable is not used. |
| main.rs:302:13:302:15 | num | Variable is not used. |
| main.rs:317:12:317:12 | j | Variable is not used. |
| main.rs:337:25:337:25 | y | Variable is not used. |
| main.rs:340:28:340:28 | a | Variable is not used. |
| main.rs:343:9:343:9 | p | Variable is not used. |
| main.rs:358:9:358:13 | right | Variable is not used. |
| main.rs:364:9:364:14 | right2 | Variable is not used. |
| main.rs:371:13:371:13 | y | Variable is not used. |
| main.rs:379:21:379:21 | y | Variable is not used. |
| main.rs:427:27:427:29 | val | Variable is not used. |
| main.rs:430:22:430:24 | acc | Variable is not used. |

94
rust/ql/test/query-tests/unusedentities/main.rs
View File

@@ -3,10 +3,10 @@
// --- locals ---
fn locals_1() {
let a = 1; // BAD: unused value [NOT DETECTED]
let a = 1; // BAD: unused value
let b = 1;
let c = 1;
let d = String::from("a"); // BAD: unused value [NOT DETECTED]
let d = String::from("a"); // BAD: unused value
let e = String::from("b");
let f = 1;
let _ = 1; // (deliberately unused)
@@ -32,22 +32,22 @@ fn locals_2() {
let h: i32;
let i: i32;
b = 1; // BAD: unused value [NOT DETECTED]
b = 1; // BAD: unused value
c = 1; // BAD: unused value [NOT DETECTED]
c = 1; // BAD: unused value
c = 2;
println!("use {}", c);
c = 3; // BAD: unused value [NOT DETECTED]
c = 3; // BAD: unused value
d = 1;
if cond() {
d = 2; // BAD: unused value [NOT DETECTED]
d = 2; // BAD: unused value
d = 3;
} else {
}
println!("use {}", d);
e = 1; // BAD: unused value [NOT DETECTED]
e = 1; // BAD: unused value
if cond() {
e = 2;
} else {
@@ -58,16 +58,16 @@ fn locals_2() {
f = 1;
f += 1;
println!("use {}", f);
f += 1; // BAD: unused value [NOT DETECTED]
f += 1; // BAD: unused value
f = 1;
f += 1; // BAD: unused value [NOT DETECTED]
f += 1; // BAD: unused value
g = if cond() { 1 } else { 2 }; // BAD: unused value (x2) [NOT DETECTED]
g = if cond() { 1 } else { 2 }; // BAD: unused value
h = if cond() { 3 } else { 4 };
i = if cond() { h } else { 5 };
println!("use {}", i);
_ = 1; // (deliberately unused) [NOT DETECTED]
_ = 1; // GOOD (deliberately unused)
}
// --- structs ---
@@ -84,7 +84,7 @@ impl MyStruct {
}
fn structs() {
let a = MyStruct { val: 1 }; // BAD: unused value [NOT DETECTED]
let a = MyStruct { val: 1 }; // BAD: unused value
let b = MyStruct { val: 2 };
let c = MyStruct { val: 3 };
let mut d: MyStruct; // BAD: unused variable
@@ -105,7 +105,7 @@ fn structs() {
// --- arrays ---
fn arrays() {
let is = [1, 2, 3]; // BAD: unused values (x3) [NOT DETECTED]
let is = [1, 2, 3]; // BAD: unused value
let js = [1, 2, 3];
let ks = [1, 2, 3];
@@ -130,7 +130,7 @@ fn statics() {
static mut STAT4: i32 = 0; // BAD: unused value [NOT DETECTED]
unsafe {
let total = CON1 + STAT1 + STAT3;
let total = CON1 + STAT1 + STAT3; // BAD: unused value
}
}
@@ -200,7 +200,7 @@ fn loops() {
for x // SPURIOUS: unused variable
in 1..10 {
_ = format!("x is {x}");
_ = format!("x is {x}"); // SPURIOUS: unused value `res`
}
for x
@@ -215,12 +215,12 @@ fn loops() {
for x
in 1..10 {
assert_eq!(x, 1);
assert_eq!(x, 1); // SPURIOUS: unused value `kind`
}
for x
in 1..10 {
assert_eq!(id(x), id(1));
assert_eq!(id(x), id(1)); // SPURIOUS: unused value `kind`
}
}
@@ -255,7 +255,8 @@ fn if_lets_matches() {
}
let mut next = Some(30);
while let Some(val) = next // BAD: unused variable
while let Some(val) = // BAD: unused variable
next
{
next = None;
}
@@ -270,25 +271,22 @@ fn if_lets_matches() {
match c {
Some(val) => { // BAD: unused variable
}
None => {
}
None => {}
}
let d = Some(70);
match d {
Some(val) => {
total += val;
}
None => {
total += val; // BAD: unused value
}
None => {}
}
let e = Option::Some(80);
match e {
Option::Some(val) => { // BAD: unused variable
}
Option::None => {
}
Option::None => {}
}
let f = MyOption::Some(90);
@@ -298,10 +296,9 @@ fn if_lets_matches() {
MyOption::None => {}
}
let g : Result<i64, i64> = Ok(100);
let g: Result<i64, i64> = Ok(100);
match g {
Ok(_) => {
}
Ok(_) => {}
Err(num) => {} // BAD: unused variable
}
@@ -327,8 +324,7 @@ fn if_lets_matches() {
}
let l = Yes;
if let Yes = l {
}
if let Yes = l {}
match 1 {
1 => {}
@@ -337,22 +333,19 @@ fn if_lets_matches() {
let p1 = MyPoint { x: 1, y: 2 };
match p1 {
MyPoint { x: 0, y: 0 } => {
}
MyPoint { x: 0, y: 0 } => {}
MyPoint { x: 1, y } => { // BAD: unused variable
}
MyPoint { x: 2, y: _ } => {
}
MyPoint { x: 2, y: _ } => {}
MyPoint { x: 3, y: a } => { // BAD: unused variable
}
MyPoint { x: 4, .. } => {
}
MyPoint { x: 4, .. } => {}
p => { // BAD: unused variable
}
}
let duration1 = std::time::Duration::new(10, 0); // ten seconds
assert_eq!(duration1.as_secs(), 10);
assert_eq!(duration1.as_secs(), 10); // SPURIOUS: unused value `kind`
let duration2:Result<std::time::Duration, String> =
Ok(std::time::Duration::new(10, 0));
@@ -374,7 +367,7 @@ fn if_lets_matches() {
}
fn shadowing() -> i32 {
let x = 1; // BAD: unused value [NOT DETECTED]
let x = 1; // BAD: unused value
let mut y: i32; // BAD: unused variable
{
@@ -382,7 +375,7 @@ fn shadowing() -> i32 {
let mut y: i32;
{
let x = 3; // BAD: unused value [NOT DETECTED]
let x = 3; // BAD: unused value
let mut y: i32; // BAD: unused variable
}
@@ -436,7 +429,7 @@ fn folds_and_closures() {
let a5 = 1..10;
_ = a5.fold(0, | acc, val | val); // BAD: unused variable
let i6 = 1;
let i6 = 1; // SPURIOUS: unused value
let a6 = 1..10;
_ = a6.fold(0, | acc, val | acc + val + i6);
}
@@ -449,20 +442,21 @@ fn main() {
structs();
arrays();
statics();
println!("lets use result {}", parameters(1, 2, 3));
println!("lets use result {}", parameters(1, 2, 3));
loops();
if_lets_matches();
shadowing();
func_ptrs();
folds_and_closures();
unreachable_if_1();
unreachable_panic();
unreachable_match();
unreachable_loop();
unreachable_paren();
unreachable_let_1();
unreachable_let_2();
unreachable_if_2();
unreachable_if_3();
unreachable_if_1();
unreachable_panic();
unreachable_match();
unreachable_loop();
unreachable_paren();
unreachable_let_1();
unreachable_let_2();
unreachable_if_2();
unreachable_if_3();
}

119
rust/ql/test/query-tests/unusedentities/more.rs Normal file
View File

@@ -0,0 +1,119 @@
// --- traits ---
trait Incrementable {
fn increment(
&mut self,
times: i32, // SPURIOUS: unused value
unused: i32 // SPURIOUS: unused value
);
}
struct MyValue {
value: i32,
}
impl Incrementable for MyValue {
fn increment(
&mut self,
times: i32,
unused: i32 // BAD: unused variable [NOT DETECTED] SPURIOUS: unused value
) {
self.value += times;
}
}
fn traits() {
let mut i = MyValue { value: 0 };
let a = 1;
let b = 2;
i.increment(a, b);
}
// --- generics ---
trait MySettable<T> {
fn set(&mut self, val: T); // SPURIOUS: unused value
}
trait MyGettable<T> {
fn get(&self, val: T) -> &T; // SPURIOUS: unused value
}
struct MyContainer<T> {
val: T
}
impl<T> MySettable<T> for MyContainer<T> {
fn set(&mut self, val: T) {
self.val = val;
}
}
impl<T> MyGettable<T> for MyContainer<T> {
fn get(
&self,
val: T // BAD: unused variable [NOT DETECTED] SPURIOUS: unused value
) -> &T {
return &(self.val);
}
}
fn generics() {
let mut a = MyContainer { val: 1 }; // BAD: unused value [NOT DETECTED]
let b = MyContainer { val: 2 };
a.set(
*b.get(3)
);
}
// --- pointers ---
fn pointers() {
let a = 1;
let a_ptr1 = &a;
let a_ptr2 = &a;
let a_ptr3 = &a; // BAD: unused value [NOT DETECTED]
let a_ptr4 = &a; // BAD: unused value
println!("{}", *a_ptr1);
println!("{}", a_ptr2);
println!("{}", &a_ptr3);
let b = 2; // BAD: unused value [NOT DETECTED]
let b_ptr = &b;
println!("{}", b_ptr);
let c = 3;
let c_ptr = &c;
let c_ptr_ptr = &c_ptr;
println!("{}", **c_ptr_ptr);
let d = 4;
let d_ptr = &d; // BAD: unused value
let d_ptr_ptr = &&d;
println!("{}", **d_ptr_ptr);
let e = 5; // BAD: unused value [NOT DETECTED]
let f = 6;
let mut f_ptr = &e; // BAD: unused value
f_ptr = &f;
println!("{}", *f_ptr);
let mut g = 7; // BAD: unused value [NOT DETECTED]
let g_ptr = &mut g;
*g_ptr = 77; // BAD: unused value [NOT DETECTED]
let mut h = 8; // BAD: unused value [NOT DETECTED]
let h_ptr = &mut h;
*h_ptr = 88;
println!("{}", h);
let mut i = 9; // BAD: unused value [NOT DETECTED]
let i_ptr = &mut i;
*i_ptr = 99;
let i_ptr2 = &mut i;
println!("{}", *i_ptr2);
}

18
shared/dataflow/codeql/dataflow/internal/DataFlowImpl.qll
View File

@@ -224,6 +224,13 @@ module MakeImpl<LocationSig Location, InputSig<Location> Lang> {
hasFilteredSource()
)
}
bindingset[source, sink]
pragma[inline_late]
predicate isRelevantSourceSinkPair(Node source, Node sink) {
isFilteredSource(source) or
isFilteredSink(sink)
}
}
private import SourceSinkFiltering
@@ -3511,6 +3518,17 @@ module MakeImpl<LocationSig Location, InputSig<Location> Lang> {
* included in the module `PathGraph`.
*/
predicate flowPath(PathNode source, PathNode sink) {
(
// When there are both sources and sinks in the diff range,
// diff-informed dataflow falls back to computing all paths without
// any filtering. To prevent significant alert flip-flopping due to
// minor code changes triggering the fallback, we consistently apply
// source-or-sink filtering here to ensure that we return the same
// paths regardless of whether the fallback is triggered.
if Config::observeDiffInformedIncrementalMode()
then isRelevantSourceSinkPair(source.getNode(), sink.getNode())
else any()
) and
exists(PathNodeImpl flowsource, PathNodeImpl flowsink |
source = flowsource and sink = flowsink
|