From 81e71c4669e291336a9d3a8e19df760f847dddef Mon Sep 17 00:00:00 2001 From: Ian Lynagh Date: Tue, 7 Mar 2023 15:29:01 +0000 Subject: [PATCH 1/5] Kotlin: Add a test for double niterception --- .../kotlin_double_interception/code/build.py | 30 +++++++++++++++++++ .../code/doubleIntercepted.kt | 0 .../kotlin_double_interception/code/manual.kt | 0 .../code/manuallyIntercepted.kt | 0 .../kotlin_double_interception/code/normal.kt | 0 .../code/notSeen.kt | 0 .../kotlin_double_interception/files.expected | 2 ++ .../kotlin_double_interception/files.ql | 4 +++ .../kotlin/kotlin_double_interception/test.py | 7 +++++ 9 files changed, 43 insertions(+) create mode 100644 java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/build.py create mode 100644 java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/doubleIntercepted.kt create mode 100644 java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/manual.kt create mode 100644 java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/manuallyIntercepted.kt create mode 100644 java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/normal.kt create mode 100644 java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/notSeen.kt create mode 100644 java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/files.expected create mode 100644 java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/files.ql create mode 100644 java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/test.py diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/build.py b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/build.py new file mode 100644 index 00000000000..efcb24799b1 --- /dev/null +++ b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/build.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +import glob +import re +from create_database_utils import * + +# This is a normal intercepted compilation +runSuccessfully([get_cmd('kotlinc'), 'normal.kt']) + +# Find the extractor jar that is being used +trapDir = os.environ['CODEQL_EXTRACTOR_JAVA_TRAP_DIR'] +invocationTraps = glob.glob('invocations/*.trap', root_dir = trapDir) +if len(invocationTraps) != 1: + raise Exception('Expected to find 1 invocation TRAP, but found ' + str(invocationTraps)) +invocationTrap = trapDir + '/' + invocationTraps[0] +with open(invocationTrap, 'r') as f: + content = f.read() + m = re.search('^// Using extractor: (.*)$', content, flags = re.MULTILINE) + extractorJar = m.group(1) + +def getManualFlags(invocationTrapName): + return ['-Xplugin=' + extractorJar, '-P', 'plugin:kotlin-extractor:invocationTrapFile=' + trapDir + '/invocations/' + invocationTrapName + '.trap'] + +# This is both normally intercepted, and it has the extractor flags manually added +runUnsuccessfully([get_cmd('kotlinc'), 'doubleIntercepted.kt'] + getManualFlags('doubleIntercepted')) +os.environ['CODEQL_EXTRACTOR_JAVA_AGENT_DISABLE_KOTLIN'] = 'true' +# We don't see this compilation at all +runSuccessfully([get_cmd('kotlinc'), 'notSeen.kt']) +# This is extracted as it has the extractor flags manually added +runSuccessfully([get_cmd('kotlinc'), 'manual.kt'] + getManualFlags('manual')) diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/doubleIntercepted.kt b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/doubleIntercepted.kt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/manual.kt b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/manual.kt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/manuallyIntercepted.kt b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/manuallyIntercepted.kt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/normal.kt b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/normal.kt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/notSeen.kt b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/notSeen.kt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/files.expected b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/files.expected new file mode 100644 index 00000000000..a8cf3381706 --- /dev/null +++ b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/files.expected @@ -0,0 +1,2 @@ +| code/manual.kt:0:0:0:0 | manual | +| code/normal.kt:0:0:0:0 | normal | diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/files.ql b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/files.ql new file mode 100644 index 00000000000..c6d9064f1e9 --- /dev/null +++ b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/files.ql @@ -0,0 +1,4 @@ +import java + +from File f +select f diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/test.py b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/test.py new file mode 100644 index 00000000000..9abcb43e304 --- /dev/null +++ b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/test.py @@ -0,0 +1,7 @@ +import sys + +from create_database_utils import * + +run_codeql_database_create( + ['"%s" build.py' % sys.executable], + source="code", lang="java") From 8b6047dfd101822f6bb89304de5344ce58d4fc62 Mon Sep 17 00:00:00 2001 From: Ian Lynagh Date: Tue, 7 Mar 2023 15:56:04 +0000 Subject: [PATCH 2/5] Kotlin: Handle double-interceptions without failing --- .../kotlin/kotlin_double_interception/code/build.py | 2 +- .../kotlin/kotlin_double_interception/files.expected | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/build.py b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/build.py index efcb24799b1..18fe2153140 100644 --- a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/build.py +++ b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/build.py @@ -22,7 +22,7 @@ def getManualFlags(invocationTrapName): return ['-Xplugin=' + extractorJar, '-P', 'plugin:kotlin-extractor:invocationTrapFile=' + trapDir + '/invocations/' + invocationTrapName + '.trap'] # This is both normally intercepted, and it has the extractor flags manually added -runUnsuccessfully([get_cmd('kotlinc'), 'doubleIntercepted.kt'] + getManualFlags('doubleIntercepted')) +runSuccessfully([get_cmd('kotlinc'), 'doubleIntercepted.kt'] + getManualFlags('doubleIntercepted')) os.environ['CODEQL_EXTRACTOR_JAVA_AGENT_DISABLE_KOTLIN'] = 'true' # We don't see this compilation at all runSuccessfully([get_cmd('kotlinc'), 'notSeen.kt']) diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/files.expected b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/files.expected index a8cf3381706..bc69208b7e8 100644 --- a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/files.expected +++ b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/files.expected @@ -1,2 +1,3 @@ +| code/doubleIntercepted.kt:0:0:0:0 | doubleIntercepted | | code/manual.kt:0:0:0:0 | manual | | code/normal.kt:0:0:0:0 | normal | From fae4a8f37b045ac355fd76920384fa1c1065b898 Mon Sep 17 00:00:00 2001 From: Ian Lynagh Date: Tue, 7 Mar 2023 17:20:09 +0000 Subject: [PATCH 3/5] Kotlin: double interception test: Fix for old python versions --- .../kotlin/kotlin_double_interception/code/build.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/build.py b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/build.py index 18fe2153140..d9ebd2aefa0 100644 --- a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/build.py +++ b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/build.py @@ -9,10 +9,11 @@ runSuccessfully([get_cmd('kotlinc'), 'normal.kt']) # Find the extractor jar that is being used trapDir = os.environ['CODEQL_EXTRACTOR_JAVA_TRAP_DIR'] -invocationTraps = glob.glob('invocations/*.trap', root_dir = trapDir) +invocationTrapDir = trapDir + '/invocations' +invocationTraps = os.listdir(invocationTrapDir) if len(invocationTraps) != 1: raise Exception('Expected to find 1 invocation TRAP, but found ' + str(invocationTraps)) -invocationTrap = trapDir + '/' + invocationTraps[0] +invocationTrap = invocationTrapDir + '/' + invocationTraps[0] with open(invocationTrap, 'r') as f: content = f.read() m = re.search('^// Using extractor: (.*)$', content, flags = re.MULTILINE) From fd8f7e071b1ed41ecdfe68e5ac6afdeacbfa6d32 Mon Sep 17 00:00:00 2001 From: Ian Lynagh Date: Thu, 9 Mar 2023 14:17:23 +0000 Subject: [PATCH 4/5] Kotlin: Tweak double_interception test --- .../kotlin_double_interception/code/build.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/build.py b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/build.py index d9ebd2aefa0..45ab2315b54 100644 --- a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/build.py +++ b/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/build.py @@ -1,31 +1,41 @@ #!/usr/bin/env python3 import glob +import os import re from create_database_utils import * +def say(s): + print(s) + sys.stdout.flush() + +say('Doing normal compilation') # This is a normal intercepted compilation runSuccessfully([get_cmd('kotlinc'), 'normal.kt']) +say('Identifying extractor jar') # Find the extractor jar that is being used trapDir = os.environ['CODEQL_EXTRACTOR_JAVA_TRAP_DIR'] -invocationTrapDir = trapDir + '/invocations' +invocationTrapDir = os.path.join(trapDir, 'invocations') invocationTraps = os.listdir(invocationTrapDir) if len(invocationTraps) != 1: raise Exception('Expected to find 1 invocation TRAP, but found ' + str(invocationTraps)) -invocationTrap = invocationTrapDir + '/' + invocationTraps[0] +invocationTrap = os.path.join(invocationTrapDir, invocationTraps[0]) with open(invocationTrap, 'r') as f: content = f.read() m = re.search('^// Using extractor: (.*)$', content, flags = re.MULTILINE) extractorJar = m.group(1) def getManualFlags(invocationTrapName): - return ['-Xplugin=' + extractorJar, '-P', 'plugin:kotlin-extractor:invocationTrapFile=' + trapDir + '/invocations/' + invocationTrapName + '.trap'] + return ['-Xplugin=' + extractorJar, '-P', 'plugin:kotlin-extractor:invocationTrapFile=' + os.path.join(trapDir, 'invocations', invocationTrapName + '.trap')] # This is both normally intercepted, and it has the extractor flags manually added +say('Doing double-interception compilation') runSuccessfully([get_cmd('kotlinc'), 'doubleIntercepted.kt'] + getManualFlags('doubleIntercepted')) os.environ['CODEQL_EXTRACTOR_JAVA_AGENT_DISABLE_KOTLIN'] = 'true' # We don't see this compilation at all +say('Doing unseen compilation') runSuccessfully([get_cmd('kotlinc'), 'notSeen.kt']) # This is extracted as it has the extractor flags manually added +say('Doing manual compilation') runSuccessfully([get_cmd('kotlinc'), 'manual.kt'] + getManualFlags('manual')) From 4fbc747f93122c56d88c879ad0a189b82df903bf Mon Sep 17 00:00:00 2001 From: Ian Lynagh Date: Thu, 9 Mar 2023 17:08:34 +0000 Subject: [PATCH 5/5] Kotlin: Move kotlin_double_interception test to posix_only It's failing on Windows --- .../kotlin/kotlin_double_interception/code/build.py | 0 .../kotlin/kotlin_double_interception/code/doubleIntercepted.kt | 0 .../kotlin/kotlin_double_interception/code/manual.kt | 0 .../kotlin/kotlin_double_interception/code/manuallyIntercepted.kt | 0 .../kotlin/kotlin_double_interception/code/normal.kt | 0 .../kotlin/kotlin_double_interception/code/notSeen.kt | 0 .../kotlin/kotlin_double_interception/files.expected | 0 .../kotlin/kotlin_double_interception/files.ql | 0 .../kotlin/kotlin_double_interception/test.py | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename java/ql/integration-tests/{all-platforms => posix-only}/kotlin/kotlin_double_interception/code/build.py (100%) rename java/ql/integration-tests/{all-platforms => posix-only}/kotlin/kotlin_double_interception/code/doubleIntercepted.kt (100%) rename java/ql/integration-tests/{all-platforms => posix-only}/kotlin/kotlin_double_interception/code/manual.kt (100%) rename java/ql/integration-tests/{all-platforms => posix-only}/kotlin/kotlin_double_interception/code/manuallyIntercepted.kt (100%) rename java/ql/integration-tests/{all-platforms => posix-only}/kotlin/kotlin_double_interception/code/normal.kt (100%) rename java/ql/integration-tests/{all-platforms => posix-only}/kotlin/kotlin_double_interception/code/notSeen.kt (100%) rename java/ql/integration-tests/{all-platforms => posix-only}/kotlin/kotlin_double_interception/files.expected (100%) rename java/ql/integration-tests/{all-platforms => posix-only}/kotlin/kotlin_double_interception/files.ql (100%) rename java/ql/integration-tests/{all-platforms => posix-only}/kotlin/kotlin_double_interception/test.py (100%) diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/build.py b/java/ql/integration-tests/posix-only/kotlin/kotlin_double_interception/code/build.py similarity index 100% rename from java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/build.py rename to java/ql/integration-tests/posix-only/kotlin/kotlin_double_interception/code/build.py diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/doubleIntercepted.kt b/java/ql/integration-tests/posix-only/kotlin/kotlin_double_interception/code/doubleIntercepted.kt similarity index 100% rename from java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/doubleIntercepted.kt rename to java/ql/integration-tests/posix-only/kotlin/kotlin_double_interception/code/doubleIntercepted.kt diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/manual.kt b/java/ql/integration-tests/posix-only/kotlin/kotlin_double_interception/code/manual.kt similarity index 100% rename from java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/manual.kt rename to java/ql/integration-tests/posix-only/kotlin/kotlin_double_interception/code/manual.kt diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/manuallyIntercepted.kt b/java/ql/integration-tests/posix-only/kotlin/kotlin_double_interception/code/manuallyIntercepted.kt similarity index 100% rename from java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/manuallyIntercepted.kt rename to java/ql/integration-tests/posix-only/kotlin/kotlin_double_interception/code/manuallyIntercepted.kt diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/normal.kt b/java/ql/integration-tests/posix-only/kotlin/kotlin_double_interception/code/normal.kt similarity index 100% rename from java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/normal.kt rename to java/ql/integration-tests/posix-only/kotlin/kotlin_double_interception/code/normal.kt diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/notSeen.kt b/java/ql/integration-tests/posix-only/kotlin/kotlin_double_interception/code/notSeen.kt similarity index 100% rename from java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/code/notSeen.kt rename to java/ql/integration-tests/posix-only/kotlin/kotlin_double_interception/code/notSeen.kt diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/files.expected b/java/ql/integration-tests/posix-only/kotlin/kotlin_double_interception/files.expected similarity index 100% rename from java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/files.expected rename to java/ql/integration-tests/posix-only/kotlin/kotlin_double_interception/files.expected diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/files.ql b/java/ql/integration-tests/posix-only/kotlin/kotlin_double_interception/files.ql similarity index 100% rename from java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/files.ql rename to java/ql/integration-tests/posix-only/kotlin/kotlin_double_interception/files.ql diff --git a/java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/test.py b/java/ql/integration-tests/posix-only/kotlin/kotlin_double_interception/test.py similarity index 100% rename from java/ql/integration-tests/all-platforms/kotlin/kotlin_double_interception/test.py rename to java/ql/integration-tests/posix-only/kotlin/kotlin_double_interception/test.py