From 8322e3114897b1cbebcf48a227a3bb8318c2eed4 Mon Sep 17 00:00:00 2001 From: Ian Lynagh Date: Tue, 20 Aug 2024 14:56:16 +0100 Subject: [PATCH] KE2: Copy Kotlin extractor 1 to start Kotlin extractor 2 Sans deps. --- java/kotlin-extractor2/BUILD.bazel | 204 + java/kotlin-extractor2/build.gradle | 71 + java/kotlin-extractor2/defaults/BUILD.bazel | 30 + java/kotlin-extractor2/deps.bzl | 124 + java/kotlin-extractor2/dev/.gitignore | 1 + java/kotlin-extractor2/dev/kotlin | 3 + java/kotlin-extractor2/dev/kotlin.bat | 4 + java/kotlin-extractor2/dev/kotlinc | 3 + java/kotlin-extractor2/dev/kotlinc.bat | 4 + java/kotlin-extractor2/dev/wrapper.py | 183 + java/kotlin-extractor2/generate_dbscheme.py | 183 + java/kotlin-extractor2/gradle.properties | 8 + java/kotlin-extractor2/pick-kotlin-version.py | 41 + java/kotlin-extractor2/settings.gradle | 8 + .../semmle/extractor/java/OdasaOutput.java | 741 ++ .../semmle/extractor/java/PopulateFile.java | 152 + .../java/com/semmle/util/array/ArrayUtil.java | 246 + .../com/semmle/util/basic/ObjectUtil.java | 73 + .../semmle/util/concurrent/LockDirectory.java | 395 + .../semmle/util/concurrent/ThreadUtil.java | 43 + .../java/com/semmle/util/data/IntRef.java | 19 + .../main/java/com/semmle/util/data/Pair.java | 62 + .../com/semmle/util/data/StringDigestor.java | 173 + .../java/com/semmle/util/data/StringUtil.java | 1247 +++ .../java/com/semmle/util/data/Tuple1.java | 106 + .../java/com/semmle/util/data/Tuple2.java | 93 + .../java/com/semmle/util/data/TupleN.java | 85 + .../util/exception/CatastrophicError.java | 117 + .../com/semmle/util/exception/Exceptions.java | 120 + .../util/exception/InterruptedError.java | 26 + .../semmle/util/exception/NestedError.java | 47 + .../semmle/util/exception/ResourceError.java | 30 + .../com/semmle/util/exception/UserError.java | 46 + .../util/expansion/ExpansionEnvironment.java | 893 ++ .../semmle/util/extraction/SpecFileEntry.java | 48 + .../java/com/semmle/util/files/FileUtil.java | 1896 ++++ .../com/semmle/util/files/PathMatcher.java | 160 + .../semmle/util/io/BufferedLineReader.java | 103 + .../com/semmle/util/io/RawStreamMuncher.java | 34 + .../com/semmle/util/io/StreamMuncher.java | 49 + .../java/com/semmle/util/io/StreamUtil.java | 201 + .../main/java/com/semmle/util/io/WholeIO.java | 548 + .../com/semmle/util/io/csv/CSVParser.java | 207 + .../com/semmle/util/io/csv/CSVReader.java | 192 + .../com/semmle/util/io/csv/CSVWriter.java | 226 + .../java/com/semmle/util/logging/Streams.java | 101 + .../util/process/AbstractProcessBuilder.java | 398 + .../java/com/semmle/util/process/Builder.java | 81 + .../java/com/semmle/util/process/Env.java | 699 ++ .../semmle/util/process/LeakPrevention.java | 95 + .../util/projectstructure/ProjectLayout.java | 529 + .../util/trap/CompressedFileInputStream.java | 29 + .../util/trap/dependencies/TextFile.java | 125 + .../trap/dependencies/TrapDependencies.java | 109 + .../util/trap/dependencies/TrapSet.java | 196 + .../pathtransformers/NoopTransformer.java | 8 + .../pathtransformers/PathTransformer.java | 54 + .../ProjectLayoutTransformer.java | 37 + .../com/semmle/util/unicode/UTF8Util.java | 52 + .../util/zip/MultiMemberGZIPInputStream.java | 71 + .../src/main/kotlin/ExternalDeclExtractor.kt | 204 + .../KotlinExtractorCommandLineProcessor.kt | 100 + .../KotlinExtractorComponentRegistrar.kt | 29 + .../main/kotlin/KotlinExtractorExtension.kt | 584 ++ .../src/main/kotlin/KotlinFileExtractor.kt | 9210 +++++++++++++++++ .../src/main/kotlin/KotlinUsesExtractor.kt | 2251 ++++ .../src/main/kotlin/Label.kt | 21 + .../src/main/kotlin/LinesOfCode.kt | 30 + .../src/main/kotlin/LinesOfCodePSI.kt | 150 + .../src/main/kotlin/MetaAnnotationSupport.kt | 513 + .../src/main/kotlin/PrimitiveTypeInfo.kt | 105 + .../src/main/kotlin/TrapWriter.kt | 464 + .../main/kotlin/comments/CommentExtractor.kt | 97 + .../kotlin/comments/CommentExtractorPSI.kt | 122 + .../src/main/kotlin/comments/CommentType.kt | 7 + .../src/main/kotlin/utils/AutoCloseableUse.kt | 47 + .../src/main/kotlin/utils/ClassNames.kt | 150 + .../src/main/kotlin/utils/ExternalDecls.kt | 56 + .../src/main/kotlin/utils/GetByFqName.kt | 26 + .../src/main/kotlin/utils/Helpers.kt | 13 + .../src/main/kotlin/utils/IrVisitorLookup.kt | 40 + .../src/main/kotlin/utils/Iterable.kt | 13 + .../src/main/kotlin/utils/JvmNames.kt | 95 + .../src/main/kotlin/utils/List.kt | 13 + .../src/main/kotlin/utils/Location.kt | 24 + .../src/main/kotlin/utils/Logger.kt | 356 + .../src/main/kotlin/utils/Psi2IrFacade.kt | 12 + .../src/main/kotlin/utils/TypeResults.kt | 40 + .../src/main/kotlin/utils/TypeSubstitution.kt | 283 + .../v_1_5_0/CommentExtractorLighterAST.kt | 15 + .../v_1_5_0/ExperimentalCompilerApi.kt | 4 + .../utils/versions/v_1_5_0/FileEntry.kt | 5 + .../v_1_5_0/FirMetadataSourceFirFile.kt | 1 + .../utils/versions/v_1_5_0/Functions.kt | 8 + .../versions/v_1_5_0/IrSymbolInternals.kt | 3 + .../versions/v_1_5_0/IsUnderscoreParameter.kt | 21 + .../v_1_5_0/JavaBinarySourceElement.kt | 11 + .../versions/v_1_5_0/JvmDefaultModeEnabled.kt | 7 + .../v_1_5_0/Kotlin2ComponentRegistrar.kt | 12 + .../versions/v_1_5_0/LinesOfCodeLighterAST.kt | 18 + .../kotlin/utils/versions/v_1_5_0/Psi2Ir.kt | 5 + .../utils/versions/v_1_5_0/ReferenceEntity.kt | 33 + .../versions/v_1_5_0/SyntheticBodyKind.kt | 5 + .../kotlin/utils/versions/v_1_5_0/Types.kt | 6 + .../kotlin/utils/versions/v_1_5_0/UsesK2.kt | 7 + .../v_1_5_0/allOverriddenIncludingSelf.kt | 6 + .../utils/versions/v_1_5_0/annotationType.kt | 8 + .../kotlin/utils/versions/v_1_5_0/copyTo.kt | 7 + ...rameterDeclarationWithWrappedDescriptor.kt | 7 + .../versions/v_1_5_0/getFileClassFqName.kt | 8 + .../utils/versions/v_1_5_0/getKotlinType.kt | 6 + .../utils/versions/v_1_5_0/packageFqName.kt | 10 + .../kotlin/utils/versions/v_1_5_0/parents.kt | 11 + .../versions/v_1_5_0/withHasQuestionMark.kt | 8 + .../utils/versions/v_1_5_20/FileEntry.kt | 5 + .../kotlin/utils/versions/v_1_5_20/Psi2Ir.kt | 21 + .../utils/versions/v_1_6_0/Functions.kt | 5 + .../utils/versions/v_1_6_0/annotationType.kt | 5 + .../v_1_6_20/IsUnderscoreParameter.kt | 7 + .../kotlin/utils/versions/v_1_6_20/Types.kt | 6 + .../versions/v_1_7_0/getFileClassFqName.kt | 41 + .../utils/versions/v_1_7_0/getKotlinType.kt | 5 + .../versions/v_1_7_0/withHasQuestionMark.kt | 13 + .../v_1_7_20/allOverriddenIncludingSelf.kt | 6 + .../kotlin/utils/versions/v_1_7_20/copyTo.kt | 7 + ...rameterDeclarationWithWrappedDescriptor.kt | 7 + .../v_1_8_0/ExperimentalCompilerApi.kt | 4 + .../utils/versions/v_1_8_0/ReferenceEntity.kt | 35 + .../versions/v_1_8_0/SyntheticBodyKind.kt | 5 + .../CommentExtractorLighterAST.kt | 131 + .../v_1_9_0-Beta/FirMetadataSourceFirFile.kt | 7 + .../v_1_9_0-Beta/Kotlin2ComponentRegistrar.kt | 13 + .../v_1_9_0-Beta/LinesOfCodeLighterAST.kt | 148 + .../utils/versions/v_1_9_0-Beta/UsesK2.kt | 7 + .../versions/v_1_9_20-Beta/packageFqName.kt | 3 + .../v_2_0_0-RC1/FirMetadataSourceFirFile.kt | 7 + .../versions/v_2_0_0-RC1/IrSymbolInternals.kt | 4 + .../v_2_0_0-RC1/JavaBinarySourceElement.kt | 3 + .../v_2_0_0-RC1/JvmDefaultModeEnabled.kt | 7 + .../utils/versions/v_2_0_0-RC1/Psi2Ir.kt | 21 + .../utils/versions/v_2_0_0-RC1/parents.kt | 1 + .../versions/v_2_0_20-Beta2/getKotlinType.kt | 5 + ...otlin.compiler.plugin.CommandLineProcessor | 1 + ....kotlin.compiler.plugin.ComponentRegistrar | 1 + java/kotlin-extractor2/versions.bzl | 47 + 145 files changed, 27694 insertions(+) create mode 100644 java/kotlin-extractor2/BUILD.bazel create mode 100644 java/kotlin-extractor2/build.gradle create mode 100644 java/kotlin-extractor2/defaults/BUILD.bazel create mode 100644 java/kotlin-extractor2/deps.bzl create mode 100644 java/kotlin-extractor2/dev/.gitignore create mode 100755 java/kotlin-extractor2/dev/kotlin create mode 100644 java/kotlin-extractor2/dev/kotlin.bat create mode 100755 java/kotlin-extractor2/dev/kotlinc create mode 100644 java/kotlin-extractor2/dev/kotlinc.bat create mode 100755 java/kotlin-extractor2/dev/wrapper.py create mode 100755 java/kotlin-extractor2/generate_dbscheme.py create mode 100644 java/kotlin-extractor2/gradle.properties create mode 100755 java/kotlin-extractor2/pick-kotlin-version.py create mode 100644 java/kotlin-extractor2/settings.gradle create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/extractor/java/OdasaOutput.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/extractor/java/PopulateFile.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/array/ArrayUtil.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/basic/ObjectUtil.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/concurrent/LockDirectory.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/concurrent/ThreadUtil.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/data/IntRef.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/data/Pair.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/data/StringDigestor.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/data/StringUtil.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/data/Tuple1.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/data/Tuple2.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/data/TupleN.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/exception/CatastrophicError.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/exception/Exceptions.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/exception/InterruptedError.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/exception/NestedError.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/exception/ResourceError.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/exception/UserError.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/expansion/ExpansionEnvironment.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/extraction/SpecFileEntry.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/files/FileUtil.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/files/PathMatcher.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/io/BufferedLineReader.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/io/RawStreamMuncher.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/io/StreamMuncher.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/io/StreamUtil.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/io/WholeIO.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/io/csv/CSVParser.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/io/csv/CSVReader.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/io/csv/CSVWriter.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/logging/Streams.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/process/AbstractProcessBuilder.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/process/Builder.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/process/Env.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/process/LeakPrevention.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/projectstructure/ProjectLayout.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/trap/CompressedFileInputStream.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/trap/dependencies/TextFile.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/trap/dependencies/TrapDependencies.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/trap/dependencies/TrapSet.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/trap/pathtransformers/NoopTransformer.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/trap/pathtransformers/PathTransformer.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/trap/pathtransformers/ProjectLayoutTransformer.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/unicode/UTF8Util.java create mode 100644 java/kotlin-extractor2/src/main/java/com/semmle/util/zip/MultiMemberGZIPInputStream.java create mode 100644 java/kotlin-extractor2/src/main/kotlin/ExternalDeclExtractor.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/KotlinExtractorCommandLineProcessor.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/KotlinExtractorComponentRegistrar.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/KotlinExtractorExtension.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/KotlinFileExtractor.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/KotlinUsesExtractor.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/Label.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/LinesOfCode.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/LinesOfCodePSI.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/MetaAnnotationSupport.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/PrimitiveTypeInfo.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/TrapWriter.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/comments/CommentExtractor.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/comments/CommentExtractorPSI.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/comments/CommentType.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/AutoCloseableUse.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/ClassNames.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/ExternalDecls.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/GetByFqName.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/Helpers.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/IrVisitorLookup.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/Iterable.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/JvmNames.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/List.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/Location.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/Logger.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/Psi2IrFacade.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/TypeResults.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/TypeSubstitution.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/CommentExtractorLighterAST.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/ExperimentalCompilerApi.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/FileEntry.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/FirMetadataSourceFirFile.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/Functions.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/IrSymbolInternals.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/IsUnderscoreParameter.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/JavaBinarySourceElement.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/JvmDefaultModeEnabled.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/Kotlin2ComponentRegistrar.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/LinesOfCodeLighterAST.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/Psi2Ir.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/ReferenceEntity.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/SyntheticBodyKind.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/Types.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/UsesK2.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/allOverriddenIncludingSelf.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/annotationType.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/copyTo.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/createImplicitParameterDeclarationWithWrappedDescriptor.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/getFileClassFqName.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/getKotlinType.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/packageFqName.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/parents.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/withHasQuestionMark.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_20/FileEntry.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_20/Psi2Ir.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_6_0/Functions.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_6_0/annotationType.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_6_20/IsUnderscoreParameter.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_6_20/Types.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_0/getFileClassFqName.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_0/getKotlinType.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_0/withHasQuestionMark.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_20/allOverriddenIncludingSelf.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_20/copyTo.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_20/createImplicitParameterDeclarationWithWrappedDescriptor.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_8_0/ExperimentalCompilerApi.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_8_0/ReferenceEntity.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_8_0/SyntheticBodyKind.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/CommentExtractorLighterAST.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/FirMetadataSourceFirFile.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/Kotlin2ComponentRegistrar.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/LinesOfCodeLighterAST.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/UsesK2.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_20-Beta/packageFqName.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/FirMetadataSourceFirFile.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/IrSymbolInternals.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/JavaBinarySourceElement.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/JvmDefaultModeEnabled.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/Psi2Ir.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/parents.kt create mode 100644 java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_20-Beta2/getKotlinType.kt create mode 100644 java/kotlin-extractor2/src/main/resources/META-INF/services/org.jetbrains.kotlin.compiler.plugin.CommandLineProcessor create mode 100644 java/kotlin-extractor2/src/main/resources/META-INF/services/org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar create mode 100644 java/kotlin-extractor2/versions.bzl diff --git a/java/kotlin-extractor2/BUILD.bazel b/java/kotlin-extractor2/BUILD.bazel new file mode 100644 index 00000000000..f95661f8128 --- /dev/null +++ b/java/kotlin-extractor2/BUILD.bazel @@ -0,0 +1,204 @@ +""" +# Usage overview +Building the extractor can be done with bazel. If building from the internal repository, it is recommended to use +`tools/bazel` from there. + +A specific kotlin extractor variant can be built with +``` +bazel build @codeql//java/kotlin-extractor:codeql-extractor-kotlin-- +``` +where `` is either `standalone` or `embeddable`, and `` is one of the supported versions. +``` +bazel build @codeql//java/kotlin-extractor +``` +will build a default variant: +* standalone, unless `CODEQL_KOTLIN_SINGLE_VERSION_EMBEDDABLE` is set to true, in which case it will go for embeddable +* the version will be taken as the last supported version less than the version of the currently available `kotlinc`, + or `CODEQL_KOTLIN_SINGLE_VERSION` if set. + +If building from the `codeql` repository, `@codeql` can be skipped. + +It is recommended to use the `kotlinc` wrapper in `dev` (which is also available in `tools` from `semmle-code`), which +takes care about providing a sensible default version and keep the version of the default target up to date. + +If the wrapper is not used and `kotlinc` is updated, bazel won't be aware of it and will therefore keep the same default +version. Possible workarounds for that: +* switch to using the `kotlinc` wrapper in `dev` as mentioned above +* `bazel clean` +* `bazel fetch --force @codeql//java/kotlin-extractor` +* `bazel fetch --force @codeql_kotlin_defaults//:all` (only from `codeql`) +""" + +# This file is used in the `@codeql_kotlin_embeddable` external repo, which means we need to +# reference explicitly @codeql +load( + "@codeql//java/kotlin-extractor:versions.bzl", + "VERSIONS", + "get_compatilibity_sources", + "get_language_version", + "version_less", +) +load("@rules_kotlin//kotlin:core.bzl", "kt_javac_options", "kt_kotlinc_options") +load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") + +package(default_visibility = ["//java/kotlin-extractor:__subpackages__"]) + +_for_embeddable = repo_name().endswith("codeql_kotlin_embeddable") + +_common_extractor_name_prefix = "codeql-extractor-kotlin" + +_extractor_name_prefix = "%s-%s" % ( + _common_extractor_name_prefix, + "embeddable" if _for_embeddable else "standalone", +) + +py_binary( + name = "generate_dbscheme", + srcs = ["generate_dbscheme.py"], +) + +_resources = [ + ( + r, + r[len("src/main/resources/"):], + ) + for r in glob(["src/main/resources/**"]) +] + +kt_javac_options( + name = "javac-options", + release = "8", +) + +[ + ( + kt_kotlinc_options( + name = "kotlinc-options-%s" % v, + include_stdlibs = "none", + jvm_target = "1.8", + language_version = get_language_version(v), + warn = "error", + x_optin = [ + "kotlin.RequiresOptIn", + "org.jetbrains.kotlin.ir.symbols.%s" % + ("IrSymbolInternals" if version_less(v, "2.0.0") else "UnsafeDuringIrConstructionAPI"), + ], + x_suppress_version_warnings = True, + ), + # * extractor.name is different for each version, so we need to put it in different output dirs + # * in order to put it in `resources`, we need to define `resource_strip_prefix` to strip this version + # * `resource_strip_prefix` is unique per jar, so we must also put other resources under the same version prefix + genrule( + name = "resources-%s" % v, + srcs = [src for src, _ in _resources], + outs = [ + "%s/com/github/codeql/extractor.name" % v, + ] + [ + "%s/%s" % (v, target) + for _, target in _resources + ], + cmd = "\n".join([ + "echo %s-%s > $(RULEDIR)/%s/com/github/codeql/extractor.name" % (_extractor_name_prefix, v, v), + ] + [ + "cp $(execpath %s) $(RULEDIR)/%s/%s" % (source, v, target) + for source, target in _resources + ]), + ), + kt_jvm_library( + name = "%s-%s" % (_extractor_name_prefix, v), + srcs = + ["@codeql//java/kotlin-extractor:generated-dbscheme"] + + glob( + [ + "src/**/*.kt", + "src/**/*.java", + ], + exclude = [ + # a specific version is included back by `get_compatibility_sources` + "src/main/kotlin/utils/versions/**", + # this appears if `generated_dbscheme.py` is run manually, while we want the one built by bazel + "src/main/kotlin/KotlinExtractorDbScheme.kt", + ], + ) + get_compatilibity_sources(v, "src/main/kotlin/utils/versions"), + javac_opts = ":javac-options", + kotlinc_opts = ":kotlinc-options-%s" % v, + module_name = "codeql-kotlin-extractor", + # resource_strip_prefix is very nit-picky: the following makes it work from + # `codeql`, `@codeql_kotlin_embeddable` and `semmle-code` + resource_strip_prefix = ( + ("../%s/" % repo_name() if repo_name() else "") + + ("%s/" % package_name() if package_name() else "") + + v + ), + resources = [ + ":resources-%s" % v, + ], + visibility = ["//visibility:public"], + deps = [ + "@kotlin-compiler%s-%s" % ( + "-embeddable" if _for_embeddable else "", + v, + ), + "@kotlin-stdlib-%s" % v, + ], + ), + # if in main repository, alias the embeddable versions from the modified @codeql_kotlin_embeddable repo + alias( + name = "%s-embeddable-%s" % (_common_extractor_name_prefix, v), + actual = "@codeql_kotlin_embeddable//:%s-embeddable-%s" % (_common_extractor_name_prefix, v), + visibility = ["//visibility:public"], + ) if not _for_embeddable else None, + ) + for v in VERSIONS +] + +( + genrule( + name = "generated-dbscheme", + srcs = ["@codeql//java:dbscheme"], + outs = ["KotlinExtractorDbScheme.kt"], + cmd = "$(execpath :generate_dbscheme) $< $@", + tools = [":generate_dbscheme"], + visibility = ["@codeql_kotlin_embeddable//:__pkg__"], + ), + [ + alias( + name = n, + actual = "//java/kotlin-extractor/defaults:%s" % n, + visibility = ["//visibility:public"], + ) + for n in ( + "%s-standalone" % _common_extractor_name_prefix, + "%s-embeddable" % _common_extractor_name_prefix, + _common_extractor_name_prefix, + ) + ], + alias( + name = "kotlin-extractor", + actual = _common_extractor_name_prefix, + visibility = ["//visibility:public"], + ), + filegroup( + name = "many", + srcs = ["%s-%s-%s" % ( + _common_extractor_name_prefix, + variant, + version, + ) for variant in ("standalone", "embeddable") for version in VERSIONS], + visibility = ["//visibility:public"], + ), + genrule( + name = "versions-list", + outs = ["kotlin-versions.list"], + cmd = "\n".join(["cat > $@ << EOF"] + VERSIONS + ["EOF"]), + ), + # these are packed in the extractor pack for running QL tests + filegroup( + name = "version-picker", + srcs = [ + "pick-kotlin-version.py", + ":versions-list", + ], + visibility = ["//visibility:public"], + ), +) if not _for_embeddable else None diff --git a/java/kotlin-extractor2/build.gradle b/java/kotlin-extractor2/build.gradle new file mode 100644 index 00000000000..57229642a92 --- /dev/null +++ b/java/kotlin-extractor2/build.gradle @@ -0,0 +1,71 @@ +plugins { + id 'org.jetbrains.kotlin.jvm' version "${kotlinVersion}" + id 'org.jetbrains.dokka' version '1.4.32' +} + +group 'com.github.codeql' +version '0.0.1' + +dependencies { + implementation "org.jetbrains.kotlin:kotlin-stdlib" + compileOnly("org.jetbrains.kotlin:kotlin-compiler") +} + +repositories { + mavenCentral() +} + +tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).configureEach { + kotlinOptions { + jvmTarget = "1.8" + freeCompilerArgs += "-Xopt-in=kotlin.RequiresOptIn" + // enable the below for building with kotlinVersion=1.4.32: + // languageVersion = "1.5" + } +} + +sourceSets { + main { + kotlin { + // change the excludes for building with other versions. + // Currently 1.7.0 is configured: + excludes = [ + // For 1.7.20-Beta, the below two files should be included, and the corresponding v_1_7_20-Beta ones should be excluded from this list. + //"utils/versions/v_1_4_32/allOverriddenIncludingSelf.kt", + //"utils/versions/v_1_4_32/createImplicitParameterDeclarationWithWrappedDescriptor.kt", + "utils/versions/v_1_4_32/Descriptors.kt", + "utils/versions/v_1_4_32/FileEntry.kt", + "utils/versions/v_1_4_32/Functions.kt", + "utils/versions/v_1_4_32/IsUnderscoreParameter.kt", + "utils/versions/v_1_4_32/Psi2Ir.kt", + "utils/versions/v_1_4_32/Types.kt", + "utils/versions/v_1_4_32/withHasQuestionMark.kt", + + "utils/versions/v_1_5_20/Descriptors.kt", + "utils/versions/v_1_6_0/Descriptors.kt", + + "utils/versions/v_1_7_20-Beta/createImplicitParameterDeclarationWithWrappedDescriptor.kt", + "utils/versions/v_1_7_20-Beta/allOverriddenIncludingSelf.kt", + + "utils/versions/v_1_8_0/ExperimentalCompilerApi.kt", + "utils/versions/v_1_8_0/FirIncompatiblePluginAPI.kt", + ] + } + } +} + +jar { + archiveName = "${OUTPUT_JAR_NAME}" +} + +task getHomeDir { + doLast { + println gradle.gradleHomeDir + } +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(8)) + } +} diff --git a/java/kotlin-extractor2/defaults/BUILD.bazel b/java/kotlin-extractor2/defaults/BUILD.bazel new file mode 100644 index 00000000000..bd7d502b064 --- /dev/null +++ b/java/kotlin-extractor2/defaults/BUILD.bazel @@ -0,0 +1,30 @@ +load("@codeql_kotlin_defaults//:defaults.bzl", "kotlin_extractor_defaults") + +package(default_visibility = ["//java/kotlin-extractor:__pkg__"]) + +_common_extractor_name_prefix = "codeql-extractor-kotlin" + +alias( + name = "%s-standalone" % _common_extractor_name_prefix, + actual = "//java/kotlin-extractor:%s-standalone-%s" % ( + _common_extractor_name_prefix, + kotlin_extractor_defaults.extractor_version, + ), +) + +alias( + name = "%s-embeddable" % _common_extractor_name_prefix, + actual = "//java/kotlin-extractor:%s-embeddable-%s" % ( + _common_extractor_name_prefix, + kotlin_extractor_defaults.extractor_version, + ), +) + +alias( + name = _common_extractor_name_prefix, + actual = "//java/kotlin-extractor:%s-%s-%s" % ( + _common_extractor_name_prefix, + kotlin_extractor_defaults.variant, + kotlin_extractor_defaults.extractor_version, + ), +) diff --git a/java/kotlin-extractor2/deps.bzl b/java/kotlin-extractor2/deps.bzl new file mode 100644 index 00000000000..6e55969f251 --- /dev/null +++ b/java/kotlin-extractor2/deps.bzl @@ -0,0 +1,124 @@ +load("//java/kotlin-extractor:versions.bzl", "VERSIONS") +load("//misc/bazel:lfs.bzl", "lfs_smudge") + +_kotlin_dep_build = """ +load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_import") + +kt_jvm_import( + name = "{name}", + jar = "{name}.jar", + visibility = ["//visibility:public"], +) +""" + +_empty_zip = "PK\005\006\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" + +def _get_dep(repository_ctx, name): + return repository_ctx.path(Label("//java/kotlin-extractor/deps:%s" % name)) + +def _kotlin_dep_impl(repository_ctx): + _, _, name = repository_ctx.name.rpartition("~") + lfs_smudge(repository_ctx, [_get_dep(repository_ctx, name + ".jar")]) + + # for some reason rules_kotlin warns about these jars missing, this is to silence those warnings + repository_ctx.file("empty.zip", _empty_zip) + for jar in ( + "annotations-13.0.jar", + "kotlin-stdlib.jar", + "kotlin-reflect.jar", + "kotlin-script-runtime.jar", + "trove4j.jar", + ): + repository_ctx.symlink("empty.zip", jar) + repository_ctx.file("BUILD.bazel", _kotlin_dep_build.format(name = name)) + +_kotlin_dep = repository_rule( + implementation = _kotlin_dep_impl, +) + +def _walk(dir): + res = [] + next_dirs = [dir] + + # loops must be bounded in starlark + for i in range(100): + current_dirs = next_dirs + next_dirs = [] + for d in current_dirs: + children = d.readdir() + next_dirs.extend([c for c in children if c.is_dir]) + res.extend([c for c in children if not c.is_dir]) + if not next_dirs: + return res + fail("%s directory too deep" % dir) + +def _embeddable_source_impl(repository_ctx): + src_dir = repository_ctx.path(Label("//java/kotlin-extractor:src")) + repository_ctx.watch_tree(src_dir) + for src in _walk(src_dir): + contents = repository_ctx.read(src) + contents = contents.replace( + "import com.intellij", + "import org.jetbrains.kotlin.com.intellij", + ) + repository_ctx.file(str(src).replace(str(src_dir), "src"), contents) + repository_ctx.symlink( + Label("//java/kotlin-extractor:BUILD.bazel"), + "BUILD.bazel", + ) + +_embeddable_source = repository_rule(implementation = _embeddable_source_impl) + +def _get_version(repository_ctx, available = []): + default_version = repository_ctx.getenv("CODEQL_KOTLIN_SINGLE_VERSION") + if default_version: + return default_version + repository_ctx.watch(Label("//java/kotlin-extractor:dev/.kotlinc_version")) + version_picker = repository_ctx.path(Label("//java/kotlin-extractor:pick-kotlin-version.py")) + python = repository_ctx.which("python3") or repository_ctx.which("python") + + # use the kotlinc wrapper as fallback + path = repository_ctx.getenv("PATH") + path_to_add = repository_ctx.path(Label("//java/kotlin-extractor:dev")) + if not path: + path = str(path_to_add) + elif repository_ctx.os.name == "windows": + path = "%s;%s" % (path, path_to_add) + else: + path = "%s:%s" % (path, path_to_add) + res = repository_ctx.execute([python, version_picker] + available, environment = {"PATH": path}) + if res.return_code != 0: + fail(res.stderr) + return res.stdout.strip() + +def _defaults_impl(repository_ctx): + default_version = _get_version(repository_ctx) + default_variant = "standalone" + if repository_ctx.getenv("CODEQL_KOTLIN_SINGLE_VERSION_EMBEDDABLE") in ("true", "1"): + default_variant = "embeddable" + available_version = _get_version(repository_ctx, VERSIONS) + info = struct( + version = default_version, + variant = default_variant, + extractor_version = available_version, + ) + repository_ctx.file( + "defaults.bzl", + "kotlin_extractor_defaults = %s\n" % repr(info), + ) + repository_ctx.file("BUILD.bazel") + +_defaults = repository_rule(implementation = _defaults_impl) + +def _kotlin_deps_impl(module_ctx): + for v in VERSIONS: + for lib in ("compiler", "compiler-embeddable", "stdlib"): + _kotlin_dep(name = "kotlin-%s-%s" % (lib, v)) + _embeddable_source(name = "codeql_kotlin_embeddable") + _defaults(name = "codeql_kotlin_defaults") + return module_ctx.extension_metadata( + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +kotlin_extractor_deps = module_extension(implementation = _kotlin_deps_impl) diff --git a/java/kotlin-extractor2/dev/.gitignore b/java/kotlin-extractor2/dev/.gitignore new file mode 100644 index 00000000000..07cf473f632 --- /dev/null +++ b/java/kotlin-extractor2/dev/.gitignore @@ -0,0 +1 @@ +/.kotlinc_* diff --git a/java/kotlin-extractor2/dev/kotlin b/java/kotlin-extractor2/dev/kotlin new file mode 100755 index 00000000000..cabef3a8b2b --- /dev/null +++ b/java/kotlin-extractor2/dev/kotlin @@ -0,0 +1,3 @@ +#!/bin/bash + +exec -a "$0" "$(dirname "$0")/wrapper.py" kotlin "$@" diff --git a/java/kotlin-extractor2/dev/kotlin.bat b/java/kotlin-extractor2/dev/kotlin.bat new file mode 100644 index 00000000000..8cdf794bd5a --- /dev/null +++ b/java/kotlin-extractor2/dev/kotlin.bat @@ -0,0 +1,4 @@ +@echo off + +python "%~dp0wrapper.py" kotlin %* +exit /b %ERRORLEVEL% diff --git a/java/kotlin-extractor2/dev/kotlinc b/java/kotlin-extractor2/dev/kotlinc new file mode 100755 index 00000000000..c6370743212 --- /dev/null +++ b/java/kotlin-extractor2/dev/kotlinc @@ -0,0 +1,3 @@ +#!/bin/bash + +exec -a "$0" "$(dirname "$0")/wrapper.py" kotlinc "$@" diff --git a/java/kotlin-extractor2/dev/kotlinc.bat b/java/kotlin-extractor2/dev/kotlinc.bat new file mode 100644 index 00000000000..6a8857082a0 --- /dev/null +++ b/java/kotlin-extractor2/dev/kotlinc.bat @@ -0,0 +1,4 @@ +@echo off + +python "%~dp0wrapper.py" kotlinc %* +exit /b %ERRORLEVEL% diff --git a/java/kotlin-extractor2/dev/wrapper.py b/java/kotlin-extractor2/dev/wrapper.py new file mode 100755 index 00000000000..f51db289269 --- /dev/null +++ b/java/kotlin-extractor2/dev/wrapper.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 + +""" +Wrapper script that manages kotlin versions. +Usage: add this directory to your PATH, then +* `kotlin* --select x.y.z` will select the version for the next invocations, checking it actually exists +* `kotlin* --clear` will remove any state of the wrapper (deselecting a previous version selection) +* `kotlinc -version` will print the selected version information. It will not print `JRE` information as a normal + `kotlinc` invocation would do though. In exchange, the invocation incurs no overhead. +* Any other invocation will forward to the selected kotlin tool version, downloading it if necessary. If no version was + previously selected with `--select`, a default will be used (see `DEFAULT_VERSION` below) + +In order to install kotlin, ripunzip will be used if installed, or if running on Windows within `semmle-code` (ripunzip +is available in `resources/lib/windows/ripunzip` then). +""" + +import pathlib +import urllib +import urllib.request +import urllib.error +import argparse +import sys +import platform +import subprocess +import zipfile +import shutil +import io +import os + +DEFAULT_VERSION = "2.0.0" + + +def options(): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("tool") + parser.add_argument("--select") + parser.add_argument("--clear", action="store_true") + parser.add_argument("-version", action="store_true") + return parser.parse_known_args() + + +file_template = "kotlin-compiler-{version}.zip" +url_template = "https://github.com/JetBrains/kotlin/releases/download/v{version}/kotlin-compiler-{version}.zip" +this_dir = pathlib.Path(__file__).resolve().parent +version_file = this_dir / ".kotlinc_version" +install_dir = this_dir / ".kotlinc_installed" +zips_dir = this_dir / ".kotlinc_zips" +windows_ripunzip = ( + this_dir.parents[4] / "resources" / "lib" / "windows" / "ripunzip" / "ripunzip.exe" +) + + +class Error(Exception): + pass + + +class ZipFilePreservingPermissions(zipfile.ZipFile): + def _extract_member(self, member, targetpath, pwd): + if not isinstance(member, zipfile.ZipInfo): + member = self.getinfo(member) + + targetpath = super()._extract_member(member, targetpath, pwd) + + attr = member.external_attr >> 16 + if attr != 0: + os.chmod(targetpath, attr) + return targetpath + + +def get_version(): + try: + return version_file.read_text() + except FileNotFoundError: + return None + + +def install(version: str, quiet: bool): + if quiet: + info_out = subprocess.DEVNULL + info = lambda *args: None + else: + info_out = sys.stderr + info = lambda *args: print(*args, file=sys.stderr) + file = file_template.format(version=version) + url = url_template.format(version=version) + if install_dir.exists(): + shutil.rmtree(install_dir) + install_dir.mkdir() + zips_dir.mkdir(exist_ok=True) + zip = zips_dir / file + + if not zip.exists(): + info(f"downloading {url}") + tmp_zip = zip.with_suffix(".tmp") + with open(tmp_zip, "wb") as out, urllib.request.urlopen(url) as response: + shutil.copyfileobj(response, out) + tmp_zip.rename(zip) + ripunzip = shutil.which("ripunzip") + if ( + ripunzip is None + and platform.system() == "Windows" + and windows_ripunzip.exists() + ): + ripunzip = windows_ripunzip + if ripunzip: + info(f"extracting {zip} using ripunzip") + subprocess.run( + [ripunzip, "unzip-file", zip], + stdout=info_out, + stderr=info_out, + cwd=install_dir, + check=True, + ) + else: + info(f"extracting {zip}") + with ZipFilePreservingPermissions(zip) as archive: + archive.extractall(install_dir) + + +def forward(tool, forwarded_opts): + tool = install_dir / "kotlinc" / "bin" / tool + if platform.system() == "Windows": + tool = tool.with_suffix(".bat") + assert tool.exists(), f"{tool} not found" + cmd = [tool] + forwarded_opts + if platform.system() == "Windows": + # kotlin bat script is pretty sensible to unquoted args on windows + ret = subprocess.run(" ".join(f'"{a}"' for a in cmd)).returncode + sys.exit(ret) + else: + os.execv(cmd[0], cmd) + + +def clear(): + if install_dir.exists(): + print(f"removing {install_dir}", file=sys.stderr) + shutil.rmtree(install_dir) + if version_file.exists(): + print(f"removing {version_file}", file=sys.stderr) + version_file.unlink() + if zips_dir.exists(): + print(f"removing {zips_dir}", file=sys.stderr) + shutil.rmtree(zips_dir) + + +def main(opts, forwarded_opts): + if opts.clear: + clear() + return + current_version = get_version() + if opts.select == "default": + selected_version = DEFAULT_VERSION + elif opts.select is not None: + selected_version = opts.select + else: + selected_version = current_version or DEFAULT_VERSION + if selected_version != current_version: + # don't print information about install procedure unless explicitly using --select + install(selected_version, quiet=opts.select is None) + version_file.write_text(selected_version) + if opts.select and not forwarded_opts and not opts.version: + print(f"selected {selected_version}") + return + if opts.version: + if opts.tool == "kotlinc": + print( + f"info: kotlinc-jvm {selected_version} (codeql dev wrapper)", + file=sys.stderr, + ) + return + forwarded_opts.append("-version") + + forward(opts.tool, forwarded_opts) + + +if __name__ == "__main__": + try: + main(*options()) + except Error as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + except KeyboardInterrupt: + sys.exit(1) diff --git a/java/kotlin-extractor2/generate_dbscheme.py b/java/kotlin-extractor2/generate_dbscheme.py new file mode 100755 index 00000000000..be0c5622ed1 --- /dev/null +++ b/java/kotlin-extractor2/generate_dbscheme.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 + +import re +import sys + +enums = {} +unions = {} +tables = {} + +dbscheme = sys.argv[1] if len(sys.argv) >= 2 else '../ql/lib/config/semmlecode.dbscheme' +output = sys.argv[2] if len(sys.argv) >= 3 else 'src/main/kotlin/KotlinExtractorDbScheme.kt' + +def parse_dbscheme(filename): + with open(filename, 'r') as f: + dbscheme = f.read() + + # Remove comments + dbscheme = re.sub(r'/\*.*?\*/', '', dbscheme, flags=re.DOTALL) + dbscheme = re.sub(r'//[^\r\n]*', '', dbscheme) + + # kind enums + for name, kind, body in re.findall(r'case\s+@([^.\s]*)\.([^.\s]*)\s+of\b(.*?);', + dbscheme, + flags=re.DOTALL): + mapping = [] + for num, typ in re.findall(r'(\d+)\s*=\s*@(\S+)', body): + mapping.append((int(num), typ)) + enums[name] = (kind, mapping) + + # unions + for name, rhs in re.findall(r'@(\w+)\s*=\s*(@\w+(?:\s*\|\s*@\w+)*)', + dbscheme, + flags=re.DOTALL): + typs = re.findall(r'@(\w+)', rhs) + unions[name] = typs + + # tables + for relname, body in re.findall('\n([\w_]+)(\([^)]*\))', + dbscheme, + flags=re.DOTALL): + columns = list(re.findall('(\S+)\s*:\s*([^\s,]+)(?:\s+(ref)|)', body)) + tables[relname] = columns + +parse_dbscheme(dbscheme) + +type_aliases = {} + +for alias, typs in unions.items(): + if len(typs) == 1: + real = typs[0] + if real in type_aliases: + real = type_aliases[real] + type_aliases[alias] = real + +def unalias(t): + return type_aliases.get(t, t) + +type_leaf = set() +type_union = {} + +for name, (kind, mapping) in enums.items(): + s = set() + for num, typ in mapping: + s.add(typ) + type_leaf.add(typ) + type_union[name] = s + +for name, typs in unions.items(): + if name not in type_aliases: + type_union[name] = set(map(unalias, typs)) + +for relname, columns in tables.items(): + for _, db_type, ref in columns: + if db_type[0] == '@' and ref == '': + db_type_name = db_type[1:] + if db_type_name not in enums: + type_leaf.add(db_type_name) + +type_union_of_leaves = {} + +def to_leaves(t): + if t not in type_union_of_leaves: + xs = type_union[t] + leaves = set() + for x in xs: + if x in type_leaf: + leaves.add(x) + else: + to_leaves(x) + leaves.update(type_union_of_leaves[x]) + type_union_of_leaves[t] = leaves + +for t in type_union: + to_leaves(t) + +supertypes = {} +for t in type_leaf: + supers = set() + for sup, s in type_union_of_leaves.items(): + if t in s: + supers.add(sup) + supertypes[t] = supers +for t, leaves in type_union_of_leaves.items(): + supers = set() + for sup, s in type_union_of_leaves.items(): + if t != sup and leaves.issubset(s): + supers.add(sup) + supertypes[t] = supers + +def upperFirst(string): + return string[0].upper() + string[1:] + +def genTable(kt, relname, columns, enum = None, kind = None, num = None, typ = None): + kt.write('fun TrapWriter.write' + upperFirst(relname)) + if kind is not None: + kt.write('_' + typ) + kt.write('(') + for colname, db_type, _ in columns: + if colname != kind: + kt.write(colname + ': ') + if db_type == 'int': + kt.write('Int') + elif db_type == 'float': + kt.write('Double') + elif db_type == 'string': + kt.write('String') + elif db_type == 'date': + kt.write('Date') + elif db_type == 'boolean': + kt.write('Boolean') + elif db_type[0] == '@': + label = db_type[1:] + if label == enum: + label = typ + kt.write('Label') + else: + raise Exception('Bad db_type: ' + db_type) + kt.write(', ') + kt.write(') {\n') + kt.write(' this.writeTrap("' + relname + '(') + comma = '' + for colname, db_type, _ in columns: + kt.write(comma) + if colname == kind: + kt.write(str(num)) + elif db_type == 'string': + kt.write('\\"${this.escapeTrapString(this.truncateString(' + colname + '))}\\"') + elif db_type == 'date': + kt.write('D\\"${' + colname + '}\\"') + else: + kt.write('$' + colname) + comma = ', ' + kt.write(')\\n")\n') + kt.write('}\n') + +with open(output, 'w') as kt: + kt.write('/* Generated by ' + sys.argv[0] + ': Do not edit manually. */\n') + kt.write('package com.github.codeql\n') + kt.write('import java.util.Date\n') + + for relname, columns in tables.items(): + enum = None + for _, db_type, ref in columns: + if db_type[0] == '@' and ref == '': + db_type_name = db_type[1:] + if db_type_name in enums: + enum = db_type_name + if enum is None: + genTable(kt, relname, columns) + else: + (kind, mapping) = enums[enum] + for num, typ in mapping: + genTable(kt, relname, columns, enum, kind, num, typ) + + kt.write('sealed interface AnyDbType\n') + for typ in sorted(supertypes): + kt.write('sealed interface Db' + upperFirst(typ) + ': AnyDbType') + # Sorting makes the output deterministic. + names = sorted(supertypes[typ]) + kt.write(''.join(map(lambda name: ', Db' + upperFirst(name), names))) + kt.write('\n') + for alias in sorted(type_aliases): + kt.write('typealias Db' + upperFirst(alias) + ' = Db' + upperFirst(type_aliases[alias]) + '\n') diff --git a/java/kotlin-extractor2/gradle.properties b/java/kotlin-extractor2/gradle.properties new file mode 100644 index 00000000000..f9cd575cdd3 --- /dev/null +++ b/java/kotlin-extractor2/gradle.properties @@ -0,0 +1,8 @@ +kotlin.code.style=official +kotlinVersion=1.7.21 + +GROUP=com.github.codeql +VERSION_NAME=0.0.1 +POM_DESCRIPTION=CodeQL Kotlin extractor +OUTPUT_JAR_NAME=codeql-extractor-kotlin.jar + diff --git a/java/kotlin-extractor2/pick-kotlin-version.py b/java/kotlin-extractor2/pick-kotlin-version.py new file mode 100755 index 00000000000..d4d85820a8e --- /dev/null +++ b/java/kotlin-extractor2/pick-kotlin-version.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +""" +Script to get currently installed kotlinc version. If a list of available versions is provided as input, +the last version of those lower or equal to the kotlinc version is printed. +""" + +import subprocess +import re +import shutil +import argparse +import sys + + +def version_tuple(v): + v, _, _ = v.partition('-') + return tuple(int(x) for x in v.split(".", 2)) + + +p = argparse.ArgumentParser(description=__doc__, fromfile_prefix_chars='@') +p.add_argument("available_versions", nargs="*", metavar="X.Y.Z") +opts = p.parse_args() + +kotlinc = shutil.which('kotlinc') +if kotlinc is None: + raise Exception("kotlinc not found") +res = subprocess.run([kotlinc, "-version"], text=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) +if res.returncode != 0: + raise Exception(f"kotlinc -version failed: {res.stderr}") +m = re.match(r'.* kotlinc-jvm ([0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z][a-zA-Z0-9]*)?) .*', res.stderr) +if m is None: + raise Exception(f'Cannot detect version of kotlinc (got {res.stderr})') +version = m[1] +if opts.available_versions: + vt = version_tuple(version) + available = sorted(opts.available_versions, key=version_tuple, reverse=True) + for v in available: + if version_tuple(v) <= vt: + print(v) + sys.exit(0) + raise Exception(f'Cannot find an available version for {version}') +print(version) diff --git a/java/kotlin-extractor2/settings.gradle b/java/kotlin-extractor2/settings.gradle new file mode 100644 index 00000000000..fa1b7937da6 --- /dev/null +++ b/java/kotlin-extractor2/settings.gradle @@ -0,0 +1,8 @@ +pluginManagement { + repositories { + mavenCentral() + gradlePluginPortal() + } +} + +rootProject.name = 'codeql-kotlin-extractor' diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/extractor/java/OdasaOutput.java b/java/kotlin-extractor2/src/main/java/com/semmle/extractor/java/OdasaOutput.java new file mode 100644 index 00000000000..ef7df5d4053 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/extractor/java/OdasaOutput.java @@ -0,0 +1,741 @@ +package com.semmle.extractor.java; + +import java.lang.reflect.*; +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import com.github.codeql.Logger; +import static com.github.codeql.ClassNamesKt.getIrElementBinaryName; +import static com.github.codeql.ClassNamesKt.getIrClassVirtualFile; + +import org.jetbrains.kotlin.ir.IrElement; +import org.jetbrains.kotlin.ir.declarations.IrClass; + +import com.intellij.openapi.vfs.VirtualFile; + +import org.jetbrains.kotlin.ir.declarations.IrDeclaration; +import org.jetbrains.kotlin.ir.declarations.IrDeclarationWithName; +import org.jetbrains.org.objectweb.asm.ClassVisitor; +import org.jetbrains.org.objectweb.asm.ClassReader; +import org.jetbrains.org.objectweb.asm.Opcodes; + +import com.semmle.util.concurrent.LockDirectory; +import com.semmle.util.concurrent.LockDirectory.LockingMode; +import com.semmle.util.data.Pair; +import com.semmle.util.exception.CatastrophicError; +import com.semmle.util.exception.NestedError; +import com.semmle.util.exception.ResourceError; +import com.semmle.util.extraction.SpecFileEntry; +import com.semmle.util.files.FileUtil; +import com.semmle.util.io.WholeIO; +import com.semmle.util.process.Env; +import com.semmle.util.process.Env.Var; +import com.semmle.util.trap.dependencies.TrapDependencies; +import com.semmle.util.trap.dependencies.TrapSet; +import com.semmle.util.trap.pathtransformers.PathTransformer; + +import com.github.codeql.Compression; + +public class OdasaOutput { + private final File trapFolder; + private final File sourceArchiveFolder; + + private File currentSourceFile; + private TrapSet trapsCreated; + private TrapDependencies trapDependenciesForSource; + + private SpecFileEntry currentSpecFileEntry; + + // should origin tracking be used? + private final boolean trackClassOrigins; + + private final Logger log; + private final Compression compression; + + /** + * DEBUG only: just use the given file as the root for TRAP, source archive etc + */ + OdasaOutput(File outputRoot, Compression compression, Logger log) { + this.trapFolder = new File(outputRoot, "trap"); + this.sourceArchiveFolder = new File(outputRoot, "src_archive"); + this.trackClassOrigins = false; + this.log = log; + this.compression = compression; + } + + public OdasaOutput(boolean trackClassOrigins, Compression compression, Logger log) { + String trapFolderVar = Env.systemEnv().get("CODEQL_EXTRACTOR_JAVA_TRAP_DIR"); + if (trapFolderVar == null) { + throw new ResourceError("CODEQL_EXTRACTOR_JAVA_TRAP_DIR was not set"); + } + String sourceArchiveVar = Env.systemEnv().get("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR"); + if (sourceArchiveVar == null) { + throw new ResourceError("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR was not set"); + } + this.trapFolder = new File(trapFolderVar); + this.sourceArchiveFolder = new File(sourceArchiveVar); + this.trackClassOrigins = trackClassOrigins; + this.log = log; + this.compression = compression; + } + + public File getTrapFolder() { + return trapFolder; + } + + public boolean getTrackClassOrigins() { + return trackClassOrigins; + } + + /** + * Set the source file that is currently being processed. This may affect + * things like trap and source archive directories, and persists as a + * setting until this method is called again. + * + * @param f the current source file + */ + public void setCurrentSourceFile(File f) { + currentSourceFile = f; + currentSpecFileEntry = entryFor(); + trapsCreated = new TrapSet(); + trapsCreated.addSource(PathTransformer.std().fileAsDatabaseString(f)); + trapDependenciesForSource = null; + } + + /** The output paths for that file, or null if it shouldn't be included */ + private SpecFileEntry entryFor() { + return new SpecFileEntry(trapFolder, sourceArchiveFolder, + Arrays.asList(PathTransformer.std().fileAsDatabaseString(currentSourceFile))); + } + + /* + * Trap sets and dependencies. + */ + + public void writeTrapSet() { + trapsCreated.save(trapSetFor(currentSourceFile).toPath()); + } + + private File trapSetFor(File file) { + return FileUtil.appendAbsolutePath( + currentSpecFileEntry.getTrapFolder(), PathTransformer.std().fileAsDatabaseString(file) + ".set"); + } + + public void addDependency(IrDeclaration sym, String signature) { + String path = trapFilePathForDecl(sym, signature); + trapDependenciesForSource.addDependency(path); + } + + /* + * Source archive. + */ + + /** + * Write the given source file to the right source archive, encoded in UTF-8, + * or do nothing if the file shouldn't be populated. + */ + public void writeCurrentSourceFileToSourceArchive(String contents) { + if (currentSpecFileEntry != null && currentSpecFileEntry.getSourceArchivePath() != null) { + File target = sourceArchiveFileFor(currentSourceFile); + target.getParentFile().mkdirs(); + new WholeIO().write(target, contents); + } + } + + public void writeFileToSourceArchive(File srcFile) { + File target = sourceArchiveFileFor(srcFile); + target.getParentFile().mkdirs(); + String contents = new WholeIO().strictread(srcFile); + new WholeIO().write(target, contents); + } + + private File sourceArchiveFileFor(File file) { + return FileUtil.appendAbsolutePath(currentSpecFileEntry.getSourceArchivePath(), + PathTransformer.std().fileAsDatabaseString(file)); + } + + /* + * Trap file names and paths. + */ + + private static final String CLASSES_DIR = "classes"; + private static final String JARS_DIR = "jars"; + private static final String MODULES_DIR = "modules"; + + private File getTrapFileForCurrentSourceFile() { + if (currentSpecFileEntry == null) + return null; + return trapFileFor(currentSourceFile); + } + + private File getTrapFileForJarFile(File jarFile) { + if (!jarFile.getAbsolutePath().endsWith(".jar")) + return null; + return FileUtil.appendAbsolutePath( + currentSpecFileEntry.getTrapFolder(), + JARS_DIR + "/" + PathTransformer.std().fileAsDatabaseString(jarFile) + ".trap" + + compression.getExtension()); + } + + private File getTrapFileForModule(String moduleName) { + return FileUtil.appendAbsolutePath( + currentSpecFileEntry.getTrapFolder(), + MODULES_DIR + "/" + moduleName + ".trap" + compression.getExtension()); + } + + private File trapFileFor(File file) { + return FileUtil.appendAbsolutePath(currentSpecFileEntry.getTrapFolder(), + PathTransformer.std().fileAsDatabaseString(file) + ".trap" + compression.getExtension()); + } + + private File getTrapFileForDecl(IrElement sym, String signature) { + if (currentSpecFileEntry == null) + return null; + return trapFileForDecl(sym, signature); + } + + private File trapFileForDecl(IrElement sym, String signature) { + return FileUtil.fileRelativeTo(currentSpecFileEntry.getTrapFolder(), + trapFilePathForDecl(sym, signature)); + } + + private String trapFilePathForDecl(IrElement sym, String signature) { + String binaryName = getIrElementBinaryName(sym); + // TODO: Reinstate this? + // if (getTrackClassOrigins()) + // classId += "-" + StringDigestor.digest(sym.getSourceFileId()); + String result = CLASSES_DIR + "/" + + binaryName.replace('.', '/') + + signature + + ".members" + + ".trap" + compression.getExtension(); + return result; + } + + /* + * Trap writers. + */ + + /** + * Get a {@link TrapFileManager} to write members + * about a declaration, or null if the declaration shouldn't be + * populated. + * + * @param sym + * The declaration's symbol, including, in particular, its + * fully qualified + * binary class name. + * @param signature + * Any unique suffix needed to distinguish `sym` from other + * declarations with the same name. + * For functions for example, this means its parameter + * signature. + */ + private TrapFileManager getMembersWriterForDecl(File trap, File trapFileBase, TrapClassVersion trapFileVersion, + IrElement sym, String signature) { + // If the TRAP file already exists then we + // don't need to write it. + if (trap.exists()) { + log.trace("Not rewriting trap file for " + trap.toString() + " as it exists"); + return null; + } + // If the TRAP file was written in the past, and + // then renamed to its trap-old name, then we + // don't need to rewrite it only to rename it + // again. + File trapFileDir = trap.getParentFile(); + File trapOld = new File(trapFileDir, + trap.getName().replace(".trap" + compression.getExtension(), ".trap-old" + compression.getExtension())); + if (trapOld.exists()) { + log.trace("Not rewriting trap file for " + trap.toString() + " as the trap-old exists"); + return null; + } + // Otherwise, if any newer TRAP file has already + // been written then we don't need to write + // anything. + if (trapFileBase != null && trapFileVersion != null && trapFileDir.exists()) { + String trapFileBaseName = trapFileBase.getName(); + + for (File f : FileUtil.list(trapFileDir)) { + String name = f.getName(); + Matcher m = selectClassVersionComponents.matcher(name); + if (m.matches() && m.group(1).equals(trapFileBaseName)) { + TrapClassVersion v = new TrapClassVersion(Integer.valueOf(m.group(2)), Integer.valueOf(m.group(3)), + Long.valueOf(m.group(4)), m.group(5)); + if (v.newerThan(trapFileVersion)) { + log.trace("Not rewriting trap file for " + trap.toString() + " as " + f.toString() + " exists"); + return null; + } + } + } + } + return trapWriter(trap, sym, signature); + } + + private TrapFileManager trapWriter(File trapFile, IrElement sym, String signature) { + if (!trapFile.getName().endsWith(".trap" + compression.getExtension())) + throw new CatastrophicError("OdasaOutput only supports writing to compressed trap files"); + String relative = FileUtil.relativePath(trapFile, currentSpecFileEntry.getTrapFolder()); + trapFile.getParentFile().mkdirs(); + trapsCreated.addTrap(relative); + return concurrentWriter(trapFile, relative, log, sym, signature); + } + + private TrapFileManager concurrentWriter(File trapFile, String relative, Logger log, IrElement sym, + String signature) { + if (trapFile.exists()) + return null; + return new TrapFileManager(trapFile, relative, true, log, sym, signature); + } + + public class TrapFileManager implements AutoCloseable { + + private TrapDependencies trapDependenciesForClass; + private File trapFile; + private IrElement sym; + private String signature; + private boolean hasError = false; + + private TrapFileManager(File trapFile, String relative, boolean concurrentCreation, Logger log, IrElement sym, + String signature) { + trapDependenciesForClass = new TrapDependencies(relative); + this.trapFile = trapFile; + this.sym = sym; + this.signature = signature; + } + + public File getFile() { + return trapFile; + } + + public void addDependency(IrElement dep, String signature) { + trapDependenciesForClass.addDependency(trapFilePathForDecl(dep, signature)); + } + + public void addDependency(IrClass c) { + addDependency(c, ""); + } + + public void close() { + if (hasError) { + return; + } + + writeTrapDependencies(trapDependenciesForClass); + } + + private void writeTrapDependencies(TrapDependencies trapDependencies) { + String dep = trapDependencies.trapFile().replace(".trap" + compression.getExtension(), ".dep"); + trapDependencies.save( + currentSpecFileEntry.getTrapFolder().toPath().resolve(dep)); + } + + public void setHasError() { + hasError = true; + } + } + + /* + * Trap file locking. + */ + + private final Pattern selectClassVersionComponents = Pattern + .compile("(.*)#(-?[0-9]+)\\.(-?[0-9]+)-(-?[0-9]+)-(.*)\\.trap.*"); + + /** + * CAUTION: to avoid the potential for deadlock between multiple + * concurrent extractor processes, + * only one source file {@link TrapLocker} may be open at any time, and the lock + * must be obtained + * before any class file lock. + * + * Trap file extensions (and paths) ensure that source and class file locks are + * distinct. + * + * @return a {@link TrapLocker} for the currently processed source file, which + * must have been + * previously set by a call to + * {@link OdasaOutput#setCurrentSourceFile(File)}. + */ + public TrapLocker getTrapLockerForCurrentSourceFile() { + return new TrapLocker((IrClass) null, null, true); + } + + /** + * CAUTION: to avoid the potential for deadlock between multiple + * concurrent extractor processes, + * only one jar file {@link TrapLocker} may be open at any time, and the lock + * must be obtained + * after any source file lock. Only one jar or class file lock may + * be open at any time. + * + * Trap file extensions (and paths) ensure that source and jar file locks are + * distinct. + * + * @return a {@link TrapLocker} for the trap file corresponding to the given jar + * file. + */ + public TrapLocker getTrapLockerForJarFile(File jarFile) { + return new TrapLocker(jarFile); + } + + /** + * CAUTION: to avoid the potential for deadlock between multiple + * concurrent extractor processes, + * only one module {@link TrapLocker} may be open at any time, and the lock must + * be obtained + * after any source file lock. Only one jar or class file or + * module lock may be open at any time. + * + * Trap file extensions (and paths) ensure that source and module file locks are + * distinct. + * + * @return a {@link TrapLocker} for the trap file corresponding to the given + * module. + */ + public TrapLocker getTrapLockerForModule(String moduleName) { + return new TrapLocker(moduleName); + } + + /** + * CAUTION: to avoid the potential for deadlock between multiple + * concurrent extractor processes, + * only one class file {@link TrapLocker} may be open at any time, and the lock + * must be obtained + * after any source file lock. Only one jar or class file lock may + * be open at any time. + * + * Trap file extensions (and paths) ensure that source and class file locks are + * distinct. + * + * @return a {@link TrapLocker} for the trap file corresponding to the given + * class symbol. + */ + public TrapLocker getTrapLockerForDecl(IrElement sym, String signature, boolean fromSource) { + return new TrapLocker(sym, signature, fromSource); + } + + public class TrapLocker implements AutoCloseable { + private final IrElement sym; + private final File trapFile; + // trapFileBase is used when doing lockless TRAP file writing. + // It is trapFile without the #metadata.trap.gz suffix. + private File trapFileBase = null; + private TrapClassVersion trapFileVersion = null; + private final String signature; + + private TrapLocker(IrElement decl, String signature, boolean fromSource) { + this.sym = decl; + this.signature = signature; + if (sym == null) { + log.error("Null symbol passed for Kotlin TRAP locker"); + trapFile = null; + } else { + File normalTrapFile = getTrapFileForDecl(sym, signature); + // We encode the metadata into the filename, so that the + // TRAP filenames for different metadatas don't overlap. + if (fromSource) + trapFileVersion = new TrapClassVersion(0, 0, 0, "kotlin"); + else + trapFileVersion = TrapClassVersion.fromSymbol(sym, log); + String baseName = normalTrapFile.getName().replace(".trap" + compression.getExtension(), ""); + // If a class has lots of inner classes, then we get lots of files + // in a single directory. This makes our directory listings later slow. + // To avoid this, rather than using files named .../Foo*, we use .../Foo/Foo*. + trapFileBase = new File(new File(normalTrapFile.getParentFile(), baseName), baseName); + trapFile = new File(trapFileBase.getPath() + '#' + trapFileVersion.toString() + ".trap" + + compression.getExtension()); + } + } + + private TrapLocker(File jarFile) { + sym = null; + signature = null; + trapFile = getTrapFileForJarFile(jarFile); + } + + private TrapLocker(String moduleName) { + sym = null; + signature = null; + trapFile = getTrapFileForModule(moduleName); + } + + public TrapFileManager getTrapFileManager() { + if (trapFile != null) { + return getMembersWriterForDecl(trapFile, trapFileBase, trapFileVersion, sym, signature); + } else { + return null; + } + } + + @Override + public void close() { + if (trapFile != null) { + // Now that we have finished writing our TRAP file, we want + // to rename and TRAP file that matches our trapFileBase + // but doesn't have the latest metadata. + // Renaming it to trap-old means that it won't be imported, + // but we can still use its presence to avoid future + // invocations rewriting it, and it means that the information + // is in the TRAP directory if we need it for debugging. + if (sym != null) { + File trapFileDir = trapFileBase.getParentFile(); + String trapFileBaseName = trapFileBase.getName(); + + List> pairs = new LinkedList>(); + for (File f : FileUtil.list(trapFileDir)) { + String name = f.getName(); + Matcher m = selectClassVersionComponents.matcher(name); + if (m.matches()) { + if (m.group(1).equals(trapFileBaseName)) { + TrapClassVersion v = new TrapClassVersion(Integer.valueOf(m.group(2)), + Integer.valueOf(m.group(3)), Long.valueOf(m.group(4)), m.group(5)); + pairs.add(new Pair(f, v)); + } else { + // Everything in this directory should be for the same TRAP file base + log.error("Unexpected sibling " + m.group(1) + " when extracting " + trapFileBaseName); + } + } + } + if (pairs.isEmpty()) { + log.error("Wrote TRAP file, but no TRAP files exist for " + trapFile.getAbsolutePath()); + } else { + Comparator> comparator = new Comparator>() { + @Override + public int compare(Pair p1, Pair p2) { + TrapClassVersion v1 = p1.snd(); + TrapClassVersion v2 = p2.snd(); + if (v1.equals(v2)) { + return 0; + } else if (v1.newerThan(v2)) { + return 1; + } else { + return -1; + } + } + }; + TrapClassVersion latestVersion = Collections.max(pairs, comparator).snd(); + + for (Pair p : pairs) { + if (!latestVersion.equals(p.snd())) { + File f = p.fst(); + File fOld = new File(f.getParentFile(), + f.getName().replace(".trap" + compression.getExtension(), + ".trap-old" + compression.getExtension())); + // We aren't interested in whether or not this succeeds; + // it may fail because a concurrent extractor has already + // renamed it. + f.renameTo(fOld); + } + } + } + } + } + } + } + + /* + * Class version tracking. + */ + + private static class TrapClassVersion { + private int majorVersion; + private int minorVersion; + private long lastModified; + private String extractorName; // May be null if not given + + public int getMajorVersion() { + return majorVersion; + } + + public int getMinorVersion() { + return minorVersion; + } + + public long getLastModified() { + return lastModified; + } + + public String getExtractorName() { + return extractorName; + } + + private TrapClassVersion(int majorVersion, int minorVersion, long lastModified, String extractorName) { + this.majorVersion = majorVersion; + this.minorVersion = minorVersion; + this.lastModified = lastModified; + this.extractorName = extractorName; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof TrapClassVersion) { + TrapClassVersion other = (TrapClassVersion) obj; + return majorVersion == other.majorVersion && minorVersion == other.minorVersion + && lastModified == other.lastModified && extractorName.equals(other.extractorName); + } else { + return false; + } + } + + @Override + public int hashCode() { + int hash = 7; + hash = 31 * hash + majorVersion; + hash = 31 * hash + minorVersion; + hash = 31 * hash + (int) lastModified; + hash = 31 * hash + (extractorName == null ? 0 : extractorName.hashCode()); + return hash; + } + + private boolean newerThan(TrapClassVersion tcv) { + // Classes being compiled from source have major version 0 but should take + // precedence + // over any classes with the same qualified name loaded from the classpath + // in previous or subsequent extractor invocations. + if (tcv.majorVersion == 0 && majorVersion != 0) + return false; + else if (majorVersion == 0 && tcv.majorVersion != 0) + return true; + // Always consider the Kotlin extractor superior to the Java extractor, because + // we may decode and extract + // Kotlin metadata that the Java extractor can't understand: + if (!Objects.equals(tcv.extractorName, extractorName)) { + if (Objects.equals(tcv.extractorName, "kotlin")) + return false; + if (Objects.equals(extractorName, "kotlin")) + return true; + } + // Otherwise, determine precedence in the following order: + // majorVersion, minorVersion, lastModified. + return tcv.majorVersion < majorVersion || + (tcv.majorVersion == majorVersion && tcv.minorVersion < minorVersion) || + (tcv.majorVersion == majorVersion && tcv.minorVersion == minorVersion && + tcv.lastModified < lastModified); + } + + private static Map> jarFileEntryTimeStamps = new HashMap<>(); + + private static Map getZipFileEntryTimeStamps(String path, Logger log) { + try { + Map result = new HashMap<>(); + ZipFile zf = new ZipFile(path); + Enumeration entries = zf.entries(); + while (entries.hasMoreElements()) { + ZipEntry ze = entries.nextElement(); + result.put(ze.getName(), ze.getLastModifiedTime().toMillis()); + } + return result; + } catch (IOException e) { + log.warn("Failed to get entry timestamps from " + path, e); + return null; + } + } + + private static long getVirtualFileTimeStamp(VirtualFile vf, Logger log) { + if (vf.getFileSystem().getProtocol().equals("jar")) { + String[] parts = vf.getPath().split("!/"); + if (parts.length == 2) { + String jarFilePath = parts[0]; + String entryPath = parts[1]; + if (!jarFileEntryTimeStamps.containsKey(jarFilePath)) { + jarFileEntryTimeStamps.put(jarFilePath, getZipFileEntryTimeStamps(jarFilePath, log)); + } + Map entryTimeStamps = jarFileEntryTimeStamps.get(jarFilePath); + if (entryTimeStamps != null) { + Long entryTimeStamp = entryTimeStamps.get(entryPath); + if (entryTimeStamp != null) + return entryTimeStamp; + else + log.warn("Couldn't find timestamp for jar file " + jarFilePath + " entry " + entryPath); + } + } else { + log.warn("Expected JAR-file path " + vf.getPath() + " to have exactly one '!/' separator"); + } + } + + // For all files except for jar files, and a fallback in case of I/O problems + // reading a jar file: + return vf.getTimeStamp(); + } + + private static VirtualFile getVirtualFileIfClass(IrElement e) { + if (e instanceof IrClass) + return getIrClassVirtualFile((IrClass) e); + else + return null; + } + + private static TrapClassVersion fromSymbol(IrElement sym, Logger log) { + VirtualFile vf = getVirtualFileIfClass(sym); + if (vf == null && sym instanceof IrDeclaration) + vf = getVirtualFileIfClass(((IrDeclaration) sym).getParent()); + if (vf == null) + return new TrapClassVersion(-1, 0, 0, null); + + final int[] versionStore = new int[1]; + + try { + // Opcodes has fields called ASM4, ASM5, ... + // We want to use the latest one that there is. + Field asmField = null; + int asmNum = -1; + for (Field f : Opcodes.class.getDeclaredFields()) { + String name = f.getName(); + if (name.startsWith("ASM")) { + try { + int i = Integer.parseInt(name.substring(3)); + if (i > asmNum) { + asmNum = i; + asmField = f; + } + } catch (NumberFormatException ex) { + // Do nothing; this field doesn't have a name of the right format + } + } + } + int asm = asmField.getInt(null); + ClassVisitor versionGetter = new ClassVisitor(asm) { + public void visit(int version, int access, java.lang.String name, java.lang.String signature, + java.lang.String superName, java.lang.String[] interfaces) { + versionStore[0] = version; + } + }; + (new ClassReader(vf.contentsToByteArray())).accept(versionGetter, + ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES); + + return new TrapClassVersion(versionStore[0] & 0xffff, versionStore[0] >> 16, + getVirtualFileTimeStamp(vf, log), "kotlin"); + } catch (IllegalAccessException e) { + log.warn("Failed to read class file version information", e); + return new TrapClassVersion(-1, 0, 0, null); + } catch (IOException e) { + log.warn("Failed to read class file version information", e); + return new TrapClassVersion(-1, 0, 0, null); + } + } + + private boolean isValid() { + return majorVersion >= 0 && minorVersion >= 0; + } + + @Override + public String toString() { + return majorVersion + "." + minorVersion + "-" + lastModified + "-" + extractorName; + } + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/extractor/java/PopulateFile.java b/java/kotlin-extractor2/src/main/java/com/semmle/extractor/java/PopulateFile.java new file mode 100644 index 00000000000..c42f847367f --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/extractor/java/PopulateFile.java @@ -0,0 +1,152 @@ +package com.semmle.extractor.java; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +import com.github.codeql.Label; +import com.github.codeql.DbFile; +import com.github.codeql.TrapWriter; +import com.github.codeql.KotlinExtractorDbSchemeKt; +import com.semmle.util.exception.CatastrophicError; +import com.semmle.util.files.FileUtil; +import com.semmle.util.trap.pathtransformers.PathTransformer; +import kotlin.Unit; + +public class PopulateFile { + + private TrapWriter tw; + private PathTransformer transformer; + public PopulateFile(TrapWriter tw) { + this.tw = tw; + this.transformer = PathTransformer.std(); + } + + private static final String[] keyReplacementMap = new String[127]; + static { + keyReplacementMap['&'] = "&"; + keyReplacementMap['{'] = "{"; + keyReplacementMap['}'] = "}"; + keyReplacementMap['"'] = """; + keyReplacementMap['@'] = "@"; + keyReplacementMap['#'] = "#"; + } + + /** + * Escape a string for use in a TRAP key, by replacing special characters with HTML entities. + *

+ * The given string cannot contain any sub-keys, as the delimiters { and } + * are escaped. + *

+ * To construct a key containing both sub-keys and arbitrary input data, escape the individual parts of + * the key rather than the key as a whole, for example: + * 

+	 * "foo;{" + label.toString() + "};" + escapeKey(data)
+	 *
+ */ + public static String escapeKey(String s) { + StringBuilder sb = null; + int lastIndex = 0; + for (int i = 0; i < s.length(); ++i) { + char ch = s.charAt(i); + switch (ch) { + case '&': + case '{': + case '}': + case '"': + case '@': + case '#': + if (sb == null) { + sb = new StringBuilder(); + } + sb.append(s, lastIndex, i); + sb.append(keyReplacementMap[ch]); + lastIndex = i + 1; + break; + } + } + if (sb != null) { + sb.append(s, lastIndex, s.length()); + return sb.toString(); + } else { + return s; + } + } + + public Label populateFile(File absoluteFile) { + return getFileLabel(absoluteFile, true); + } + + public Label getFileLabel(File absoluteFile, boolean populateTables) { + String databasePath = transformer.fileAsDatabaseString(absoluteFile); + Label result = tw.getLabelFor("@\"" + escapeKey(databasePath) + ";sourcefile" + "\"", label -> { + if(populateTables) { + KotlinExtractorDbSchemeKt.writeFiles(tw, label, databasePath); + populateParents(new File(databasePath), label); + } + return Unit.INSTANCE; + }); + return result; + } + + private Label addFolderTuple(String databasePath) { + Label result = tw.getLabelFor("@\"" + escapeKey(databasePath) + ";folder" + "\""); + KotlinExtractorDbSchemeKt.writeFolders(tw, result, databasePath); + return result; + } + + /** + * Populate the parents of an already-normalised file. The path transformers + * and canonicalisation of {@link PathTransformer#fileAsDatabaseString(File)} will not be + * re-applied to this, so it should only be called after proper normalisation + * has happened. It will fill in all parent folders in the current TRAP file. + */ + private void populateParents(File normalisedFile, Label label) { + File parent = normalisedFile.getParentFile(); + if (parent == null) return; + + Label parentLabel = addFolderTuple(FileUtil.normalisePath(parent.getPath())); + populateParents(parent, parentLabel); + KotlinExtractorDbSchemeKt.writeContainerparent(tw, parentLabel, label); + } + + public Label relativeFileId(File jarFile, String pathWithinJar) { + return getFileInJarLabel(jarFile, pathWithinJar, true); + } + + public Label getFileInJarLabel(File jarFile, String pathWithinJar, boolean populateTables) { + if (pathWithinJar.contains("\\")) + throw new CatastrophicError("Invalid jar path: '" + pathWithinJar + "' should not contain '\\'."); + + String databasePath = transformer.fileAsDatabaseString(jarFile); + if(!populateTables) + return tw.getLabelFor("@\"" + databasePath + "/" + pathWithinJar + ";jarFile\""); + + Label jarFileId = this.populateFile(jarFile); + Label jarFileLocation = tw.getLocation(jarFileId, 0, 0, 0, 0); + KotlinExtractorDbSchemeKt.writeHasLocation(tw, jarFileId, jarFileLocation); + + StringBuilder fullName = new StringBuilder(databasePath); + String[] split = pathWithinJar.split("/"); + Label current = jarFileId; + for (int i = 0; i < split.length; i++) { + String shortName = split[i]; + + fullName.append("/"); + fullName.append(shortName); + Label fileId = tw.getLabelFor("@\"" + fullName + ";jarFile" + "\""); + + boolean file = i == split.length - 1; + if (file) { + KotlinExtractorDbSchemeKt.writeFiles(tw, fileId, fullName.toString()); + } else { + KotlinExtractorDbSchemeKt.writeFolders(tw, fileId, fullName.toString()); + } + KotlinExtractorDbSchemeKt.writeContainerparent(tw, current, fileId); + current = fileId; + } + + return current; + } + +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/array/ArrayUtil.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/array/ArrayUtil.java new file mode 100644 index 00000000000..6553bb5511d --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/array/ArrayUtil.java @@ -0,0 +1,246 @@ +package com.semmle.util.array; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Set; + +import com.semmle.util.basic.ObjectUtil; + +/** + * Convenience methods for manipulating arrays. + */ +public class ArrayUtil +{ + + /** + * A number slightly smaller than the maximum length of an array on most vms. + * This matches the constant in ArrayList. + */ + public static final int MAX_ARRAY_LENGTH = Integer.MAX_VALUE - 8; + + /** + * Comparator for primitive int values. + */ + public static interface IntComparator + { + /** + * Compare ints {@code a} and {@code b}, returning a negative value if {@code a} is 'less' than + * {@code b}, zero if they are equal, otherwise a positive value. + */ + public int compare (int a, int b); + } + + /** + * Find the index of the first occurrence of the given {@code value} in the given {@code array}, + * returning -1 if there is no such element. + */ + public static int findFirst(boolean[] array, boolean value) + { + for(int i=0; i int findFirst(T[] array, T value) + { + for(int i=0; i int findFirstSame(T[] array, T value) + { + for(int i=0; i boolean contains (T element, T ... array) + { + return findFirst(array, element) != -1; + } + + /** + * Construct a new array with length increased by one, containing all elements of a given array + * followed by an additional element. + */ + public static T[] append (T[] array, T element) + { + array = Arrays.copyOf(array, array.length + 1); + array[array.length-1] = element; + return array; + } + + /** + * Construct a new array containing the concatenation of the elements in a number of arrays. + * + * @param arrays The arrays to concatenate; may be null (in which case the result will be null). + * Null elements will be treated as empty arrays. + * @return If {@code arrays} is null, a null array, otherwise a newly allocated array containing + * the elements of every non-null array in {@code arrays} concatenated consecutively. + */ + public static byte[] concatenate (byte[] ... arrays) + { + // Quick break-out if arrays is null + if (arrays == null) { + return null; + } + // Find the total length that will be required + int totalLength = 0; + for(byte[] array : arrays) { + totalLength += array == null ? 0 : array.length; + } + // Allocate a new array for the concatenation + byte[] concatenation = new byte[totalLength]; + // Copy each non-null array into the concatenation + int offset = 0; + for(byte[] array : arrays) { + if (array != null) { + System.arraycopy(array, 0, concatenation, offset, array.length); + offset += array.length; + } + } + + return concatenation; + } + + /** Trivial short-hand for building an array (returns {@code elements} unchanged). */ + public static T[] toArray (T ... elements) + { + return elements; + } + + /** + * Swap two elements in an array. + * + * @param array The array containing the elements to be swapped; must be non-null. + * @param index1 The index of the first element to swap; must be in-bounds. + * @param index2 The index of the second element to swap; must be in-bounds. + * @return The given {@code array}. + */ + public static int[] swap (int[] array, int index1, int index2) + { + int value = array[index1]; + array[index1] = array[index2]; + array[index2] = value; + + return array; + } + + /** + * Returns a fresh Set containing all the elements in the array. + * + * @param + * the class of the objects in the array + * @param array + * the array containing the elements + * @return a Set containing all the elements in the array. + */ + @SafeVarargs + public static Set asSet (T ... array) + { + Set ts = new LinkedHashSet<>(); + Collections.addAll(ts, array); + return ts; + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/basic/ObjectUtil.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/basic/ObjectUtil.java new file mode 100644 index 00000000000..ac329ab1a2d --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/basic/ObjectUtil.java @@ -0,0 +1,73 @@ +package com.semmle.util.basic; + +/** + * Trivial utility methods. + */ +public class ObjectUtil { + + /** Query if {@code object1} and {@code object2} are reference-equal, or both null. */ + public static boolean isSame (Object object1, Object object2) + { + return object1 == object2; // Reference equality comparison is deliberate + } + + /** + * Query if {@code object1} and {@code object2} are both null, or both non-null and equal + * according to {@link Object#equals(Object)} (applied as {@code object1.equals(object2)}). + */ + public static boolean equals (Object object1, Object object2) + { + return object1 == null ? object2 == null : object1.equals(object2); + } + + /** + * Query whether {@code object} is equal to any element in {@code objects}, short-circuiting + * the evaluation if possible. + */ + public static boolean equalsAny (Object object, Object ... objects) + { + // Quick break-out if there are no objects to be equal to + if (objects == null || objects.length == 0) { + return false; + } + // Compare against each object in turn + for(Object other : objects) { + if (equals(object, other)) { + return true; + } + } + + return false; + } + + /** + * Return {@code object1.compareTo(object2)}, but handle the case of null input by returning 0 if + * both objects are null, or 1 if only {@code object1} is null (implying that null is always + * 'greater' than non-null). + */ + public static int compareTo (Comparable object1, T2 object2) + { + if (object1 == null) { + return object2 == null ? 0 : 1; + } + return object1.compareTo(object2); + } + + /** + * Return {@code value} if non-null, otherwise {@code replacement}. + */ + public static T replaceNull (T value, T replacement) + { + return value == null ? replacement : value; + } + + @SafeVarargs + public static T nullCoalesce(T... values) { + for(T value : values) { + if (value != null) { + return value; + } + } + return null; + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/concurrent/LockDirectory.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/concurrent/LockDirectory.java new file mode 100644 index 00000000000..b5cdd79d641 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/concurrent/LockDirectory.java @@ -0,0 +1,395 @@ +package com.semmle.util.concurrent; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.util.LinkedHashMap; +import java.util.Map; + +import com.semmle.util.data.StringDigestor; +import com.semmle.util.exception.CatastrophicError; +import com.semmle.util.exception.ResourceError; +import com.semmle.util.files.FileUtil; +import com.semmle.util.io.WholeIO; + +import com.github.codeql.Logger; +import com.github.codeql.Severity; + +/** + * Helper class to simplify handling of file-system-based inter-process + * locking and mutual exclusion. + * + * Both files and directories can be locked; locks are provided in the + * usual flavours of "shared" and "exclusive", plus a no-op variety to + * help unify code -- see the {@link LockingMode} enum. + * + * Note that each locked file requires one file descriptor to be held open. + * It is vital for clients to avoid creating too many locks, and to release + * locks when possible. + * + * The locks obtained by this class are VM-wide, and cannot be used to + * ensure mutual exclusion between threads of the same VM. Rather, they + * can enforce mutual exclusion between separate VMs trying to acquire + * locks for the same paths. + */ +public class LockDirectory { + private final Logger logger; + + private final File lockDir; + + /** + * An enum describing the possible locking modes. + */ + public enum LockingMode { + /** + * Shared mode: A shared lock can be taken any number of times, but only + * if no exclusive lock is in place. + */ + Shared(true), + /** + * An exclusive lock can only be taken if no other lock is in place; it + * prevents all other locks. + */ + Exclusive(false), + /** + * A dummy mode: Lock and unlock operations are no-ops. + */ + None(true), + ; + + private boolean shared; + + private LockingMode(boolean shared) { + this.shared = shared; + } + + public boolean isShared() { return shared; } + } + + /** + * An internal representation of a locked path. Contains some immutable state: The canonical + * path being locked, and the (derived) lock and status files. When the {@link #lock(LockDirectory.LockingMode, String)} + * method is called, a file descriptor to the lock file is opened; {@link #unlock(LockDirectory.LockingMode)} must be + * called to release it when the lock is no longer required. + * + * This class is not thread-safe, but it is expected that its clients ({@link LockDirectory}) + * enforce thread-safe access to instances. + */ + private class LockFile { + private final String lockedPath; + private final File lockFile; + private final File statusFile; + + private LockingMode mode = null; + private RandomAccessFile lockStream = null; + private FileChannel lockChannel = null; + private FileLock lock = null; + + public LockFile(File f) { + try { + lockedPath = f.getCanonicalPath(); + } catch (IOException e) { + throw new ResourceError("Failed to canonicalise path for locking: " + f, e); + } + String sha = StringDigestor.digest(lockedPath); + lockFile = new File(lockDir, sha); + statusFile = new File(lockDir, sha + ".log"); + } + + /** + * Get the (canonical) path associated with this lock file -- this is the + * path that is being locked. + */ + public String getLockedPath() { + return lockedPath; + } + + /** + * Acquire a lock with the given mode. If this method returns normally, + * the lock has been acquired -- an exception is thrown otherwise. This + * method does not block. + * + * If no exception is thrown, a file descriptor is kept open until + * {@link #unlock(LockDirectory.LockingMode)} is called. + * @param mode The desired locking mode. If {@link LockingMode#None}, this + * operation is a no-op (and does not in fact open a file descriptor). + * @param message A message to be recorded alongside the lock file. This + * is included in the exception message of other processes using this + * infrastructure when the lock acquisition fails. + * @throws CatastrophicError if a lock has already been obtained and not released. + * @throws ResourceError if an exception occurs while obtaining the lock, including + * if it cannot be acquired because another process holds it. + */ + public void lock(LockingMode mode, String message) { + if (mode == LockingMode.None) return; + if (lock != null) + throw new CatastrophicError("Trying to re-lock existing lock for " + lockedPath); + this.mode = mode; + try { + lockStream = new RandomAccessFile(lockFile, "rw"); + lockChannel = lockStream.getChannel(); + tryLock(mode); + new WholeIO().strictwrite(statusFile, mode + " lock acquired for " + lockedPath + ": " + message); + } catch (IOException e) { + throw new ResourceError("Failed to obtain lock for " + lockedPath + " at " + lockFile, e); + } + } + + /** + * Acquire a lock with the given mode. If this method returns normally, + * the lock has been acquired -- an exception is thrown otherwise. This + * method blocks indefinitely while waiting to acquire the lock. + * + * If no exception is thrown, a file descriptor is kept open until + * {@link #unlock(LockDirectory.LockingMode)} is called. + * @param mode The desired locking mode. If {@link LockingMode#None}, this + * operation is a no-op (and does not in fact open a file descriptor). + * @param message A message to be recorded alongside the lock file. This + * is included in the exception message of other processes using this + * infrastructure when the lock acquisition fails. + * @throws ResourceError if an exception occurs while obtaining the lock,. + */ + public void blockingLock(LockingMode mode, String message) { + if (mode == LockingMode.None) return; + if (lock != null) + throw new CatastrophicError("Trying to re-lock existing lock for " + lockedPath); + this.mode = mode; + try { + lockStream = new RandomAccessFile(lockFile, "rw"); + lockChannel = lockStream.getChannel(); + lock = lockChannel.tryLock(0, Long.MAX_VALUE, mode.isShared()); + while (lock == null) { + ThreadUtil.sleep(500, true); + lock = lockChannel.tryLock(0, Long.MAX_VALUE, mode.isShared()); + } + new WholeIO().strictwrite(statusFile, mode + " lock acquired for " + lockedPath + ": " + message); + } catch (IOException e) { + throw new ResourceError("Failed to obtain lock for " + lockedPath + " at " + lockFile, e); + } + } + + /** + * Internal helper method: Try to acquire a particular kind of lock, assuming the + * {@link #lockChannel} has been set up. Throws if acquisition fails, rather than + * blocking. + * @param mode The desired lock mode -- exclusive or shared. + * @throws IOException if acquisition of the lock fails for reasons other than + * an incompatible lock already being held by another process. + * @throws ResourceError if the lock is already held by another process. The exception + * message includes the status string, if it can be determined. + */ + private void tryLock(LockingMode mode) throws IOException { + lock = lockChannel.tryLock(0, Long.MAX_VALUE, mode.isShared()); + if (lock == null) { + String status = new WholeIO().read(statusFile); + throw new ResourceError("Failed to acquire " + mode + " lock for " + lockedPath + "." + + (status == null ? "" : "\nExisting lock message: " + status)); + } + } + + /** + * Release this lock. This will close the file descriptor opened by {@link #lock(LockDirectory.LockingMode, String)}. + * @param mode A mode, which must match the mode passed into {@link #lock(LockDirectory.LockingMode, String)} + * (unless it is {@link LockingMode#None}, in which case the method is a no-op). + * @throws CatastrophicError if the passed mode does not match the one used for locking. + * @throws ResourceError if releasing the lock or clearing up temporary files fails. + */ + public void unlock(LockingMode mode) { + if (mode == LockingMode.None) + return; + if (mode != this.mode) + throw new CatastrophicError("Attempting to unlock " + lockedPath + " with incompatible mode: " + + this.mode + " lock was obtained, but " + mode + " lock is being released."); + release(mode); + } + + private void release(LockingMode mode) { + try { + if (lock != null) + try { + // On Windows, the lockChannel/lockStream prevents the lockFile from being + // deleted. The statusFile should only be written after the lock is held, + // so deleting it before releasing the lock is not expected to fail if the + // lock is exclusive. + // Deleting the lock file may fail, if another process just acquires it + // after we release it. + try { + if (statusFile.exists() && !statusFile.delete()) { + if (!mode.isShared()) throw new ResourceError("Could not clear status file " + statusFile); + } + } finally { + lock.release(); + FileUtil.close(lockStream); + FileUtil.close(lockChannel); + if (!lockFile.delete()) + logger.error("Could not clear lock file " + lockFile + " (it might have been locked by another process)."); + } + } catch (IOException e) { + throw new ResourceError("Couldn't release lock for " + lockedPath, e); + } + } finally { + mode = null; + lockStream = null; + lockChannel = null; + lock = null; + } + } + } + + private static final Map instances = new LinkedHashMap(); + + /** + * Obtain the {@link LockDirectory} instance for a given lock directory. The directory + * in question will be created if it doesn't exist. + * @param lockDir A directory -- must be writable, and will be created if it doesn't + * already exist. + * @return The {@link LockDirectory} instance responsible for the specified lock directory. + * @throws ResourceError if the directory cannot be created, exists as a non-directory + * or cannot be canonicalised. + */ + public static synchronized LockDirectory instance(File lockDir) { + return instance(lockDir, null); + } + + /** + * See {@link #instance(File)}. + * Use this method only if log output should be directed to a custom {@link Logger}. + */ + public static synchronized LockDirectory instance(File lockDir, Logger logger) { + // First try to create the directory -- canonicalisation will fail if it doesn't exist. + try { + FileUtil.mkdirs(lockDir); + } catch(ResourceError e) { + throw new ResourceError("Couldn't ensure lock directory " + lockDir + " exists.", e); + } + + // Canonicalise. + try { + lockDir = lockDir.getCanonicalFile(); + } catch (IOException e) { + throw new ResourceError("Couldn't canonicalise requested lock directory " + lockDir, e); + } + + // Find and return the right instance. + LockDirectory instance = instances.get(lockDir); + if (instance == null) { + instance = new LockDirectory(lockDir, logger); + instances.put(lockDir, instance); + } + return instance; + } + + /** + * A map from canonical locked paths to the associated {@link LockFile} instances. + */ + private final Map locks = new LinkedHashMap(); + + /** + * Create a new instance of {@link LockDirectory}, holding all locks in the + * specified log directory. + * @param lockDir A writable directory in which locks will be stored. + * @param logger The {@link Logger} to use, if non-null. + */ + private LockDirectory(File lockDir, Logger logger) { + this.lockDir = lockDir; + this.logger = logger; + } + + /** + * Acquire a lock of the specified kind for the path represented by the given file. + * The file should exist, and its path should be canonicalisable. + * + * Calling this method keeps one file descriptor open + * @param mode The desired locking mode. If {@link LockingMode#None} is passed, this is a no-op, + * otherwise it determines whether a shared or exclusive lock is acquired. + * @param f The path that should be locked -- does not need to be writable, and will not + * be opened. + * @param message A message describing the purpose of the lock acquisition. This is + * potentially displayed when other processes fail to acquire the lock for the given + * path. + * @throws CatastrophicError if an attempt is made to lock an already locked path. + */ + public synchronized void lock(LockingMode mode, File f, String message) { + if (mode == LockingMode.None) return; + LockFile lock = new LockFile(f); + if (locks.containsKey(lock.getLockedPath())) + throw new CatastrophicError("Trying to lock already locked path " + lock.getLockedPath() + "."); + lock.lock(mode, message); + locks.put(lock.getLockedPath(), lock); + } + + /** + * Acquire a lock of the specified kind for the path represented by the given file. + * The file should exist, and its path should be canonicalisable. This method waits + * indefinitely for the lock to become available. There is no ordering on processes + * that are waiting to acquire the lock in this manner. + * + * Calling this method keeps one file descriptor open + * @param mode The desired locking mode. If {@link LockingMode#None} is passed, this is a no-op, + * otherwise it determines whether a shared or exclusive lock is acquired. + * @param f The path that should be locked -- does not need to be writable, and will not + * be opened. + * @param message A message describing the purpose of the lock acquisition. This is + * potentially displayed when other processes fail to acquire the lock for the given + * path. + */ + public synchronized void blockingLock(LockingMode mode, File f, String message) { + if (mode == LockingMode.None) return; + LockFile lock = new LockFile(f); + if (locks.containsKey(lock.getLockedPath())) + throw new CatastrophicError("Trying to lock already locked path " + lock.getLockedPath() + "."); + lock.blockingLock(mode, message); + locks.put(lock.getLockedPath(), lock); + } + + /** + * Release a lock held on a particular path. + * + * This method closes the file descriptor associated with the lock, freeing related + * resources. + * @param mode the mode of the lock. If it equals {@link LockingMode#None}, this is a no-op; otherwise + * it is expected to match the mode passed to the corresponding {@link #lock(LockingMode, File, String)} + * call. + * @param f The path which should be unlocked. As with {@link #lock(LockingMode, File, String)}, it is + * expected to exist and be canonicalisable. It also must be currently locked. + * @throws CatastrophicError on API contract violation: The path isn't currently locked, or the + * mode doesn't correspond to the mode specified when it was locked. + * @throws ResourceError if something goes wrong while releasing resources. + */ + public synchronized void unlock(LockingMode mode, File f) { + if (!maybeUnlock(mode, f)) + throw new CatastrophicError("Trying to unlock " + new LockFile(f).getLockedPath() + ", but it is not locked."); + } + + /** + * Release a lock that may be held on a particular path. + * + * This method closes the file descriptor associated with the lock, freeing related + * resources. Unlike {@link #unlock(LockingMode, File)}, this method will not throw + * if the specified {@link File} is not locked, making it more suitable for post-exception + * cleanup -- false will be returned in that case. + * @param mode the mode of the lock. If it equals {@link LockingMode#None}, this is a no-op; otherwise + * it is expected to match the mode passed to the corresponding {@link #lock(LockingMode, File, String)} + * call. + * @param f The path which should be unlocked. As with {@link #lock(LockingMode, File, String)}, it is + * expected to exist and be canonicalisable. + * @return true if mode == LockingMode.None, or the unlock operation completed + * successfully; false if the path f isn't currently locked. + * @throws ResourceError if something goes wrong while releasing resources. + */ + public synchronized boolean maybeUnlock(LockingMode mode, File f) { + if (mode == LockingMode.None) return true; + // New instance constructed just to share the logic of computing the canonical path. + LockFile key = new LockFile(f); + LockFile existing = locks.get(key.getLockedPath()); + if (existing == null) + return false; + locks.remove(key.getLockedPath()); + existing.unlock(mode); + return true; + } + + public File getDir(){ return lockDir; } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/concurrent/ThreadUtil.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/concurrent/ThreadUtil.java new file mode 100644 index 00000000000..4ebbf2198ee --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/concurrent/ThreadUtil.java @@ -0,0 +1,43 @@ +package com.semmle.util.concurrent; + +import com.semmle.util.exception.CatastrophicError; +import com.semmle.util.exception.Exceptions; + + +/** + * Utility methods related to Threads. + */ +public enum ThreadUtil +{ + /** Singleton instance of {@link ThreadUtil}. */ + SINGLETON; + + /** + * Sleep for {@code millis} milliseconds. + *

+ * Unlike {@link Thread#sleep(long)} (which is wrapped), this method does not throw an + * {@link InterruptedException}, rather in the event of interruption it either throws an + * {@link CatastrophicError} (if {@code allowInterrupt} is false), or accepts the interruption and + * returns false. + *

+ * + * @return true if a sleep of {@code millis} milliseconds was performed without interruption, or + * false if an interruption occurred. + */ + public static boolean sleep(long millis, boolean allowInterrupt) + { + try { + Thread.sleep(millis); + } + catch (InterruptedException ie) { + if (allowInterrupt) { + Exceptions.ignore(ie, "explicitly permitted interruption"); + return false; + } + else { + throw new CatastrophicError("Interrupted", ie); + } + } + return true; + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/data/IntRef.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/data/IntRef.java new file mode 100644 index 00000000000..ce78ded951a --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/data/IntRef.java @@ -0,0 +1,19 @@ +package com.semmle.util.data; + +/** + * A mutable reference to a primitive int. Specialised to avoid + * boxing. + * + */ +public class IntRef { + private int value; + + public IntRef(int value) { + this.value = value; + } + + public int get() { return value; } + public void set(int value) { this.value = value; } + public void inc() { value++; } + public void add(int val) { value += val; }; +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/data/Pair.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/data/Pair.java new file mode 100644 index 00000000000..7c8ff7b6ae3 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/data/Pair.java @@ -0,0 +1,62 @@ +package com.semmle.util.data; + + +/** + * An (immutable) ordered pair of values. + *

+ * Pairs are compared with structural equality: (x,y) = (x', y') iff x=x' + * and y=y'. + *

+ * + * @param the type of the first component of the pair + * @param the type of the second component of the pair + */ +public class Pair extends Tuple2 +{ + private static final long serialVersionUID = -2871892357006076659L; + + /* + * Constructor and factory + */ + + + /** + * Create a new pair of values + * @param x the first component of the pair + * @param y the second component of the pair + */ + public Pair(X x, Y y) { + super(x, y); + } + + /** + * Create a new pair of values. This behaves identically + * to the constructor, but benefits from type inference + * @param x the first component of the pair + * @param y the second component of the pair + */ + public static Pair make(X x, Y y) { + return new Pair(x, y); + } + + /* + * Getters + */ + + /** + * Get the first component of this pair + * @return the first component of the pair + */ + public X fst() { + return value0(); + } + + /** + * Get the second component of this pair + * @return the second component of the pair + */ + public Y snd() { + return value1(); + } + +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/data/StringDigestor.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/data/StringDigestor.java new file mode 100644 index 00000000000..36ab45a7742 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/data/StringDigestor.java @@ -0,0 +1,173 @@ +package com.semmle.util.data; + +import java.nio.charset.Charset; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +import com.semmle.util.exception.CatastrophicError; + +/** + * Encapsulate the creation of message digests from strings. + * + *

+ * This class acts as a (partial) output stream, until the getDigest() method is + * called. After this the class can no longer be used, except to repeatedly call + * {@link #getDigest()}. + * + *

+ * UTF-8 is used internally as the {@link Charset} for this class when converting Strings to bytes. + */ +public class StringDigestor { + private static final Charset UTF8 = Charset.forName("UTF-8"); + private static final String NULL_STRING = ""; + private static final int CHUNK_SIZE = 32; + + private MessageDigest digest; + private byte[] digestBytes; + private final byte[] buf = new byte[CHUNK_SIZE * 3]; // A Java char becomes at most 3 bytes of UTF-8 + + /** + * Create a StringDigestor using SHA-1, ready to accept data + */ + public StringDigestor() { + this("SHA-1"); + } + + /** + * @param digestAlgorithm the algorithm to use in the internal {@link MessageDigest}. + */ + public StringDigestor(String digestAlgorithm) { + try { + digest = MessageDigest.getInstance(digestAlgorithm); + } catch (NoSuchAlgorithmException e) { + throw new CatastrophicError("StringDigestor failed to find the required digest algorithm: " + digestAlgorithm, e); + } + } + + public void reset() { + if (digestBytes == null) throw new CatastrophicError("API violation: Digestor is not finished."); + digest.reset(); + digestBytes = null; + } + + /** + * Write an object into this digestor. This converts the object to a + * string using toString(), writes the length, and then writes the + * string itself. + */ + public StringDigestor write(Object toAppend) { + String str; + if (toAppend == null) { + str = NULL_STRING; + } else { + str = toAppend.toString(); + } + writeBinaryInt(str.length()); + writeNoLength(str); + return this; + } + + /** + * Write the given string without prefixing it by its length. + */ + public StringDigestor writeNoLength(Object toAppend) { + String s = toAppend.toString(); + int len = s.length(); + int i = 0; + while(i + CHUNK_SIZE < len) { + i = writeUTF8(s, i, i + CHUNK_SIZE); + } + writeUTF8(s, i, len); + return this; + } + + private int writeUTF8(String s, int begin, int end) { + if (digestBytes != null) throw new CatastrophicError("API violation: Digestor is finished."); + byte[] buf = this.buf; + int len = 0; + for(int i = begin; i < end; ++i) { + int c = s.charAt(i); + if (c <= 0x7f) { + buf[len++] = (byte)c; + } else if (c <= 0x7ff) { + buf[len] = (byte)(0xc0 | (c >> 6)); + buf[len+1] = (byte)(0x80 | (c & 0x3f)); + len += 2; + } else if (c < 0xd800 || c > 0xdfff) { + buf[len] = (byte)(0xe0 | (c >> 12)); + buf[len+1] = (byte)(0x80 | ((c >> 6) & 0x3f)); + buf[len+2] = (byte)(0x80 | (c & 0x3f)); + len += 3; + } else if (i + 1 < end) { + int c2 = s.charAt(i + 1); + if (c > 0xdbff || c2 < 0xdc00 || c2 > 0xdfff) { + // Invalid UTF-16 + } else { + c = 0x10000 + ((c - 0xd800) << 10) + (c2 - 0xdc00); + buf[len] = (byte)(0xf0 | (c >> 18)); + buf[len+1] = (byte)(0x80 | ((c >> 12) & 0x3f)); + buf[len+2] = (byte)(0x80 | ((c >> 6) & 0x3f)); + buf[len+3] = (byte)(0x80 | (c & 0x3f)); + len += 4; + ++i; + } + } else { + --end; + break; + } + } + digest.update(buf, 0, len); + return end; + } + + /** + * Write an array of raw bytes to the digestor. This appends the contents + * of the array to the accumulated data used for the digest. + */ + public StringDigestor writeBytes(byte[] data) { + if (digestBytes != null) throw new CatastrophicError("API violation: Digestor is finished."); + digest.update(data); + return this; + } + + /** + * Return the hex-encoded digest as a {@link String}. + * + * Get the digest from the data previously appended using write(Object). + * After this is called, the instance's {@link #write(Object)} and {@link #writeBytes(byte[])} + * methods may no longer be used. + */ + public String getDigest() { + if (digestBytes == null) { + digestBytes = digest.digest(); + } + return StringUtil.toHex(digestBytes); + } + + public static String digest(Object o) { + StringDigestor digestor = new StringDigestor(); + digestor.writeNoLength(o); + return digestor.getDigest(); + } + + /** Compute a git-style SHA for the given string. */ + public static String gitBlobSha(String content) { + byte[] bytes = content.getBytes(UTF8); + return digest("blob " + bytes.length + "\0" + content); + } + + /** + * Convert an int to a byte[4] using its little-endian 32bit representation, and append the + * resulting bytes to the accumulated data used for the digest. + */ + public StringDigestor writeBinaryInt(int i) { + if (digestBytes != null) throw new CatastrophicError("API violation: Digestor is finished."); + byte[] buf = this.buf; + buf[0] = (byte)(i & 0xff); + buf[1] = (byte)((i >>> 8) & 0xff); + buf[2] = (byte)((i >>> 16) & 0xff); + buf[3] = (byte)((i >>> 24) & 0xff); + digest.update(buf, 0, 4); + return this; + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/data/StringUtil.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/data/StringUtil.java new file mode 100644 index 00000000000..4ace04698d1 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/data/StringUtil.java @@ -0,0 +1,1247 @@ +package com.semmle.util.data; + +import java.math.BigDecimal; +import java.nio.charset.Charset; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Random; +import java.util.regex.Pattern; + +import com.semmle.util.exception.CatastrophicError; +import com.semmle.util.exception.Exceptions; + +public class StringUtil { + + private static final Random RANDOM = new Random(); + + private static final DecimalFormat DOUBLE_FORMATTER; + static { + // Specify the root locale to ensure we use the "." decimal separator + DOUBLE_FORMATTER = new DecimalFormat( + "#.######", + new DecimalFormatSymbols(Locale.ROOT) + ); + DecimalFormatSymbols decimalFormatSymbols = DOUBLE_FORMATTER.getDecimalFormatSymbols(); + decimalFormatSymbols.setNaN("NaN"); + decimalFormatSymbols.setInfinity("Infinity"); + DOUBLE_FORMATTER.setDecimalFormatSymbols(decimalFormatSymbols); + } + + public static String box(List strings) { + List lines = new ArrayList<>(); + for (String s : strings) + for (String line : lines(s)) + lines.add(line); + + int length = 0; + for (String s : lines) + length = Math.max(length, s.length()); + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length + 6; i++) + sb.append('*'); + for (String s : lines) { + sb.append("\n* "); + sb.append(s); + for (int i = 0; i < length - s.length(); i++) + sb.append(' '); + sb.append(" *"); + } + sb.append('\n'); + for (int i = 0; i < length + 6; i++) + sb.append('*'); + return sb.toString(); + } + + public static String escapeStringLiteralForRegexp(String literal, String charsToPreserve) { + final String charsToEscape = "(){}[].^$+\\*?"; + StringBuilder buf = new StringBuilder(); + for(int i = 0; i < literal.length(); i++) { + char c = literal.charAt(i); + if(charsToEscape.indexOf(c) != -1 && charsToPreserve.indexOf(c) == -1) { + buf.append("\\").append(c); + } + else { + buf.append(c); + } + } + return buf.toString(); + } + + public static String pad(int minWidth, Padding pad, String s) { + + int length = s.length(); + int toPad = minWidth - length; + + if (toPad > 0) { + int left; + int right; + switch (pad) { + case LEFT: + left = 0; + right = toPad; + break; + case RIGHT: + left = toPad; + right = 0; + break; + case CENTRE: + left = toPad / 2; + right = toPad - left; + break; + default: + throw new CatastrophicError("Unknown padding kind: " + pad); + } + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < left; i++) + sb.append(' '); + sb.append(s); + for (int i = 0; i < right; i++) + sb.append(' '); + + return sb.toString(); + } else + return s; + + } + + public static List pad(Padding pad, Collection strings) { + List result = new ArrayList<>(strings.size()); + int maxWidth = 0; + for (String s : strings) + maxWidth = Math.max(maxWidth, s.length()); + for (String s : strings) + result.add(pad(maxWidth, pad, s)); + return result; + } + + public static List pad(Padding pad, String... strings) { + List result = new ArrayList<>(strings.length); + int maxWidth = 0; + for (String s : strings) + maxWidth = Math.max(maxWidth, s.length()); + for (String s : strings) + result.add(pad(maxWidth, pad, s)); + return result; + } + + public static void padTable(List rows, Padding... pad) { + int width = pad.length; + int[] maxLengths = new int[width]; + for (String[] row : rows) { + if (row.length != width) + throw new CatastrophicError("padTable can only be used with a rectangular table. Expected " + width + + " columns but found row: " + Arrays.toString(row)); + for (int i = 0; i < width; i++) + maxLengths[i] = Math.max(maxLengths[i], row[i].length()); + } + for (String[] row : rows) + for (int i = 0; i < width; i++) + row[i] = pad(maxLengths[i], pad[i], row[i]); + } + + public static String glue(String separator, Iterable values) { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (Object o : values) { + if (first) + first = false; + else + sb.append(separator); + sb.append(o == null ? "" : o.toString()); + } + return sb.toString(); + } + + public static String glue(String separator, Object[] values) { + return glue(separator, Arrays.asList(values)); + } + + public static String glue(String separator, String... values) { + return glue(separator, (Object[]) values); + } + + public static enum Padding { LEFT, RIGHT, CENTRE } + + + /** + * Return a new String with any of the four characters !#:= replaced with a back-slash escaped + * equivalent, and any newline characters replaced by a back-slash n. + *

+ * This allows the String to be used in a .properties file (assuming it does not contain any + * extended unicode characters, which must be converted to unicode escapes). + *

+ *

+ * Note that it does not ensure that the String can be used as a key in a .properties + * file, which requires additional escaping of any spaces. + *

+ * + * @param string The String to escape; must not be null. + * @return The given {@code string} with a back-slash inserted before each instance of any of the + * four characters: #!:= + * @see #escapePropertiesValue(String) + */ + public static String escapePropertiesValue (String string) + { + return string.replace("!", "\\!") + .replace(":", "\\:") + .replace("#", "\\#") + .replace("=", "\\=") + .replace("\n", "\\n"); + } + + /** + * See {@link #escapePropertiesValue(String)}. This method also escapes spaces, so that the + * {@code string} can be used as a .properties key. + * + * @param string The String to escape; must not be null. + * @return The given {@code string} with a back-slash inserted before each instance of any of the + * four characters: #!:= or the space character, and newlines replaced by a backslash n. + */ + public static String escapePropertiesKey (String string) + { + return escapePropertiesValue(string).replace(" ", "\\ "); + } + + /** + * Print a float in a locale-independent way suitable for reading with Double.valueOf(). + */ + public static String printFloat(double value) { + if (Math.abs(value) > 999999999999999.0 && !Double.isInfinite(value)) { + // `DecimalFormat` for `double` loses precision on large numbers, + // printing the least significant digits as all 0. + return DOUBLE_FORMATTER.format(new BigDecimal(value)); + } else { + return DOUBLE_FORMATTER.format(value); + } + } + + public static String escapeHTML(String s) { + if (s == null) return null; + + int length = s.length(); + // initialize a StringBuilder with initial capacity of twice the size of the string, + // except when its size is zero, or when doubling the size causes overflow + StringBuilder sb = new StringBuilder(length * 2 > 0 ? length * 2 : length); + for (int i = 0; i < length; i++) { + char c = s.charAt(i); + switch (c) { + case '<': + sb.append("<"); + break; + case '>': + sb.append(">"); + break; + case '&': + sb.append("&"); + break; + case '"': + sb.append("""); + break; + case '\'': + sb.append("'"); + break; + // be careful with this one (non-breaking white space) + /* + case ' ': + sb.append(" "); + break;*/ + + default: + sb.append(c); + break; + } + } + return sb.toString(); + } + + /** + * Escape special characters in the given string like JSON.stringify does + * (see ECMAScript 5.1, Section 15.12.3). + */ + public static String escapeJSON(String str) { + if (str == null) + return null; + StringBuilder res = new StringBuilder(); + for (int i=0, n=str.length(); i specialMarkdownChars = Arrays.asList( + '\\', '`', '_', '*', '(', ')', '[', ']', '#', '+', '-', '.', '!'); + /** + * Escape special markdown characters in the given string. + */ + public static String escapeMarkdown(String str) { + return escapeMarkdown(specialMarkdownChars, str); + } + + /** + * Escape special markdown characters in the given string. + */ + public static String escapeMarkdown(List specialMarkdownChars, String str) { + if (str == null) + return null; + StringBuilder res = new StringBuilder(); + + boolean escapeOctothorp = true; + for (int i=0, n=str.length(); icols marks the + * start of a new line (and ends up on the new line). For this method, + * "word" means sequence of non-whitespace characters. + * @param text The text that should be wrapped. + * @param cols The number of characters to permit on each line; it is + * only exceeded if there are single words that are longer. + * @return The text with sequences of whitespace before words that would + * exceed the permitted width replaced with '\n'. + */ + public static String wrap(String text, int cols) { + if(text == null) + return null; + List lines = new ArrayList<>(); + int lineStart = -1; int wordStart = -1; int col = 0; + for (int cur = 0; cur < text.length(); cur++) { + if (text.charAt(cur) == '\n') { + // Forced new line. + if (lineStart < 0) { + // Empty new line. + lines.add(""); + } else { + lines.add(text.substring(lineStart, cur).trim()); + } + lineStart = -1; + wordStart = -1; + col = 0; + } else if (Character.isWhitespace(text.charAt(cur))) { + // Possible break. + if (col > cols) { + // Break is needed. + if (lineStart < 0) { + // Long run of whitespace. + continue; + } else if (wordStart < 0) { + // Sequence of whitespace went over after real word. + String line = text.substring(lineStart, cur).trim(); + if (line.length() > 0) lines.add(line); + lineStart = -1; + } else if (wordStart > lineStart) { + // Word goes onto new line. + lines.add(text.substring(lineStart, wordStart - 1).trim()); + lineStart = wordStart; + col = cur - lineStart + 1; + wordStart = -1; + } else { + // Word is a line on its own. + lines.add(text.substring(wordStart, cur).trim()); + lineStart = -1; + wordStart = -1; + } + } else { + // No break, but new word + wordStart = -1; + } + } else { + if (lineStart < 0) { + lineStart = cur; + col = 0; + } + if (wordStart < 0) + wordStart = cur; + } + if (lineStart >= 0) + col++; + } + if (lineStart > -1) + lines.add(text.substring(lineStart).trim()); + return glue("\n", lines); + } + + /** + * Get the first word of the given string, delimited by whitespace. Leading whitespace + * is ignored. + */ + public static String firstWord(String s) { + s = s.trim(); + int i = 0; + while (i < s.length() && !Character.isWhitespace(s.charAt(i))) + i++; + return s.substring(0, i); + } + + /** + * Strip the first word (delimited by whitespace, leading whitespace ignored) and get the + * remainder of the string, trimmed. + */ + public static String stripFirstWord(String s) { + s = s.trim(); + int i = 0; + while (i < s.length() && !Character.isWhitespace(s.charAt(i))) + i++; + return s.substring(i).trim(); + } + + /** + * Trim leading and trailing occurrences of a character from a string + * @param str the string to trim + * @param c the character to remove + * @return A string whose value is str, with any leading and trailing occurrences of c removed, + * or str if it has no leading or trailing occurrences of c. + */ + public static String trim(String str, char c) { + return trim(str, c, true, true); + } + + public static String trim(String str, char c, boolean trimLeading, boolean trimTrailing) { + int begin = 0; + int end = str.length(); + + if (trimLeading) { + while ((begin < end) && (str.charAt(begin) == c)) { + begin++; + } + } + if (trimTrailing) { + while ((begin < end) && (str.charAt(end - 1) == c)) { + end--; + } + } + if ((begin > 0) || (end < str.length())) + str = str.substring(begin, end); + return str; + } + + public static String trimTrailingWhitespace(String str) { + int begin = 0; + int end = str.length(); + while((begin < end) && (Character.isWhitespace(str.charAt(end-1)))) { + end--; + } + if (end < str.length()) + str = str.substring(begin, end); + return str; + } + + private static final Charset UTF8_CHARSET = Charset.forName("UTF-8"); + + /** + * Invert the conversion performed by {@link #stringToBytes(String)}. + */ + public static String bytesToString(byte[] bytes) + { + return new String(bytes, 0, bytes.length, UTF8_CHARSET); + } + + public static String bytesToString(byte[] bytes, int offset, int length) { + return new String(bytes, offset, length, UTF8_CHARSET); + } + + /** Convert a String into a sequence of bytes (according to a UTF-8 representation of the String). */ + public static byte[] stringToBytes (String str) + { + return str.getBytes(Charset.forName("UTF-8")); + } + + /** + * Compute a SHA-1 sum for the given String. + *

+ * The SHA-1 is obtained by first converting the String to bytes, which is Charset-dependent, + * though this method always uses {@link #stringToBytes(String)}. + *

+ * + * @see #toHex(byte[]) + */ + public static byte[] stringToSHA1 (String str) + { + MessageDigest messageDigest; + try { + messageDigest = MessageDigest.getInstance("SHA-1"); + return messageDigest.digest(stringToBytes(str)); + } + catch (NoSuchAlgorithmException e) { + throw new CatastrophicError("Failed to obtain MessageDigest for computing SHA-1", e); + } + + } + + /** + * Constructs a string that repeats the repeatee the specified number of times. + * For example, repeat("foo", 3) would return "foofoofoo". + * + * @param repeatee The string to be repeated. + * @param times The number of times to repeat it. + * @return The result of repeating the repeatee the specified number of times. + */ + public static String repeat(String repeatee, int times) { + if (times == 0) + return ""; + return new String(new char[times]).replace("\0", repeatee); + } + + /** + * Computes the lower-case version of the given string in a way that is independent + * of the system default locale. + * @param s A string value to lowercase. + * @return The value of {@code s} with all English letters converted to lower-case. + */ + public static String lc(String s) { + return s.toLowerCase(Locale.ENGLISH); + } + + /** + * Computes the upper-case version of the given string in a way that is independent + * of the system default locale. + * @param s A string value to uppercase. + * @return The value of {@code s} with all English letters converted to upper-case. + */ + public static String uc(String s) { + return s.toUpperCase(Locale.ENGLISH); + } + + public static String ucfirst(String s) { + if( s.isEmpty() || !Character.isLowerCase(s.charAt(0))) + return s; + else + return uc(s.substring(0,1))+s.substring(1); + } + + private static final Pattern lineEnding = Pattern.compile("\r\n|\r|\n"); + /** + * Regex to match line endings using look-behind, + * so that line separators can be included in the split lines. + * \r\n is matched eagerly, i.e. we only match on \r individually if it is not followed by \n. + */ + private static final Pattern lineEndingIncludingSeparators = Pattern.compile("(?<=(\r\n|\r(?!\n)|\n))"); + + /** + * Get the lines in a given string. All known style of line terminator (CRLF, CR, LF) + * are recognised. Trailing empty lines are not returned, and the resulting strings + * do not include the line separators. + */ + public static String[] lines(String s) { + return lines(s, false, true); + } + + /** + * Get the lines in a given string, including the line separators. + * All known style of line terminator (CRLF, CR, LF) are recognised. + * Trailing empty strings are not returned (but lines containing only separators are). + */ + public static String[] linesWithSeparators(String s) { + return lines(s, true, true); + } + + /** + * Get the lines in a given string. All known style of line terminator (CRLF, CR, LF) + * are recognised. The resulting strings do not include the line separators. If + * {@code squishTrailing} is true, trailing empty lines are not included + * in the result; otherwise, they will appear as empty strings. + */ + public static String[] lines(String s, boolean squishTrailing) { + return lines(s, false, squishTrailing); + } + + /** + * Gets the lines in a given string. All known style of line terminator (CRLF, CR, LF) + * are recognised. + * If {@code includeSeparators} is true, then the line separators are included + * at the end of their corresponding lines; otherwise they are dropped. + * If {@code squishTrailing} is true, then trailing empty lines are not included + * in the result; otherwise, they will appear as empty strings. + */ + public static String[] lines(String s, boolean includeSeparators, boolean squishTrailing) { + if (s.length() == 0) + return new String[0]; + Pattern pattern = includeSeparators ? lineEndingIncludingSeparators : lineEnding; + return pattern.split(s, squishTrailing ? 0 : -1); + } + + /** + * Replace all line endings in the given string with \n + */ + public static String toUnixLineEndings(String s) { + return lineEnding.matcher(s).replaceAll("\n"); + } + + /** + * Get a boolean indicating whether the string contains line separators + * All known style of line terminator (CRLF, CR, LF) are recognised. + */ + public static boolean isMultiLine(String s) { + return lineEnding.matcher(s).find(); + } + + private static final Pattern whitespace = Pattern.compile("\\s+"); + /** + * Get the words (i.e. whitespace-delimited chunks of non-whitespace) from the given string. + * Empty words are not included -- this means that the result is a zero-length array for + * an input string that consists entirely of whitespace. + */ + public static String[] words(String s) { + s = s.trim(); + if (s.length() == 0) + return new String[0]; + return whitespace.split(s); + } + + /** + * Split a string into paragraphs (delimited by empty lines). Line endings are not preserved. + */ + public static String[] paragraphs(String s) { + List paragraphs = new ArrayList<>(); + + StringBuilder paragraph = new StringBuilder(); + boolean emptyParagraph = true; + + for (String line : StringUtil.lines(s)) { + if (line.isEmpty()) { + // line only has line endings, i.e. is between paragraphs. + if (!emptyParagraph) + paragraphs.add(paragraph.toString()); + paragraph = new StringBuilder(); + emptyParagraph = true; + } else { + if(paragraph.length() != 0) + paragraph.append(' '); + paragraph.append(line); + emptyParagraph = false; + } + } + if (!emptyParagraph) + paragraphs.add(paragraph.toString()); + return paragraphs.toArray(new String[0]); + } + + private static final char[] HEX_CHARS = "0123456789abcdef".toCharArray(); + + /** Convert an array of bytes into an array of lower-case hex digits. */ + public static String toHex (byte ... bytes) + { + StringBuilder strBldr = new StringBuilder(bytes.length * 2); + char[] hexchars = HEX_CHARS; + for (byte b : bytes) { + strBldr.append(hexchars[(b >>> 4) & 0xF]).append(hexchars[b & 0xF]); + } + return strBldr.toString(); + + } + + /** + * Convert String of hexadecimal digits to an array of bytes. + * @throws NumberFormatException if string does not have an even length or + * contains invalid characters. + */ + public static byte[] fromHex(String string) { + int len = string.length(); + if(len % 2 != 0) + throw new NumberFormatException("Hexadecimal string should have an even number of characters."); + byte[] data = new byte[len / 2]; + int index = 0; + for (int i = 0; i < len; i += 2) { + int a = Character.digit(string.charAt(i), 16); + if(a == -1) + throw new NumberFormatException("Invalid character in hexadecimal string: " + string.charAt(i)); + int b = Character.digit(string.charAt(i+1), 16); + if(b == -1) + throw new NumberFormatException("Invalid character in hexadecimal string: " + string.charAt(i+1)); + data[index] = (byte) ((a << 4) | b); + index++; + } + return data; + } + + /** + * Return a 8 character String describing the duration in {@code nanoSeconds}. + *

+ * The duration will be scaled and expressed using the appropriate units: nano-, micro-, milli-, + * seconds, minutes, hours, days, or years. + *

+ */ + public static String getDurationString (long nanoSeconds) + { + char sign = nanoSeconds < 0 ? '-' : '+'; + nanoSeconds = nanoSeconds < 0 ? -nanoSeconds : nanoSeconds; + if (nanoSeconds < 1e4) { + return sign + getDurationString(nanoSeconds, 1, "[ns]"); + } + else if (nanoSeconds < 1e7) { + return sign + getDurationString(nanoSeconds, 1e3, "[us]"); + } + else if (nanoSeconds < 1e10) { + return sign + getDurationString(nanoSeconds, 1e6, "[ms]"); + } + else if (nanoSeconds < 1e13) { + return sign + getDurationString(nanoSeconds, 1e9, "[s] "); + } + else if (nanoSeconds < 60e13) { + return sign + getDurationString(nanoSeconds, 60e9, "[m] "); + } + else if (nanoSeconds < 3600e13) { + return sign + getDurationString(nanoSeconds, 3600e9, "[h] "); + } + else if (nanoSeconds < 86400e13) { + return sign + getDurationString(nanoSeconds, 86400e9, "[d] "); + } + else { + return sign + getDurationString(nanoSeconds, 31536000e9, "[y] "); + } + } + + /** + * Return a four character representation of the given duration in {@code nanoSeconds}, divided by + * the given {@code divisor} and suffixed with the given {@code unit}. + * + * @param nanoSeconds The duration to express; must be non-negative. + * @see #getDurationString(long) + */ + private static String getDurationString (long nanoSeconds, double divisor, String unit) + { + // Format as a 4 character floating point + String scaledStr = String.format("%-4f", nanoSeconds / divisor).substring(0, 4); + // Replace a trailing decimal with a space + return (scaledStr.endsWith(".") ? scaledStr.replace(".", " ") : scaledStr) + unit; + } + + /** + * Parse an Integer from the given {@code string}, returning null if parsing failed for any + * reason. + */ + public static Integer parseInteger (String string) + { + // Quick break-out if string is null + if (string == null) { + return null; + } + // Attempt to parse an integer + try { + return Integer.parseInt(string); + } + catch(NumberFormatException nfe) { + Exceptions.ignore(nfe, "deliberate test"); + return null; + } + } + + /** + * Append to a given {@link StringBuilder} a sequence of Objects via their + * {@link Object#toString()} method, and return the StringBuilder. + */ + public static StringBuilder appendLine (StringBuilder strBldr, Object ... args) + { + for(Object arg : args) { + strBldr.append(arg); + } + return strBldr.append("\n"); + } + + /** + * Compose a new String with every line prepended with a given prefix. + *

+ * The final portion of the {@code text} that is not terminated with a newline character will be + * prefixed if and only if it is non-empty. + *

+ * + * @param prefix The string that shall be prefixed to every line in {@code text}. + * @param text The string to split into lines and prefix. + */ + public static String prefixLines (String prefix, String text) + { + return text.replaceAll("(?m)^", prefix); + } + + /** + * Count the number of times a character occurs in a string. + * + * @param str The string to search in. + * @param ch The character to look for. + */ + public static int count (String str, char ch) + { + int r = 0; + for (int i = 0; i < str.length(); i++) { + if (str.charAt(i) == ch) { + r++; + } + } + return r; + } + + /** + * Add line numbers to the start of each line of the given {@code plainText}. + * + * @param plainText - some plain text, with lines distinguished by one of the standard + * line-endings. + * @return the {@code plainText}, with 1-indexed line numbers inserted at the start of each + * line. The line numbers will be of a fixed width comprising the length of the largest + * line number. + */ + public static String addLineNumbers(String plainText) { + /* + * Add line numbers to the plain text code sample. + */ + String[] lines = StringUtil.lines(plainText, false); + // The maximum number of characters needed to represent the line number + int lineColumnWidth = Integer.toString(lines.length).length(); + + StringBuilder sampleWithLineNumbers = new StringBuilder(); + for (int i = 0; i < lines.length; i++) { + boolean last = i == lines.length -1; + sampleWithLineNumbers.append(String.format("%" + lineColumnWidth + "s %s" + (last ? "" : "\n"), i + 1, lines[i])); + } + return sampleWithLineNumbers.toString(); + } + + // Pattern that matches a string of (at least one) decimal digits + private static final Pattern DIGITS_PATTERN = Pattern.compile("[0-9]+"); + + /** Return true iff the given string consists only of digits */ + public static boolean isDigits(String s) { + return DIGITS_PATTERN.matcher(s).matches(); + } + + /** + * Determine whether a given {@code char} is an ASCII letter. + */ + public static boolean isAsciiLetter (char c) + { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + } + + /** + * A {@link String} comparison function to hopefully help mitigate against timing attacks. + * Aims to be constant time in the length of the first argument however due to the nature + * of Java this is very hard to achieve reliably. Callers should not rely on this method + * being perfectly constant time and other defenses should be introduced as necessary + * to prevent timing attacks based on how critical it is to avoid them. + *

+ * Each argument may safely be null. + *

+ * Note there is a unit tests that asserts the timing properties of this method which is + * committed but not run by default. If any changes are made to the implementation then + * {@code StringUtilTests#secureIsEqualTiming} must be run manually. + */ + public static boolean secureIsEqual(String a, String b) { + if (a == null) { + // Since we are aiming for constant time in the length of the + // first argument only, it is ok to bail out quickly if a is null. + return b == null; + } + + byte[] aBytes = stringToBytes(a); + + boolean definitelyDifferent = b == null || b.length() != a.length(); + byte[] bBytes = stringToBytes(definitelyDifferent ? a : b); + byte[] randomBBytes = new byte[a.length()]; + RANDOM.nextBytes(randomBBytes); + if (definitelyDifferent) { + bBytes = randomBBytes; + } + + int result = 0; + for (int i = 0; i < aBytes.length; i++) { + result |= aBytes[i] ^ bBytes[i]; + } + return result == 0 && !definitelyDifferent; + } + + public static String lineSeparator(){ + return System.getProperty("line.separator"); + } + + public static String naturalGlue(String separator, String finalSeparator, Collection values) { + StringBuilder stringBuilder = new StringBuilder(); + Iterator iterator = values.iterator(); + boolean first = true; + if(iterator.hasNext()) { + boolean hasNext = true; + T current = iterator.next(); + while(hasNext) { + hasNext = iterator.hasNext(); + T next = iterator.hasNext() ? iterator.next() : null; + if(first) { + first = false; + } + else if(!hasNext) { + stringBuilder.append(finalSeparator); + } + else { + stringBuilder.append(separator); + } + stringBuilder.append(current != null ? current : ""); + current = next; + } + } + return stringBuilder.toString(); + } + + /** + * Convert a CamelCase (or camelCase) string to spinal-case. Adjacent sequences of upper-case + * characters are treated as a single word, so that "getXML" would be converted to "get-xml". + * Where a lower-case character follows an upper-case character after a sequence of at + * least on upper-case character, the last upper-case character in the sequence is assumed to + * be the first letter of a new word, rather than the last letter of an acronym. Thus, + * "getXMLFile" becomes "get-xml-file" rather than "get-xmlfile" or "get-xmlf-ile". + * + * @return The spinal-cased equivalent of {@code camelCaseStr}, or null if it is null. + */ + public static String camelToSpinal(String camelCaseStr) { + // Quick break-out if the string is null + if (camelCaseStr == null) + return null; + // Convert to spinal-case + String lcStr = camelCaseStr.toLowerCase(Locale.ENGLISH); + StringBuilder strBldr = new StringBuilder(); + for(int i=0; i 0) { + // Next character is upper-case, and not at the start of the string, + // so insert a preceding dash + strBldr.append("-"); + } + // Consume (append in l.c.) all contiguously following u.c. characters, except that + // if a sequence of two or more u.c. characters occurs followed by a l.c. character + // assume that the last u.c. character is the first in a new word and insert a - + // preceding the new word. + // + // Thus getXML becomes get-xml, but getXMLFile becomes get-xml-file rather than + // get-xmlfile. + int numUc = 0; + while(i 0 + && i+1 < camelCaseStr.length() + && camelCaseStr.charAt(i+1) == lcStr.charAt(i+1) + && isAsciiLetter(lcStr.charAt(i+1))) { + strBldr.append("-").append(lcStr.charAt(i++)); + break; + } + strBldr.append(lcStr.charAt(i++)); + ++numUc; + } + } + // Consume (append) all contiguously following l.c. characters + while(i + * + * @param ql The QL code string + * @param terminateStringsAtLineEnd If true, then strings are treated as ending at EOL; + * if false, unterminated strings result in an IllegalArgumentException. + * + * NB QL does not support multiline strings. + */ + public static String stripQlCommentsAndStrings(String ql, boolean terminateStringsAtLineEnd) { + StringBuilder returnBuilder = new StringBuilder(); + // in a quoted string you must ignore both comment starters + // in a multi-line comment you must ignore the other two (single line comment and string) starters + // in a single line comment you can just eat up to the end of the line + boolean inString = false; + boolean inMultiLineComment = false; + boolean inSingleLineComment = false; + for (int i = 0; i < ql.length(); i++) { + // String + if (!inMultiLineComment && !inSingleLineComment && matchesAt(ql, i, "\"") && !isEscaped(ql, i)) { + inString = !inString; + continue; + } else if (matchesEolAt(ql, i)) { + if (terminateStringsAtLineEnd) { + inString = false; // force strings to end at EOL - multi-line strings are invalid + } else if (inString) { + throw new IllegalArgumentException("Unterminated string found."); + } + } + + // Single-line comment + if (!inString && !inMultiLineComment && matchesAt(ql, i, "//")) { + inSingleLineComment = true; + continue; + } else if (inSingleLineComment && matchesEolAt(ql, i)) { + inSingleLineComment = false; + } + + // Multi-line comment + if (!inString && !inSingleLineComment && matchesAt(ql, i, "/*")) { + inMultiLineComment = true; + } else if (inMultiLineComment && matchesAt(ql, i, "*/")) { + inMultiLineComment = false; + i++; // skip the next character (the '/') as well as this one + + continue; + } + + if (inString || inMultiLineComment || inSingleLineComment) { + continue; + } + + returnBuilder.append(ql.charAt(i)); + } + + if (inString && !terminateStringsAtLineEnd) { + throw new IllegalArgumentException("Unterminated string found."); + } + + return returnBuilder.toString(); + } + + /** + * Calls (@link #stripQlCommentsAndStrings(String, boolean), + * passing {@code false} for the {@code terminateStringsAtLineEnd} parameter. + */ + public static String stripQlCommentsAndStrings(String ql) { + return stripQlCommentsAndStrings(ql, false); + } + + private static boolean matchesAt(String sourceString, int currentCharIndex, String subString) { + if (currentCharIndex + subString.length() > sourceString.length()) { + return false; + } + + return sourceString.substring(currentCharIndex, currentCharIndex + subString.length()).equals(subString); + } + + private static boolean matchesEolAt(String sourceString, int currentCharIndex ) { + return matchesOneOfAt(sourceString, currentCharIndex, "\n", "\r"); + } + + private static boolean matchesOneOfAt(String sourceString, int currentCharIndex, String... subStrings) { + for (String subString: subStrings) { + if (matchesAt(sourceString, currentCharIndex, subString)) { + return true; + } + } + return false; + } + + private static boolean isEscaped(String theString, int currentCharIndex) { + if (currentCharIndex == 0) { + return false; + } + return previousCharacter(theString, currentCharIndex) == '\\' && !isEscaped(theString, currentCharIndex-1); + } + + private static char previousCharacter(String theString, int currentCharIndex) { + if (currentCharIndex == 0) { + return Character.MIN_VALUE; + } + return theString.charAt(currentCharIndex - 1); + } + + /** + * Compare two arrays of strings. The two arrays are considered equal if they + * are either both null or contain equal elements in the same order (ignoring + * case). + * + * @param a + * the first array + * @param a2 + * the second array + * @return true iff the elements in the arrays are equal when ignoring case + */ + public static boolean arrayEqualsIgnoreCase(String[] a, String[] a2) { + if (a == null) return a2 == null; + if (a2 == null) return false; + if (a.length != a2.length) return false; + for (int i = 0; i < a.length; i++) { + if ((a[i] == null && a2[i] != null) || !a[i].equalsIgnoreCase(a2[i])) return false; + } + return true; + } + + public static final Pattern NEWLINE_PATTERN = Pattern.compile("\n"); + + /** + * Convert a string into a doc comment with the content as the body. + */ + public static String toCommentString(String content) { + StringBuilder result = new StringBuilder(); + result.append("/**\n"); + String[] lines = StringUtil.lines(content); + for (String line: lines) { + result.append(" *" + line); + result.append("\n"); + } + result.append(" */"); + return result.toString(); + } + + /** + * Is {@code str} composed only of printable ASCII characters (excluding + * newline, carriage return and tab but including space)? + */ + public static boolean isPrintableAscii(String str) { + if (str == null) + return false; + for(int i=0; i= 32 && c < 127; + } + + /** + * Return true if {@code str} only contains characters which are either printable ASCII or ASCII whitespace + */ + public static boolean isAsciiText(String str) { + for(int i=0; i 159) { + return false; + } + + // Most of ASCII is okay + if (c >= 32 && c < 127) { + return false; + } + + // Basic whitespace is okay + if (c == '\t' || + c == '\n' || + c == '\r') { + return false; + } + + // If we've got this far, it must be a control character + return true; + } + +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/data/Tuple1.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/data/Tuple1.java new file mode 100644 index 00000000000..171c90d2f48 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/data/Tuple1.java @@ -0,0 +1,106 @@ +package com.semmle.util.data; + +import java.io.Serializable; + + +/** + * Tuple of one typed element. + *

+ * Note that this is a sub-class of {@link TupleN} and a super-class of {@link Tuple2}, + * {@link Tuple3}, and any subsequent extensions in a similar vein. + *

+ */ +public class Tuple1 extends TupleN +{ + /** + * Serializable variant of {@link Tuple1}. + */ + public static class SerializableTuple1 + extends Tuple1 implements Serializable { + + private static final long serialVersionUID = -7989122667707773448L; + + public SerializableTuple1() { + } + + public SerializableTuple1(T0 t0) { + super(t0); + } + } + + private static final long serialVersionUID = -4317563803154647477L; + + /** The single contained value. */ + protected Type0 _value0; + + + /** Construct a new {@link Tuple1} with a null value. */ + public Tuple1 () {} + + /** Construct a new {@link Tuple1} with the given value. */ + public Tuple1 (Type0 value0) + { + _value0 = value0; + } + + /** Construct a new {@link Tuple1} with the given value. */ + public static Tuple1 make(Type0 value0) + { + return new Tuple1(value0); + } + + /** + * Get the value contained by this {@link Tuple1}. + */ + public final Type0 value0 () + { + return _value0; + } + + @Override + protected Object value_ (int n) + { + return _value0; + } + + /** + * Return the number of elements in this {@link Tuple1}. + *

+ * Sub-classes shall override this method to increase its value accordingly. + *

+ */ + @Override + public int size () + { + return 1; + } + + /** + * Return a plain string representation of the contained value (where null is represented by the + * empty string). + *

+ * Sub-classes shall implement a comma-separated concatenation. + *

+ */ + @Override + public String toPlainString () + { + return _value0 == null ? "" : _value0.toString(); + } + + @Override + public int hashCode () + { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((_value0 == null) ? 0 : _value0.hashCode()); + return result; + } + + @Override + public boolean equals (Object obj) + { + return obj == this || (super.equals(obj) && equal(((Tuple1)obj)._value0, _value0)); + } + +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/data/Tuple2.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/data/Tuple2.java new file mode 100644 index 00000000000..2b3ce9a469e --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/data/Tuple2.java @@ -0,0 +1,93 @@ +package com.semmle.util.data; + +import java.io.Serializable; + + +/** + * Tuple of two typed elements. + *

+ * Note that this is an extension of {@link Tuple1} and a super-class of {@link Tuple3} (and any + * subsequent additions). + *

+ */ +public class Tuple2 extends Tuple1 +{ + /** + * Serializable variant of {@link Tuple2}. + */ + public static class SerializableTuple2 + extends Tuple2 implements Serializable { + + private static final long serialVersionUID = 1624467154864321244L; + + public SerializableTuple2() { + } + + public SerializableTuple2(T0 t0, T1 t1) { + super(t0, t1); + } + } + + private static final long serialVersionUID = -400406676673562583L; + + /** The additional element contained by this {@link Tuple2}. */ + protected Type1 _value1; + + /** Construct a new {@link Tuple2} with null values. */ + public Tuple2 () {} + + /** Construct a new {@link Tuple2} with the given values. */ + public Tuple2 (Type0 value0, Type1 value1) + { + super(value0); + _value1 = value1; + } + + /** Construct a new {@link Tuple2} with the given value. */ + public static Tuple2 make(Type1 value0, Type2 value1) + { + return new Tuple2(value0, value1); + } + + /** + * Get the second value in this {@link Tuple2}. + */ + public final Type1 value1 () + { + return _value1; + } + + @Override + protected Object value_ (int n) + { + return n == 2 ? _value1 : super.value_(n); + } + + @Override + public int size () + { + return 2; + } + + @Override + public String toPlainString () + { + return super.toPlainString() + ", " + (_value1 == null ? "" : _value1.toString()); + } + + @Override + public int hashCode () + { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((_value1 == null) ? 0 : _value1.hashCode()); + return result; + } + + @Override + public boolean equals (Object obj) + { + return obj == this || (super.equals(obj) && equal(((Tuple2)obj)._value1, _value1)); + } + +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/data/TupleN.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/data/TupleN.java new file mode 100644 index 00000000000..a6b058316f7 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/data/TupleN.java @@ -0,0 +1,85 @@ +package com.semmle.util.data; + +import java.io.Serializable; + +/** + * Untyped base-class for the generic {@link Tuple1}, {@link Tuple2}, ... etc. + *

+ * This class also functions as a zero-element tuple. + *

+ */ +public class TupleN implements Serializable +{ + private static final long serialVersionUID = -1799116497122427806L; + + /** + * Get the n'th value contained by this {@link TupleN}. + * + * @param n The zero-based index of the value to be returned. + * @return The n'th value, or null if n is out of range. + */ + public final Object value (int n) + { + return n < 0 || n > size() ? null : value_(n); + } + + /** Internal method for obtaining the n'th value (n is guaranteed to be in-range). */ + protected Object value_ (int n) + { + return null; + } + + /** + * Get the number of values contained by this {@link TupleN}. + */ + public int size () + { + return 0; + } + + /** + * Return a plain string representation of the contained value (where null is represented by the + * empty string). + *

+ * Sub-classes shall implement a comma-separated concatenation. + *

+ */ + public String toPlainString () + { + return ""; + } + + /** + * Get a parenthesized, comma-separated string representing the values contained by this + * {@link TupleN}. Null values are represented by an empty string. + */ + @Override + public final String toString () + { + return "(" + toPlainString() + ")"; + } + + @Override + public int hashCode () + { + return 0; + } + + @Override + public boolean equals (Object obj) + { + return obj == this || (obj !=null && obj.getClass().equals(getClass())); + } + + /** + * Convenience method implementing objects.equals(object, object), which is not available due to a + * java version restriction. + */ + protected static boolean equal(Object obj1, Object obj2) + { + if (obj1 == null) { + return obj2 == null; + } + return obj1.equals(obj2); + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/exception/CatastrophicError.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/exception/CatastrophicError.java new file mode 100644 index 00000000000..e3135ec280d --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/exception/CatastrophicError.java @@ -0,0 +1,117 @@ +package com.semmle.util.exception; + +import java.util.Arrays; + +/** + * This is a standard Semmle unchecked exception. + * Usage of this should follow the guidelines described in docs/semmle-unchecked-exceptions.md + */ +public class CatastrophicError extends NestedError { + + private static final long serialVersionUID = 4132771414092814913L; + + public CatastrophicError(String message) { + super(message); + } + + public CatastrophicError(Throwable throwable) { + super(throwable); + } + + public CatastrophicError(String message, Throwable throwable) { + super(message,throwable); + } + + /** + * Utility method for throwing a {@link CatastrophicError} with the given {@code message} if the given + * {@code condition} is true. + */ + public static void throwIf(boolean condition, String message) + { + if (condition) { + throw new CatastrophicError(message); + } + } + + /** + * Utility method for throwing a {@link CatastrophicError} if the given {@code object} is null. + *

+ * See {@link #throwIfAnyNull(Object...)} which may be more convenient for checking multiple + * arguments. + *

+ */ + public static void throwIfNull(Object object) + { + if (object == null) { + throw new CatastrophicError("null object"); + } + } + + /** + * Utility method for throwing a {@link CatastrophicError} with the given {@code message} if the given + * {@code object} is null. + *

+ * See {@link #throwIfAnyNull(Object...)} which may be more convenient for checking multiple + * arguments. + *

+ */ + public static void throwIfNull (Object object, String message) + { + if (object == null) { + throw new CatastrophicError(message); + } + } + + /** + * Throw a {@link CatastrophicError} if any of the given {@code objects} is null. + *

+ * If a {@link CatastrophicError} is thrown, its message will indicate all null arguments by index. + *

+ *

+ * See {@link #throwIfNull(Object, String)} which may be a fraction more efficient if there's only + * one argument, and allows an 'optional' message parameter. + *

+ */ + public static void throwIfAnyNull (Object ... objects) + { + /* + * Check each argument for nullity, and start building a set of index strings iff at least one + * is non-null + */ + String[] nullArgs = null; + for (int argNum = 0; argNum < objects.length; ++argNum) { + if (objects[argNum] == null) { + nullArgs = nullArgs == null ? new String[1] : Arrays.copyOf(nullArgs, nullArgs.length+1); + nullArgs[nullArgs.length-1] = "" + argNum; + } + } + if (nullArgs != null) { + // Compose a message describing which arguments are null + StringBuffer strBuf = new StringBuffer(); + if (nullArgs.length == 0) { + strBuf.append("null argument(s)"); + } else { + strBuf.append("null argument" + (nullArgs.length > 1 ? "s: " : ": ") + nullArgs[0]); + for (int i = 1; i < nullArgs.length; ++i) { + strBuf.append(", " + nullArgs[i]); + } + } + String message = strBuf.toString(); + throw new CatastrophicError(message); + } + } + + /** + * Convenience method for use in constructors that assign a parameter to a + * field, assuming the former to be non-null. + * + * @param t A non-null value of type {@code T}. + * @return {@code t} + * @throws CatastrophicError if {@code t} is null. + * @see #throwIfNull(Object) + */ + public static T nonNull(T t) { + throwIfNull(t); + return t; + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/exception/Exceptions.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/exception/Exceptions.java new file mode 100644 index 00000000000..16a051537df --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/exception/Exceptions.java @@ -0,0 +1,120 @@ +package com.semmle.util.exception; + +import java.io.PrintWriter; +import java.io.StringWriter; + +/** + * Simple functions for printing exceptions. This is intended for use + * in debug output, not for formatting for user consumption + */ +public class Exceptions { + + /** + * Compose a String with the same format as that output by {@link Throwable#printStackTrace()}. + */ + public static String printStackTrace(Throwable t) + { + StringWriter stringWriter = new StringWriter(); + t.printStackTrace(new PrintWriter(stringWriter)); + return stringWriter.toString(); + } + + /** + * Print an exception in a readable format with all information, + * including the type, message, stack trace, and nested exceptions + */ + public static String print(Throwable t) { + return printDetailed(t, true); + } + + /** + * Print an exception in a somewhat readable format fitting on one line. + * Most of the time simply using print is preferable + */ + public static String printShort(Throwable t) { + return printDetailed(t, false); + } + + /** + * Ignore an exception. This method does nothing, but should be called + * (with a reasonable message) to document the reason why the exception does + * not need to be used. + */ + public static void ignore(Throwable e, String message) { + + } + + /** + * Print an exception in a long format, possibly producing multiple + * lines if the appropriate flag is passed + * @param multiline if true, produce multiple lines of output + */ + private static String printDetailed(Throwable t, boolean multiline) { + StringBuilder sb = new StringBuilder(); + + Throwable current = t; + while (current != null) { + printOneException(current, multiline, sb); + Throwable cause = current.getCause(); + if (cause == current) + current = null; + else + current = cause; + + if (current != null) { + if (multiline) + sb.append("\n\n ... caused by:\n\n"); + else + sb.append(", caused by: "); + } + } + + return sb.toString(); + } + + private static void printOneException(Throwable t, boolean multiline, StringBuilder sb) { + sb.append(multiline ? t.toString() : t.toString().replace('\n', ' ').replace('\r', ' ')); + boolean first = true; + for (StackTraceElement e : t.getStackTrace()) { + if (first) + sb.append(multiline ? "\n" : " - ["); + else + sb.append(multiline ? "\n" : ", "); + first = false; + sb.append(e.toString()); + } + if (!multiline) + sb.append("]"); + } + + /** A stand-in replacement for `assert` that throws a {@link CatastrophicError} and isn't compiled out. */ + public static void assertion(boolean cond, String message) { + if(!cond) + throw new CatastrophicError(message); + } + + /** + * Turn the given {@link Throwable} into a {@link RuntimeException} by wrapping it if necessary. + */ + public static RuntimeException asUnchecked(Throwable t) { + if (t instanceof RuntimeException) + return (RuntimeException)t; + else + return new RuntimeException(t); + } + + /** + * Throws an arbitrary {@link Throwable}, wrapping in a runtime exception if necessary. + * Unlike {@link #asUnchecked} it preserves subclasses of {@link Error}. + */ + public static T rethrowUnchecked(Throwable t) { + if (t instanceof RuntimeException) { + throw (RuntimeException) t; + } else if (t instanceof Error) { + throw (Error) t; + } + throw new RuntimeException(t); + } + + +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/exception/InterruptedError.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/exception/InterruptedError.java new file mode 100644 index 00000000000..0b4a21e9636 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/exception/InterruptedError.java @@ -0,0 +1,26 @@ +package com.semmle.util.exception; + +/** + * An exception thrown in cases where it is impossible to + * throw the (checked) Java {@link InterruptedException}, + * eg. in visitors + */ +public class InterruptedError extends RuntimeException { + + private static final long serialVersionUID = 9163340147606765395L; + + public InterruptedError() { } + + public InterruptedError(String message, Throwable cause) { + super(message, cause); + } + + public InterruptedError(String message) { + super(message); + } + + public InterruptedError(Throwable cause) { + super(cause); + } + +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/exception/NestedError.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/exception/NestedError.java new file mode 100644 index 00000000000..b35496a3f12 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/exception/NestedError.java @@ -0,0 +1,47 @@ +package com.semmle.util.exception; + +public abstract class NestedError extends RuntimeException { + + private static final long serialVersionUID = -3145876396931008989L; + + public NestedError(String message) { + super(message); + } + + public NestedError(Throwable throwable) { + super(throwable); + } + + public NestedError(String message, Throwable throwable) { + super(buildMessage(message, throwable), throwable); + } + + /** + * Subclasses should not need to call this directly -- just call the + * two-argument super constructor. + */ + private static String buildMessage(String message, Throwable throwable) { + if (throwable == null) + return message; + + while (throwable.getCause() != null && throwable.getCause() != throwable) + throwable = throwable.getCause(); + String banner = "eventual cause: " + throwable.getClass().getSimpleName(); + String rootmsg = throwable.getMessage(); + if (rootmsg == null) { + // Don't amend the banner + } else { + int p = rootmsg.indexOf('\n'); + if (p >= 0) + rootmsg = rootmsg.substring(0, p) + "..."; + if (rootmsg.length() > 100) + rootmsg = rootmsg.substring(0, 80) + "..."; + banner += " \"" + rootmsg + "\""; + } + if (message.contains(banner)) + return message; + else + return message + "\n(" + banner + ")"; + } + +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/exception/ResourceError.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/exception/ResourceError.java new file mode 100644 index 00000000000..eed3239b7ac --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/exception/ResourceError.java @@ -0,0 +1,30 @@ +package com.semmle.util.exception; + +/** + * This is a standard Semmle unchecked exception. + * Usage of this should follow the guidelines described in docs/semmle-unchecked-exceptions.md + */ +public class ResourceError extends NestedError { + + private static final long serialVersionUID = 4132771414092814913L; + + public ResourceError(String message) { + super(message); + } + + @Deprecated // A ResourceError may be presented to the user, so should always have a message + public ResourceError(Throwable throwable) { + super(throwable); + } + + public ResourceError(String message, Throwable throwable) { + super(message,throwable); + } + + @Override + public String toString() { + // The message here should always be meaningful enough that we can return that. + return getMessage() != null ? getMessage() : super.toString(); + } + +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/exception/UserError.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/exception/UserError.java new file mode 100644 index 00000000000..8a5953c5214 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/exception/UserError.java @@ -0,0 +1,46 @@ +package com.semmle.util.exception; + +/** + * This is a standard Semmle unchecked exception. + * Usage of this should follow the guidelines described in docs/semmle-unchecked-exceptions.md + */ +public class UserError extends NestedError { + + private static final long serialVersionUID = 4132771414092814913L; + + private final boolean reportAsInfoMessage; + + public UserError(String message) { + this(message, false); + } + + /** + * A user-visible error + * + * @param message The message to display + * @param reportAsInfoMessage If true, report as information only - not an error + */ + public UserError(String message, boolean reportAsInfoMessage) { + super(message); + this.reportAsInfoMessage = reportAsInfoMessage; + } + + public UserError(String message, Throwable throwable) { + super(message,throwable); + this.reportAsInfoMessage = false; + } + + /** + * If true, report the message without interpreting it as a fatal error + */ + public boolean reportAsInfoMessage() { + return reportAsInfoMessage; + } + + @Override + public String toString() { + // The message here should always be meaningful enough that we can return that. + return getMessage() != null ? getMessage() : super.toString(); + } + +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/expansion/ExpansionEnvironment.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/expansion/ExpansionEnvironment.java new file mode 100644 index 00000000000..47bbb1d2029 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/expansion/ExpansionEnvironment.java @@ -0,0 +1,893 @@ +package com.semmle.util.expansion; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.security.GeneralSecurityException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.semmle.util.data.StringUtil; +import com.semmle.util.exception.CatastrophicError; +import com.semmle.util.exception.ResourceError; +import com.semmle.util.exception.UserError; +import com.semmle.util.files.FileUtil; +import com.semmle.util.process.Builder; +import com.semmle.util.process.Env; +import com.semmle.util.process.Env.Var; +import com.semmle.util.process.LeakPrevention; + +/** + * An environment for performing variable expansions. + * + *

+ * The environment is defined by a set of variable definitions, which are + * name/value pairs of strings. Once this has been populated (via the + * {@link #defineVar(String, String)} and {@link #defineVars(Map)} methods), + * arbitrary strings can be expanded. + *

+ * + *

+ * Two modes of expansion are supported: + *

+ *
    + *
  • String mode ({@link #strExpand(String)}): The result is intended to be a + * single string.
    • + *
  • List mode ({@link #listExpand(String)}): The result will be interpreted + * as a command line, and hence is a list of strings. + *
+ * + *

+ * Variables are referenced by ${name} to trigger a string-mode + * expansion, and by ${=name} to trigger a list-mode expansion. + * This makes {@code $} a meta-character, and so it has to be escaped; the + * escape sequence for it is ${}. + *

+ * + *

+ * In list mode, strings are split in a platform-independent way similar (but + * not identical) to normal shell argument splitting. Runs of white-space + * separate arguments, and double-quotes can be used to protect whitespace from + * splitting. The escape character is backslash. All of these metacharacters + * have no special meaning in string mode. + *

+ * + *

+ * The {@code define*} and {@link #doNotExpand(String...)} methods of this + * class are not thread-safe; they mutate instance state in an unsynchronized + * way. By contrast, the expansion methods ({@link #strExpand(String)}, + * {@link #strExpandVar(String)}, {@link #listExpand(String)}, + * {@link #listExpandVar(String)} and {@link #varLookup(String)}) + * are thread safe relative to each + * other. This means that it's fine to construct an expansion environment once, + * and then use it from multiple threads concurrently, as long as no new variables + * are defined. In addition, {@link #validate(String)} is safe to call once an + * {@link ExpansionEnvironment} is fully initialised, even concurrently. + *

+ * + *

+ * Upon encountering any error (malformed variable expansion, malformed quoted + * string (in list mode), reference to unknown variable, cyclic variable + * definitions), the {@link #strExpand(String)} and {@link #listExpand(String)} + * methods will throw {@link UserError} with a suitable message. + *

+ * + *

+ * As an advanced feature, command substitutions can be supported. They take the + * form of $(cmd arg1 arg2) for string-mode expansion, and + * $(=cmd arg1 + * arg2) for list-mode. The contents of the $(..) operator + * undergo normal splitting, and are then run as a new process with the given + * list of arguments. The working directory is unspecified, and it is an error + * to depend upon it. A non-zero exit code, or a non-empty {@code stderr} stream + * of the command, will result in a {@link UserError} indicating that something + * went wrong; otherwise, the {@code stdout} output is collected and substituted + * (possibly undergoing splitting, in the second form). + *

+ */ +public class ExpansionEnvironment { + + /** + * A source for variable definitions to be used in an expansion environment. + */ + public static interface VariableSource { + /** + * A callback which is expected to add all variables in the source to + * the given environment. + * + * @param env + * The environment that should be filled in. + */ + public void fillIn(ExpansionEnvironment env); + } + + private final Map vars = new LinkedHashMap(); + + private final Set unexpandedVars = new LinkedHashSet(); + + private final boolean commandSubstitutions; + + /** + * Construct an empty {@link ExpansionEnvironment}. + */ + public ExpansionEnvironment(boolean commandSubstitutions) { + this.commandSubstitutions = commandSubstitutions; + } + + /** + * This the old default constructor, which always enables command substitutions. + * Doing so is a security risk whenever the string you expand may come + * from an untrusted source, so you should only do that when you explicitly want + * to do it and have decided that it is safe. (And then use the constructor that + * has an explicit argument to say so!) + */ + @Deprecated + public ExpansionEnvironment() { + this(true); + } + + /** + * Construct an environment based on an existing map. + */ + public ExpansionEnvironment(boolean commandSubstitutions, Map vars) { + this(commandSubstitutions); + this.vars.putAll(vars); + } + + /** + * Construct a copy of an existing {@link ExpansionEnvironment}. + */ + public ExpansionEnvironment(ExpansionEnvironment other) { + this(other.commandSubstitutions); + this.vars.putAll(other.vars); + this.unexpandedVars.addAll(other.unexpandedVars); + } + + /** + * Add a set of variable definitions to this environment. + * + * @param vars + * A mapping from variable names to variable values. Recursive + * variable references are allowed, but cycles are an error. + */ + public void defineVars(Map vars) { + this.vars.putAll(vars); + } + + /** + * Add the specified variable definition to this environment. + * + * @param name + * A variable name. + * @param value + * The value that the variable should expand to. References to + * other variables or expansions are allowed, but cycles are an + * error. + */ + public void defineVar(String name, String value) { + this.vars.put(name, value); + } + + /** + * Try to load a file as a Java properties file and add all of its key/value + * pairs as variable definitions. + * + * @param vars + * A {@link File} that will be loaded as a Java properties file, + * if it exists. May be null or a file whose + * existence has not been checked. + * @throws ResourceError + * if the file exists but can't be read, or exists as a + * directory, or reading it fails. + */ + public void defineVarsFromFile(File vars) { + if (vars == null || !vars.exists()) + return; + + if (vars.isDirectory()) + throw new ResourceError(vars + + " is a directory, cannot load variables from it."); + + Properties properties = FileUtil.loadProperties(vars); + for (String key : properties.stringPropertyNames()) + defineVar(key, properties.getProperty(key)); + } + + /** + * Add a variable definition of {@code env.foo=bar} for each system + * environment variable {@code foo=bar}. Typically it is desirable to allow + * the environment to override previously specified variables, so this + * should be called once all other variables have been defined. + * + *

+ * The values of variables taken from the environment are escaped to prevent + * recursive expansion; in particular, this prevents accidental command + * execution if a command substitution is encountered in the environment. + *

+ */ + public void defineVarsFromEnvironment(Env environment) { + String extraVars = environment.get(Var.ODASA_EXTRA_VARIABLES); + if (extraVars != null) + defineVarsFromFile(new File(extraVars)); + + for (Entry var : environment.getenv().entrySet()) + defineVar("env." + var.getKey(), var.getValue().replace("$", "${}")); + + environment.addEnvironmentToNewEnv(this); + } + + /** + * Indicate that references to the given set of variable names should not be + * expanded. This means that they need not be defined, and the output will + * contain the literal variable expansion sequences. + * + * @param vars + * A list of variable names. + */ + public void doNotExpand(String... vars) { + for (String var : vars) + unexpandedVars.add(var); + } + + /** + * Supply a "default value" for a variable, meaning that the variable will + * be set to the given default value if it hasn't already been defined. No + * change is made to this environment if a definition exists. + * @param var A variable name. + * @param defaultValue The default value for the named variable. + */ + public void setDefault(String var, String defaultValue) { + if (!vars.containsKey(var)) + vars.put(var, defaultValue); + } + + /** + * Expand the given string in "string mode", resolving variable references + * and command substitutions. + */ + public String strExpand(String s) { + try { + return new Expander().new ExpansionParser(s).parseAsString().expandAsString(); + } catch (UserError e) { + throw new UserError("Failed to expand '" + s + "'.", e); + } + } + + /** + * Expand the given string in "list mode", resolving variable references and + * command substitutions. + */ + public List listExpand(String s) { + try { + return new Expander().new ExpansionParser(s).parseAsList().expandAsList(); + } catch (UserError e) { + throw new UserError("Failed to expand '" + s + + "' as an argument list.", e); + } + } + + /** + * Expand the given variable fully in "string mode", resolving variable + * references and command substitutions. The entire string is interpreted as + * the name of the initial variable. + */ + public String strExpandVar(String varName) { + return new Expander().new Variable(varName).expandAsString(); + } + + /** + * Expand the given variable fully in "list mode", resolving variable + * references and command substitutions. The entire string is interpreted as + * the name of the initial variable. + */ + public List listExpandVar(String varName) { + return new Expander().new SplitVariable(varName).expandAsList(); + } + + /** + * Validate the given string for expansion. This verifies the absence of + * parse errors, and the fact that all directly referenced variables are + * defined by this environment. + * + *

+ * Expansion using {@link #strExpand(String)} or {@link #listExpand(String)} + * may still not succeed, if there are semantic errors (like circular + * variable definitions) or a command substitution introduces a reference to + * an undefined variable. + *

+ * + * @param str + * A string that should be validated. + * @throws UserError + * if validation fails, with a suitable error message. + */ + public void validate(String str) { + new Expander().new ExpansionParser(str).parseAsList().validate(); + } + + /** + * Look up the (raw) value of a given variable, without performing expansion + * on it. + * + * @param name + * The variable name. + * @return The value that this variable is mapped to. + * @throws UserError + * if the variable is not defined. + */ + public synchronized String varLookup(String name) { + String value = vars.get(name); + if (value == null) { + ArrayList available = new ArrayList(vars.keySet()); + Collections.sort(available); + throw new UserError("Attempting to expand unknown variable: " + + name + ", available variables are: " + available); + } + return value; + } + + /** + * Check whether this environment defines a variable of the given name, without + * performing expansion on it -- such full expansion may still fail. + * + * @param name The variable name. + * @return true if this environment contains a direct definition + */ + public boolean definesVar(String name) { + return vars.containsKey(name); + } + + private static class ExpansionTokeniser { + /** + * The delimiters which should be returned as their own tokens. Order of + * alternatives matters! The recognised tokens are, in order: + * + *
    + *
  • {@code \\}
    • + *
  • {@code \"}
    • + *
  • {@code "}
    • + *
  • ${}
    • + *
  • ${=
    • + *
  • ${
    • + *
  • $(=
    • + *
  • $(
    • + *
  • $
    • + *
  • }
    • + *
  • )
    • + *
  • Runs of whitespace.
    • + *
+ * + *

+ * By defining the alternatives in this order, longer matches will be + * preferred, so that checking for escape sequences is easy. Note that + * in the regular expression source, a literal {@code \} must undergo + * two levels of escaping: Java strings and regular expression + * metacharacters; it thus becomes {@code \\\\}. + */ + private static final Pattern delims = Pattern + .compile("\\\\\\\\|\\\\\"|\"|\\$\\{\\}|\\$\\{=|\\$\\{|" + + "\\$\\(=|\\$\\(|\\$|\\}|\\)|\\s+"); + + private final List tokens = new ArrayList(); + private final int[] positions; + private int nextToken = 0; + + public ExpansionTokeniser(String str) { + Matcher matcher = delims.matcher(str); + StringBuffer tmp = new StringBuffer(); + while (matcher.find()) { + matcher.appendReplacement(tmp, ""); + if (tmp.length() > 0) { + tokens.add(tmp.toString()); + tmp = new StringBuffer(); + } + tokens.add(matcher.group()); + } + matcher.appendTail(tmp); + if (tmp.length() > 0) + tokens.add(tmp.toString()); + + positions = new int[tokens.size()]; + int pos = 0; + for (int i = 0; i < tokens.size(); i++) { + positions[i] = pos; + pos += tokens.get(i).length(); + } + } + + public boolean hasMoreTokens() { + return nextToken < tokens.size(); + } + + public String nextToken() { + return tokens.get(nextToken++); + } + + public boolean isDelimiter(String token) { + return delims.matcher(token).matches(); + } + + public int pos() { + return positions[nextToken - 1] + 1; + } + } + + /** + * A wrapper around the various expansion classes, holding some expansion + * state to detect things like circular variable definitions. + */ + private class Expander { + + private final Set expansionsInProgress = new LinkedHashSet(); + + /** + * A string expansion. This can be a literal string, a variable reference or + * a command substitution; the latter two can optionally be "split". Each + * expansion can be interpreted to yield a single string or a list of + * strings (typically as program arguments). + */ + abstract class Expansion { + public abstract String expandAsString(); + + public abstract List expandAsList(); + + public abstract void validate(); + } + + class Sentence extends Expansion { + private final List> words = new ArrayList>(); + + public Sentence(List> words) { + this.words.addAll(words); + } + + @Override + public void validate() { + for (List expansions : words) + for (Expansion expansion : expansions) + expansion.validate(); + } + + private String expandWord(List word) { + StringBuilder result = new StringBuilder(); + for (Expansion e : word) + result.append(e.expandAsString()); + return result.toString(); + } + + @Override + public String expandAsString() { + StringBuilder result = new StringBuilder(); + + for (List word : words) { + if (result.length() > 0) + result.append(' '); + result.append(expandWord(word)); + } + + return result.toString(); + } + + @Override + public List expandAsList() { + List result = new ArrayList(); + + for (List word : words) { + List> segments = new ArrayList>(); + for (Expansion e : word) { + segments.add(e.expandAsList()); + } + result.addAll(glue(segments)); + } + + return result; + } + + /** + * This is a non-quadratic implementation of the following Haskell code: + * + * 

+			 * 
+			 * glue :: [[String]] -> [String]
+			 * glue = foldr join []
+			 *     where join [] xs = xs
+			 *           join xs [] = xs
+			 *           join xs ys = init xs ++ [last xs ++ head ys] ++ tail ys
+			 * 
+			 *
+ */ + private List glue(List> segments) { + String trailingWord = null; + List result = new ArrayList(); + for (List segment : segments) + trailingWord = glue_join_accum(result, segment, trailingWord); + + if (trailingWord != null) + result.add(trailingWord); + + return result; + } + + private String glue_join_accum(List result, + List segment, String trailingWord) { + int n = segment.size(); + switch (n) { + case 0: + return trailingWord; + case 1: + return combine(trailingWord, segment.get(0)); + default: + result.add(combine(trailingWord, segment.get(0))); + result.addAll(segment.subList(1, n - 1)); + return segment.get(n - 1); + } + } + + private String combine(String a, String b) { + if (a == null) + return b; + return a + b; + } + } + + class Literal extends Expansion { + private final String value; + + public Literal(String value) { + this.value = value; + } + + @Override + public void validate() { + // Always valid. + } + + @Override + public String expandAsString() { + return value; + } + + @Override + public List expandAsList() { + return Collections.singletonList(value); + } + } + + class QuotedString extends Sentence { + public QuotedString(List content) { + super(Collections.singletonList(content)); + } + + @Override + public List expandAsList() { + return Collections.singletonList(this.expandAsString()); + } + } + + class Variable extends Expansion { + protected final String name; + + public Variable(String name) { + this.name = name; + } + + @Override + public void validate() { + varLookup(name); // Will throw if variable is undefined. + } + + protected void startExpanding(String name) { + if (!expansionsInProgress.add(name)) + throw new UserError("Circular expansion of variable " + name); + } + + protected void doneWith(String name) { + if (!expansionsInProgress.remove(name)) + throw new CatastrophicError("Not currently expanding " + name); + } + + protected String ref() { + return "${" + name + "}"; + } + + @Override + public final String expandAsString() { + if (unexpandedVars.contains(name)) + return ref(); + startExpanding(name); + String result = expandAsStringImpl(); + doneWith(name); + return result; + } + + public String expandAsStringImpl() { + // Not calling ExpansionEnvironment.strExpand(), since + // we must run in the same enclosing instance of Expander. + return new ExpansionParser(varLookup(name)).parseAsString().expandAsString(); + } + + @Override + public final List expandAsList() { + if (unexpandedVars.contains(name)) + return Collections.singletonList(ref()); + startExpanding(name); + List result = expandAsListImpl(); + doneWith(name); + return result; + } + + public List expandAsListImpl() { + return Collections.singletonList(expandAsStringImpl()); + } + } + + class SplitVariable extends Variable { + public SplitVariable(String name) { + super(name); + } + + @Override + protected String ref() { + return "${=" + name + "}"; + } + + @Override + public String expandAsStringImpl() { + return StringUtil.glue(" ", expandAsListImpl()); + } + + @Override + public List expandAsListImpl() { + return listExpand(varLookup(name)); + } + } + + class Command extends Expansion { + private final Sentence argv; + + public Command(List> args) { + this.argv = new Sentence(args); + } + + @Override + public void validate() { + argv.validate(); + } + + protected String run() { + List args = argv.expandAsList(); + ByteArrayOutputStream result = new ByteArrayOutputStream(); + ByteArrayOutputStream err = new ByteArrayOutputStream(); + Builder builder = new Builder(args, result, err); + builder.setLeakPrevention(LeakPrevention.ALL); + try { + int exitCode = builder.execute(); + if (exitCode != 0) + throw new UserError("Exit code " + exitCode + + " from command " + + builder.toString()); + if (err.size() > 0) + throw new UserError("Command \"" + + builder.toString() + + "\" produced output on stderr: " + err.toString()); + } catch (RuntimeException e) { + throw new UserError("Could not execute command " + + builder.toString(), e); + } + return result.toString(); + } + + @Override + public String expandAsString() { + return run(); + } + + @Override + public List expandAsList() { + return Collections.singletonList(expandAsString()); + } + } + + class SplitCommand extends Command { + public SplitCommand(List> argv) { + super(argv); + } + + @Override + public String expandAsString() { + return StringUtil.glue(" ", expandAsList()); + } + + @Override + public List expandAsList() { + return new ExpansionParser(run()).splitAsString().expandAsList(); + } + } + + private class ExpansionParser { + private final ExpansionTokeniser tokens; + + public ExpansionParser(String str) { + tokens = new ExpansionTokeniser(str); + } + + public Sentence parseAsString() { + List> words = new ArrayList>(); + words.add(parseTerminatedString(null)); + return new Sentence(words); + } + + public Sentence parseAsList() { + return new Sentence(parseTerminatedList(null, false)); + } + + public Sentence splitAsString() { + return new Sentence(parseTerminatedList(null, true)); + } + + private List parseTerminatedString(String terminator) { + List result = new ArrayList(); + + while (tokens.hasMoreTokens()) { + String next = tokens.nextToken(); + if (next.equals(terminator)) { + return result; + } else if (next.equals("\\\"")) { + result.add(new Literal("\"")); + } else if (next.equals("\\\\")) { + result.add(new Literal("\\")); + } else if (!tryParseExpansion(result, next)) { + result.add(new Literal(next)); + } + } + + if (terminator != null) + throw new UserError( + "Premature end of input while looking for matching '" + + terminator + "'."); + + return result; + } + + private List> parseTerminatedList(String terminator, + boolean noExpansions) { + List> result = new ArrayList>(); + + List accum = new ArrayList(); + boolean mustSeeSpace = false; + while (tokens.hasMoreTokens()) { + String next = tokens.nextToken(); + if (next.equals(terminator)) { + if (accum.size() > 0) + result.add(accum); + return result; + } else if (mustSeeSpace + && !Character.isWhitespace(next.charAt(0))) { + throw new UserError("The quoted string ending at " + + tokens.pos() + + " must be surrounded by whitespace."); + } else if (next.length() > 0 + && Character.isWhitespace(next.charAt(0))) { + mustSeeSpace = false; + if (accum.size() > 0) { + result.add(accum); + accum = new ArrayList(); + } + } else if (next.equals("\"")) { + if (!accum.isEmpty()) + throw new UserError( + "At position " + + tokens.pos() + + ", the quote should " + + "either be preceded by a space (if it is intended to start an argument) " + + "or escaped as \\\"."); + accum.add(new QuotedString(parseTerminatedString("\""))); + result.add(accum); + accum = new ArrayList(); + mustSeeSpace = true; + } else if (next.equals("\\\"")) { + // An escaped quote means a literal quote. + accum.add(new Literal("\"")); + } else if (next.equals("\\\\")) { + // An escaped backslash means a literal backslash. + accum.add(new Literal("\\")); + } else if (noExpansions || !tryParseExpansion(accum, next)) { + accum.add(new Literal(next)); + } + } + + if (terminator != null) + throw new UserError( + "Premature end of expansion while looking for '" + + terminator + "'."); + + if (accum.size() > 0) + result.add(accum); + + return result; + } + + private boolean tryParseExpansion(List result, + String curToken) { + if (curToken.equals("${}")) { + result.add(new Literal("$")); + } else if (curToken.equals("$(=") && commandSubstitutions) { + result.add(new SplitCommand(parseTerminatedList(")", false))); + } else if (curToken.equals("$(") && commandSubstitutions) { + result.add(new Command(parseTerminatedList(")", false))); + } else if (curToken.equals("${=")) { + result.add(new SplitVariable(parseVarName())); + } else if (curToken.equals("${")) { + result.add(new Variable(parseVarName())); + } else if (curToken.equals("$")) { + throw new UserError( + "Malformed expansion: A standalone '$' character should be escaped as '${}'."); + } else { + return false; + } + return true; + } + + protected String parseVarName() { + if (!tokens.hasMoreTokens()) + throw new UserError( + "Malformed variable substitution: stray '${' at " + tokens.pos()); + String name = tokens.nextToken(); + if (tokens.isDelimiter(name)) + throw new UserError( + "Malformed variable substitution: Unexpected '" + name + + "' at " + tokens.pos()); + if (!tokens.hasMoreTokens()) + throw new UserError( + "Malformed variable substitution for '" + name + + "': Missing '}' at " + tokens.pos()); + String next = tokens.nextToken(); + if (!next.equals("}")) + throw new UserError( + "Malformed variable substitution: Expecting '}' at " + + tokens.pos() + ", found '" + next + "'."); + return name; + } + } + } + + /** + * Resolve a path. Any variables in the path will be expanded. If + * the path is an absolute path after expansion, it is returned as is. + * Otherwise, it is combined with the given base path. + */ + public File expandPath(File base, String path) { + String expanded = strExpand(path); + if (FileUtil.isAbsolute(expanded)) { + return new File(expanded); + } else { + return FileUtil.fileRelativeTo(base, expanded); + } + } + + /** + * Escape a string so that any '$'s inside it will be interpreted literally, rather than + * as parts of variable references. + */ + public static String escape(String base) { + return base.replace("$", "${}"); + } + + /** + * Escape {@code argument} as an argument, so that any {@code $}, {@code \} or {@code "} is interpreted literally. + * + * @param argument - the String to escape. + * @return the escaped String. + */ + public static String escapeArgument(String argument) { + return escape(argument).replaceAll(Matcher.quoteReplacement("\\"), Matcher.quoteReplacement("\\\\")).replaceAll(Matcher.quoteReplacement("\""), Matcher.quoteReplacement("\\\"")); + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/extraction/SpecFileEntry.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/extraction/SpecFileEntry.java new file mode 100644 index 00000000000..f5cedaebdf2 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/extraction/SpecFileEntry.java @@ -0,0 +1,48 @@ +package com.semmle.util.extraction; + +import java.io.File; +import java.util.List; + +import com.semmle.util.data.StringUtil; + +public class SpecFileEntry { + private final File trapFolder; + private final File sourceArchivePath; + private final List patterns; + + public SpecFileEntry(File trapFolder, File sourceArchivePath, List patterns) { + this.trapFolder = trapFolder; + this.sourceArchivePath = sourceArchivePath; + this.patterns = patterns; + } + + public boolean matches(String path) { + boolean matches = false; + for (String pattern : patterns) { + if (pattern.startsWith("-")) { + if (path.startsWith(pattern.substring(1))) + matches = false; + } else { + if (path.startsWith(pattern)) + matches = true; + } + } + return matches; + } + + public File getTrapFolder() { + return trapFolder; + } + + public File getSourceArchivePath() { + return sourceArchivePath; + } + + @Override + public String toString() { + return + "TRAP_FOLDER=" + trapFolder + "\n" + + "SOURCE_ARCHIVE=" + sourceArchivePath + "\n" + + StringUtil.glue("\n", patterns); + } +} \ No newline at end of file diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/files/FileUtil.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/files/FileUtil.java new file mode 100644 index 00000000000..79ce2d8d8d3 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/files/FileUtil.java @@ -0,0 +1,1896 @@ +package com.semmle.util.files; + + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.Closeable; +import java.io.File; +import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FilePermission; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Reader; +import java.io.StringWriter; +import java.io.UnsupportedEncodingException; +import java.io.Writer; +import java.lang.reflect.UndeclaredThrowableException; +import java.net.Socket; +import java.nio.charset.Charset; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.CopyOption; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.Random; +import java.util.Set; +import java.util.Stack; +import java.util.regex.Pattern; + +import com.github.codeql.Logger; +import com.github.codeql.Severity; + +import com.semmle.util.basic.ObjectUtil; +import com.semmle.util.data.Pair; +import com.semmle.util.data.StringUtil; +import com.semmle.util.exception.CatastrophicError; +import com.semmle.util.exception.Exceptions; +import com.semmle.util.exception.ResourceError; +import com.semmle.util.files.PathMatcher.Mode; +import com.semmle.util.io.StreamUtil; +import com.semmle.util.io.csv.CSVReader; +import com.semmle.util.io.csv.CSVWriter; +import com.semmle.util.process.Env; +import com.semmle.util.process.Env.OS; + + +public class FileUtil +{ + public static Logger logger = null; + + /** + * Regular expression pattern for invalid filename characters + */ + private final static Pattern rpInvalidFilenameCharacters = Pattern.compile("[\\\\/:*?\"'<>|@]"); + + /** + * The UTF-8 Charset + */ + public static final Charset UTF8 = Charset.forName("UTF-8"); + + + /** + * List all children of a directory. This throws sensible errors if there is a problem listing the + * directory, unlike Java's listFiles method (which just returns null). + * Equivalent to list(f, null). + * + * @param f the directory in which to list children + * @return the children of f (empty if f is an empty directory) + * @throws ResourceError with an appropriate message if f does not exist, is not a + * directory, cannot be read, or some other error occurred. + */ + public static File[] list (File f) + { + return list(f, null); + } + + /** + * List all children of a directory, with an optional filter. This throws sensible errors if there + * is a problem listing the directory, unlike Java's listFiles method (which just + * returns null). It also sorts the files by their file name, so that the result is + * stable. + * + * @param f the directory in which to list children + * @param filter the filter to use for selecting which files to return, or null + * @return the children of f (empty if f is an empty directory) + * @throws ResourceError with an appropriate message if f does not exist, is not a + * directory, cannot be read, or some other error occurred. + */ + public static File[] list (File f, FileFilter filter) + { + File[] files = filter == null ? f.listFiles() : f.listFiles(filter); + if (files == null) { + boolean exists = f.exists(); + boolean isDirectory = f.isDirectory(); + boolean canRead = f.canRead(); + throw new ResourceError("Could not list the contents of directory " + + f + + " - " + + (!exists + ? "file does not exist." + : !isDirectory + ? "file is not a directory." + : !canRead + ? "cannot read - permission denied." + : "unknown I/O error.")); + } + Arrays.sort(files); + return files; + } + + /** + * Traverse a directory and collect all files selected by the given filter, returning them as a + * set. The filter should not be null; files will be added using a pre-order depth-first + * traversal. + * + * @deprecated Use FileUtil8.CollectingFileVisitor instead. + * @param dir the directory to traverse + * @param filter a filter selecting files of interest + * @return a set of transitively contained files matched by the filter + * @throws ResourceError with an appropriate message upon some error during traversal + */ + @Deprecated + public static Set recursiveFind (File dir, FileFilter filter) + { + Set result = new LinkedHashSet<>(); + recursiveFind(dir, filter, result); + return result; + } + + /** + * Traverse a directory and collect all files selected by the given filter, returning them as a + * set. The filter should not be null; files will be added using a pre-order depth-first + * traversal. Unlike {@link #recursiveFind(File, FileFilter)}, this version + * applies a second filter to directories as well, and only recurses into directories that are + * accepted by that filter. + * + * @deprecated Use FileUtil8.CollectingFileVisitor instead. + * @param dir the directory to traverse + * @param filter a filter selecting files of interest + * @param recurseFilter a filter selecting directories to recurse into. + * @return a set of transitively contained files matched by the filter + * @throws ResourceError with an appropriate message upon some error during traversal + */ + @Deprecated + public static Set recursiveFind (File dir, FileFilter filter, FileFilter recurseFilter) + { + Set result = new LinkedHashSet<>(); + recursiveFind(dir, filter, recurseFilter, result); + return result; + } + + /** + * Traverse a directory and collect all files selected by the given filter, adding them to the + * given set. The filter should not be null; files will be added using a pre-order depth-first + * traversal. + * + * @deprecated Use FileUtil8.CollectingFileVisitor instead. + * @param dir the directory to traverse + * @param filter a filter selecting files of interest + * @param result the set to which transitively contained files matched by the filter should be + * added + * @throws ResourceError with an appropriate message upon some error during traversal + */ + @Deprecated + public static void recursiveFind (File dir, FileFilter filter, Set result) + { + recursiveFind(dir, filter, null, result); + } + + /** + * Traverse a directory and collect all files selected by the given filter, adding them to the + * given set. The filter should not be null; files will be added using a pre-order depth-first + * traversal. Unlike {@link #recursiveFind(File, FileFilter, Set)}, this version + * applies a second filter to directories as well, and only recurses into directories that are + * accepted by that filter. + * + * @deprecated Use FileUtil8.CollectingFileVisitor instead. + * @param dir the directory to traverse + * @param filter a filter selecting files of interest + * @param recurseFilter a filter selecting directories to recurse into. + * @param result the set to which transitively contained files matched by the filter should be + * added + * @throws ResourceError with an appropriate message upon some error during traversal + */ + @Deprecated + public static void recursiveFind (File dir, FileFilter filter, FileFilter recurseFilter, Set result) + { + for (File f : list(dir, filter)) + result.add(f); + + FileFilter recurseDirFilter = recurseFilter == null ? + FileUtil.kindFilter(false) : + FileUtil.andFilters(FileUtil.kindFilter(false), recurseFilter); + + for (File f : list(dir, recurseDirFilter)) { + recursiveFind(f, filter, recurseFilter, result); + } + } + + /** + * Ensure the specified directory exists (as a directory), creating parent directories if + * necessary. + * + * @param dir The directory to create. + * @throws ResourceError if the desired directory already exists (but isn't a directory), or if + * the creation of it or one of its parents fails. + */ + public static void mkdirs (File dir) + { + if (dir.exists()) { + if (dir.isDirectory()) + return; + else + throw new ResourceError("Can't create " + dir + " -- it exists as a non-directory."); + } + if (dir.mkdirs()) + return; + /* + * There is a possible time-of-check time-of-use race condition where someone creates the directory + * between our existence check and our attempt to create it. + * + * In this case our goal is to ensure the existence of the directory, so it's okay if this happens. + * + * There are other possible race conditions, e.g. if someone deletes the directory after we create it, + * but we want to handle this one in particular, since multiple creations of the same directory is + * especially likely when running multiple instances of a process that use a shared directory. + */ + if (dir.isDirectory()) + return; + File child = dir; + File parent = dir.getParentFile(); + while (parent != null) { + if (parent.exists()) + throw new ResourceError("Couldn't create child directory " + child.getName() + " of " + + (parent.isDirectory() ? "" : "non-directory ") + parent + "."); + child = parent; + parent = child.getParentFile(); + } + throw new ResourceError("Couldn't create "+dir.getPath()+": no ancestor even exists."); + } + + /** + * Determines whether or not the specified string represents an absolute path on either a + * Windows-based or UNIX-based system. Absolute paths are those that start with either /, \ or + * X:\, for some letter X. + * + * @param path The string containing the path to check (can safely be null or empty). + * @return true, if the string represents an absolute path, or false otherwise. + */ + public static boolean isAbsolute (String path) + { + // Handle invalid paths gracefully. + if (path == null || path.length() == 0) + return false; + + return path.charAt(0) == '/' // Starts with / + || path.charAt(0) == '\\' // Starts with \ + || ( path.length() >= 3 // Starts with X:/ or X:\ for some character X + && Character.isLetter(path.charAt(0)) + && path.charAt(1) == ':' + && ( path.charAt(2) == '/' + || path.charAt(2) == '\\')); + } + + /** + * Write the contents of the given string to the file, creating it if it does not exist and overwriting it if it does. + * + * @param file the target file + * @param content the string to write + */ + public static void write (File file, String content) + { + Writer writer = null; + try { + writer = openWriterUTF8(file, true, false); + writer.write(content); + } + catch (IOException e) { + throw new ResourceError("Failed to write to file " + file, e); + } + finally { + close(writer); + } + } + + /** + * Append the contents of the given string to the file, creating it if it does not exist. + * + * @param file the target file + * @param content the string to write + */ + public static void append (File file, String content) + { + Writer writer = null; + try { + writer = openWriterUTF8(file, false, true); + writer.write(content); + } + catch (IOException e) { + throw new ResourceError("Failed to append to file " + file, e); + } + finally { + close(writer); + } + } + + /** + * Read a text file in its entirety. + */ + public static String readText (File f) throws IOException + { + char[] cbuf = new char[10240]; + StringBuilder buf = new StringBuilder(); + InputStreamReader reader = new InputStreamReader(new FileInputStream(f), "UTF8"); + try { + int n = reader.read(cbuf); + while (n > 0) { + buf.append(cbuf, 0, n); + n = reader.read(cbuf); + } + } + finally { + reader.close(); + } + return buf.toString(); + } + + /** + * Read a text file in its entirety. + */ + public static String readText (Path f) throws IOException + { + char[] cbuf = new char[10240]; + StringBuilder buf = new StringBuilder(); + InputStreamReader reader = new InputStreamReader(Files.newInputStream(f), "UTF8"); + try { + int n = reader.read(cbuf); + while (n > 0) { + buf.append(cbuf, 0, n); + n = reader.read(cbuf); + } + } + finally { + reader.close(); + } + return buf.toString(); + } + + /** + * Load the given file as a standard Java {@link Properties} definition. The parsing is done using + * Properties.load(), which allows comments and accepts both '=' and ':' as the key/value + * separator. + * + * @param f the file to load. + * @return a {@link Properties} object containing the loaded content of the file. + * @throws ResourceError if an IO exception occurs. + */ + public static Properties loadProperties (File f) + { + Properties result = new Properties(); + try (FileInputStream fis = new FileInputStream(f); + Reader reader = new InputStreamReader(new BufferedInputStream(fis), UTF8)) { + result.load(reader); + return result; + } catch (IOException e) { + throw new ResourceError("Failed to read properties from " + f, e); + } + } + + /** + * Save the given {@link Properties} to a file, UTF-8 encoded, with the proper + * escaping. + * + * @param props the {@link Properties} to save. + * @param f the file in which the properties should be saved. + * @throws ResourceError if an IO exception occurs. + */ + public static void writeProperties (Properties props, File f) + { + try (OutputStream os = new FileOutputStream(f); + Writer writer = new OutputStreamWriter(new BufferedOutputStream(os), UTF8)) { + props.store(writer, null); + } catch (IOException ioe) { + throw new ResourceError("Failed to write properties to " + f, ioe); + } + } + + /** + * Writes the entire contents of a stream to a file. Closes input stream when writing is done + * + * @return true on success, false otherwise + */ + public static boolean writeStreamToFile (InputStream in, File fileOut) + { + if (in == null) { + return false; + } + FileOutputStream out = null; + try { + out = new FileOutputStream(fileOut); + + byte[] buf = new byte[16 * 1024]; + int count; + while ((count = in.read(buf)) > 0) { + out.write(buf, 0, count); + } + out.flush(); + out.close(); + } + catch (IOException e) { + logger.error("Error writing stream to file", e); + return false; + } + finally { + if (out != null) { + try { + out.close(); + } + catch (IOException e) { + Exceptions.ignore(e, "We don't care about exceptions during closing"); + } + } + if (in != null) { + try { + in.close(); + } + catch (IOException e) { + Exceptions.ignore(e, "We don't care about exceptions during closing"); + } + } + } + return true; + } + + + public static void writeStreamToFile(InputStream inStream, Path resolve) throws IOException { + try (OutputStream out = Files.newOutputStream(resolve)) { + StreamUtil.copy(inStream, out); + } + } + + /** + * Convenience method that handles exception handling for creating a FileInputStream + * + * @return inputstream from file, or null on exception + */ + public static InputStream getFileInputStream (File file) + { + try { + return new FileInputStream(file); + } + catch (FileNotFoundException e) { + logger.trace("Could not open file for input stream", e); + return null; + } + } + + private static final BitSet allowedCharacters = new BitSet(256); + static { + for (int i = 'a'; i <= 'z'; i++) + allowedCharacters.set(i); + for (int i = 'A'; i <= 'Z'; i++) + allowedCharacters.set(i); + for (int i = '0'; i <= '9'; i++) + allowedCharacters.set(i); + allowedCharacters.set('-'); + allowedCharacters.set('_'); + allowedCharacters.set(' '); + allowedCharacters.set('['); + allowedCharacters.set(']'); + } + + + public static final String sanitizeFilename (String name) + { + StringBuffer safe = new StringBuffer(); + for (int i = 0; i < name.length(); i++) { + char c = name.charAt(i); + if (allowedCharacters.get(c)) + safe.append(c); + } + return safe.toString(); + } + + /** + * Create a unique file in the given directory. Takes a base name and the directory to put it in, + * and returns the file that was created. Attempts to preserve the extensions (if any) - so + * "foo.trap.gz" would become "foo-5.trap.gz", not "foo.trap.gz-5". Guarantees that the file it + * returns was successfully created by this call - so in a multithreaded / multiprocess + * environment can be used to create a globally unique file. + * + * @param baseDirectory the directory that will eventually contain the file. This must exist. + * @param fileName the simple name of the file to create. It should not contain any directory + * separators. + * @return a file in baseDirectory with a name based on fileName that + * did not exist when created. + * @throws IllegalArgumentException if baseDirectory does not exist or + * fileName is not a simple file name. + */ + public static final File createUniqueFile (File baseDirectory, String fileName) + { + return createUniqueFileImpl(baseDirectory, fileName, false); + } + + /** + * Create a unique subdirectory in the given directory. Takes a base name and the directory to put + * it in, and returns the directory that was created. Attempts to preserve the extensions (if any) + * - so "foo.d" would become "foo-5.d", not "foo.d-5". Guarantees that the directory it returns + * was successfully created by this call - so in a multithreaded / multiprocess environment can be + * used to create a globally unique directory. + * + * @param baseDirectory the directory that will eventually contain the file. This must exist. + * @param dirName the simple name of the directory to create. It should not contain any directory + * separators. + * @return a directory in baseDirectory with a name based on dirName + * that did not exist when created. + * @throws ResourceError if there was a problem creating the file + * @throws IllegalArgumentException if baseDirectory does not exist or + * dirName is not a simple name. + */ + public static final File createUniqueDirectory (File baseDirectory, String dirName) + { + return createUniqueFileImpl(baseDirectory, dirName, true); + } + + private static final File createUniqueFileImpl (File baseDirectory, String fileName, boolean directory) + { + + if (!baseDirectory.exists()) + throw new IllegalArgumentException("FileUtil.makeUniqueName(" + baseDirectory + ",\"" + fileName + "\"): " + + " directory " + baseDirectory + " does not exist."); + if (!baseDirectory.isDirectory()) + throw new IllegalArgumentException("FileUtil.makeUniqueName(" + baseDirectory + ",\"" + fileName + "\"): " + + " file " + baseDirectory + " is not a directory."); + if (fileName.contains("/")) + throw new IllegalArgumentException("FileUtil.makeUniqueName(" + baseDirectory + ",\"" + fileName + "\"): " + + " file name \"" + fileName + "\" is not a simple file name."); + + fileName = replaceInvalidFilenameChars(fileName); + + String baseName = fileName; + String name = baseName; + File candidateFile = new File(baseDirectory, name); + String extension = extension(new File(baseName)); + + int i = 1; + + try { + while (!(directory ? candidateFile.mkdir() : candidateFile.createNewFile())) { + // Add a suffix, trying to do that before the extension + if (extension.length() > 0) + name = baseName.substring(0, baseName.length() - extension.length()) + "-" + i + extension; + else + name = baseName + "-" + i; + + candidateFile = new File(baseDirectory, name); + i++; + } + } + catch (IOException e) { + throw new ResourceError("Failed to create a unique file in " + baseDirectory, e); + } + + return candidateFile; + } + + public static boolean containsInvalidFilenameChars (String filename) + { + return rpInvalidFilenameCharacters.matcher(filename).find(); + } + + public static String replaceInvalidFilenameChars (String fileName) + { + // This method gets called from all over the code base. Compile the regex only once (but only + // when it is actually needed) by using a LazyRegexPatternHolder + return rpInvalidFilenameCharacters.matcher(fileName).replaceAll("_"); + } + + /** + * Append a number to a file name, with the intention of making it unique. Similar to + * makeUniqueName, but does not actually make the name unique: it relies on the + * client calling it with a unique number. Attempts to preserve the extensions of files - so + * "foo.tar.gz" would become "foo-5.tar.gz", not "foo.tar.gz-5". + * + * @param file the filename to modify + * @param indexToAdd the number to append at the end of the filename + * @return the modified filename with indexToAdd appended at the end + */ + public static final File appendToName (File file, int indexToAdd) + { + String path = file.getPath(); + String extension = extension(file); + if (extension.length() > 0) + path = path.substring(0, path.length() - extension.length()) + "-" + indexToAdd + extension; + else + path = path + "-" + indexToAdd; + return new File(path); + } + + /** + * Get the extension of the file. Results are of the form: ".java", ".C", "." or "" (no + * extension). The '.gz' extension is treated specially - if there is another extension before it + * the two extensions are lumped together (eg. '.trap.gz', '.tar.gz'). We add another special case + * for '.xml.zip', just to support our own conventions. + */ + public static String extension (File f) + { + return extension(f.getName()); + } + + + /** + * Get the extension of the file. Results are of the form: ".java", ".C", "." or "" (no + * extension). The '.gz' extension is treated specially - if there is another extension before it + * the two extensions are lumped together (eg. '.trap.gz', '.tar.gz'). We add another special case + * for '.xml.zip', just to support our own conventions. + */ + public static String extension (Path f) + { + return extension(f.getFileName().toString()); + } + + /** + * Return the extension of the file name. + * + * @see FileUtil#extension(File) + */ + public static String extension (String name) + { + int i = name.lastIndexOf('.'); + if (i == -1) + return ""; + String extension = name.substring(i); + if (extension.equals(".gz") || extension.equals(".br")) { + // Try to find another extension + int before = name.lastIndexOf('.', i - 1); + if (before == -1) + return extension; + String combinedExtension = name.substring(before); + return combinedExtension; + } + else if (extension.equals(".zip")) { + // Just special-case .xml.zip + if (name.endsWith(".xml.zip")) + return ".xml.zip"; + } + + return extension; + } + + /** + * Return the base name of the file obtained by stripping off leading directories and the + * extension. + */ + public static String basename (File f) + { + return stripExtension(f.getName()); + } + + /** + * Return the base name of the file obtained by stripping off leading directories and the + * extension. Returns the empty string if the path has no components. + */ + public static String basename(Path path) { + Path filename = path.getFileName(); + return filename == null ? "" : stripExtension(filename.toString()); + } + + /** + * Strips the extension off a file name. E.g.: 'MyFile.java' becomes 'MyFile'. Note that + * there are some special cases (see definition of {@link #extension(String)}). For example, + * 'MyFile.tar.gz' becomes 'MyFile' (.tar.gz is considered the extension), but + * 'MyFile.foo.bar' becomes 'MyFile.foo'. + * + * Note that this method retains an optional path prefix. E.g.: + * '/foo/bar/MyFile.java' becomes '/foo/bar/MyFile'. If a completely stripped filename + * (without path, without extension) is desired, then use {@link #basename(File)}. + * + * @param name name of a file (with or without fully qualified path) + * @return input without the file extension + */ + public static String stripExtension (String name) + { + return name.substring(0, name.length() - extension(name).length()); + } + + /** + * Return a file with the same name and in the same directory as a given file, but with a + * different extension. + */ + public static File withExtension (File f, String extension) + { + return new File(withExtension(f.getPath(), extension)); + } + + /** + * Given a string denoting a file name, change its extension. + */ + public static String withExtension (String path, String extension) + { + String oldExtension = extension(path); + return path.substring(0, path.length() - oldExtension.length()) + extension; + } + + /** + * A file filter that searches for files (not directories) by any of the given names. Can do + * case-sensitive or case-insensitive matching + */ + public static FileFilter nameFilter (final boolean caseSensitive, final String ... names) + { + return new FileFilterImpl(names, caseSensitive, false); + } + + /** + * A file filter that matches a set of ant-like patterns against files in a given directory. + * + * @param cwd The current directory -- the patterns are matched against relative paths under this + * directory, and files outside of it are considered to not match the patterns (though + * they will still be accepted by the filter if {@code exclude == true}). + * @param exclude Flag indicating whether files matching the patterns should be included or + * excluded by the filter. + * @param patterns A list of patterns. + * @return The new filter. + */ + public static FileFilter antlikeFilter (File cwd, boolean exclude, String ... patterns) + { + return new PatternFilter(cwd, Mode.Ant, exclude, patterns); + } + + /** + * A file filter that matches file by regular expression. + */ + public static FileFilter regexNameFilter (String regex, boolean matchFiles) + { + return new RegexFileFilter(Pattern.compile(regex), matchFiles); + } + + /** + * A file filter that finds files (not directories) by a list of extensions. Can do case-sensitive + * or case-insensitive matching on the extensions. A sample extension should be, for instance, + * ".ql". + */ + public static FileFilter extensionFilter (final boolean caseSensitive, final String ... extensions) + { + return new FileFilterImpl(extensions, caseSensitive, true); + } + + /** + * Check whether the given directory contains any files or directories + * as defined by the given FileFilter. + */ + public static boolean containsAny(File dir, FileFilter filter){ + if(!dir.isDirectory()) + return false; + + Stack search = new Stack<>(); + search.push(dir); + + while (!search.isEmpty()) { + File f = search.pop(); + + for (File c : list(f)) { + if(filter.accept(c)) + return true; + if (f.isDirectory()) + search.push(c); + } + } + return false; + } + + /** A filter that does not accept any file or directory */ + public static final FileFilter falseFilter = new FileFilter() { + @Override + public boolean accept(File pathname) { + return false; + } + }; + + /** + * A file filter that either picks all files or all directories + */ + public static FileFilter kindFilter (final boolean files) + { + return new FileFilter() { + @Override + public boolean accept (File pathname) + { + if (pathname.isFile() && files) + return true; + if (pathname.isDirectory() && !files) + return true; + return false; + } + }; + } + + /** + * A file filter that accepts precisely the set of files specified as an argument to this method + * (using the set's {@link Set#contains(Object)} method, so up to {@link File}'s equals/hashCode). + */ + public static FileFilter setFilter (final Set acceptedFiles) + { + return new FileFilter() { + @Override + public boolean accept (File pathname) + { + return acceptedFiles.contains(pathname); + } + }; + } + + /** + * Negate a file filter + */ + public static FileFilter negateFilter (final FileFilter filter) + { + return new FileFilter() { + @Override + public boolean accept (File pathname) + { + return !filter.accept(pathname); + } + }; + } + + /** + * Take the conjunction of several file filters + */ + public static FileFilter andFilters(final FileFilter... filters) { + return new FileFilter() { + @Override + public boolean accept (File pathname) + { + for (FileFilter filter : filters) { + if (!filter.accept(pathname)) { + return false; + } + } + return true; + } + }; + } + + /** + * Sanitize path string To handle windows drive letters and cross-platform builds. + * @param pathString to be sanitized + * @return sanitized path string + */ + private static String sanitizePathString(String pathString) { + // Replace ':' by '_', as the extractor does - to handle Windows drive letters + pathString = pathString.replace(':', '_'); + + // To support cross-platform builds: if the build is done on one system (eg. Windows, with \) + // but the path is then read in another system (ie with /) then the separators will be + // interpreted incorrectly. Normalise all possible separators to the current one + pathString = pathString.replace('\\', File.separatorChar).replace('/', File.separatorChar); + return pathString; + } + + /** + * Add an absolute path as a suffix to a given directory. This is used to create source archives. + * For instance, appendAbsolutePath("/home/foo/bar", "/usr/include/stdio.h") produces + * "/home/foo/bar/usr/include/stdio.h". Various transformations on the paths are done + * to avoid special characters and to give cross-platform compatibility. + * + * @param root the File to use as a root; the result will be a child of that (or itself) + * @param absolutePath the path to append, which should be a Windows or Unix absolute path + */ + public static File appendAbsolutePath (File root, String absolutePath) + { + absolutePath = sanitizePathString(absolutePath); + + return new File(root, absolutePath).getAbsoluteFile(); + } + + /** + * Add an absolute path as a suffix to a given directory. This is used to create source archives. + * For instance, appendAbsolutePath("/home/foo/bar", "/usr/include/stdio.h") produces + * "/home/foo/bar/usr/include/stdio.h". Various transformations on the paths are done + * to avoid special characters and to give cross-platform compatibility. + * + * @param root the Path to use as a root; the result will be a child of that (or itself) + * @param absolutePath the path to append, which should be a Windows or Unix absolute path + */ + public static Path appendAbsolutePath(Path root, String absolutePathString){ + + absolutePathString = sanitizePathString(absolutePathString); + + Path path = Paths.get(absolutePathString); + + if (path.getRoot() != null) + path = path.getRoot().relativize(path); + + return root.resolve(path); + } + + /** + * Close a resource if it is non-null and has been successfully created. Silently catches + * exceptions that can occur during close. + */ + public static void close (Closeable resourceToClose) + { + if (resourceToClose != null) { + try { + resourceToClose.close(); + } + catch (IOException ignored) { + Exceptions.ignore(ignored, "Contract is to ignore"); + } + /* + * Under rare circumstances classes may lie about checked exceptions that they throw. + * Since the intention of this method is to catch all exceptions that are the result + * of IO problems, we check whether the (real) exception that was thrown was an IOException, + * and if so we ignore it. + */ + catch (UndeclaredThrowableException maybeIgnored) { + if (maybeIgnored.getCause() instanceof IOException) { + Exceptions.ignore(maybeIgnored, "Undeclared exception was an IOException, ignoring"); + } else { + throw maybeIgnored; + } + } + } + } + + /** + * Close a socket if it is non-null. Silently catches exceptions that can occur during close. + *

+ * This method is necessary because a {@link Socket} is not {@link Closeable} until Java 7. + */ + public static void close (Socket socket) + { + if (socket != null) { + try { + socket.close(); + } + catch (IOException ignored) { + Exceptions.ignore(ignored, "Contract is to ignore"); + } + } + } + + public static class ResolvedCompressedSourceArchivePaths { + public final String srcArchivePath; + public final String sourceLocationPrefix; + public ResolvedCompressedSourceArchivePaths(String srcArchivePath, String sourceLocationPrefix) { + this.srcArchivePath = srcArchivePath; + this.sourceLocationPrefix = sourceLocationPrefix; + } + } + + /** + * Resolve the paths in the compressed source archive. + * + * @param srcArchiveZip + * - the zip containing the compressed source archive. + * @param convertedSourceLocationPrefix + * - the source location prefix, converted to a source archive + * compatible format using + * {@link FileUtil#convertAbsolutePathForSourceArchive(String)}. + * @return a {@link ResolvedCompressedSourceArchivePaths} class wit + */ + public static ResolvedCompressedSourceArchivePaths resolveCompressedSourceArchivePaths(File srcArchiveZip, String convertedSourceLocationPrefix) { + String srcArchivePath = ""; + String sourcePath; + boolean legacyZip = srcArchiveZip.getName().equals("src_archive.zip"); + if (legacyZip) { + // Location of the source archive in the zip + srcArchivePath = FileUtil.convertPathForSourceArchiveZip(""); + // Location of the source directory within the source archive. + sourcePath = srcArchivePath + convertedSourceLocationPrefix; + } else if (convertedSourceLocationPrefix.startsWith("/")) { + sourcePath = convertedSourceLocationPrefix.substring(1); + } else { + sourcePath = convertedSourceLocationPrefix; + } + return new ResolvedCompressedSourceArchivePaths(srcArchivePath, sourcePath); + } + + /** + * Converts an absolute path to a path that can be used in the source archive. This involves + * normalising (and canonicalising) the path, stripping any trailing slash, and + * prepending a slash if the path is non-empty. + * + * @param absolutePath The absolute path to convert (must be non-null). + * @return The converted path. + * @throws CatastrophicError If absolutePath is null. + */ + public static String convertAbsolutePathForSourceArchive(String absolutePath) + { + // Enforce preconditions. + if (absolutePath == null) { + throw new CatastrophicError("FileUtil.convertPathForSourceArchiveZip: absolutePath must be non-null"); + } + + // Normalise the path and replace any instances of the Windows-specific path character ':'. + absolutePath = normalisePath(absolutePath); + absolutePath = absolutePath.replace(':', '_'); + + // Make sure that the path starts with a forward slash (to separate it from "src_archive") + // and then strip any trailing forward slash. (Note that if the original path starts off + // empty, the net result of these two operations is still empty.) + if (!absolutePath.startsWith("/")) + absolutePath = "/" + absolutePath; + if (absolutePath.endsWith("/")) + absolutePath = absolutePath.substring(0, absolutePath.length() - 1); + + return absolutePath; + } + + /** + * Converts an absolute path to a path that can be used in the source archive .zip. This involves + * normalising (and canonicalising) the path, stripping any trailing slash and prepending the + * string "src_archive/" (unless the path is empty, when the slash after "src_archive" is + * dropped). + * + * @param absolutePath The absolute path to convert (must be non-null). + * @return The converted path. + * @throws CatastrophicError If absolutePath is null. + */ + public static String convertPathForSourceArchiveZip(String absolutePath) { + return "src_archive" + convertAbsolutePathForSourceArchive(absolutePath); + } + + /** + * Construct a child of base by appending a relative path. For example: with base="/usr" and + * relativePath="local/bin/foo", the result is "/usr/local/bin/foo"; if base="C:\odasa" and + * relativePath="projects/foo", then the result is "C:\odasa\projects\foo". The relative path must + * not start with a slash. Normalisation: treat both "/" and "\" as the path separator. As a + * special case, if {@code base} is null, a file representing just the + * {@code relativePath} is constructed. + */ + public static File fileRelativeTo (File base, String relativePath) + { + if (!isRelativePath(relativePath)) + throw new CatastrophicError("Invalid relative path '" + relativePath + "'."); + + return new File(base, relativePath); + } + + /** + * Is the given path a relative path suitable for {@link #fileRelativeTo(File, String)}? + */ + public static boolean isRelativePath (String path) + { + if (path.startsWith("/")) { + return false; + } + if (path.startsWith("\\")) { + return false; + } + if (Env.getOS() == OS.WINDOWS && path.contains(":")) { + return false; + } + return true; + } + + /** + * Converts a path to a normalised form. This involves converting any initial lowercase drive + * letter to uppercase, and converting backslashes to forward slashes. + * + * @param path The path to normalise (must be non-null). + * @return The normalised version of the path. + * @throws CatastrophicError If {@code path} is null. + */ + public static String normalisePath (String path) + { + // Enforce preconditions. + if (path == null) { + throw new CatastrophicError("FileUtil.normalisePath: path must be non-null"); + } + + // Convert any initial lowercase driver letter to uppercase. + if (path.length() >= 2 && path.charAt(1) == ':') { + char driveLetter = path.charAt(0); + if (driveLetter >= 'a' && driveLetter <= 'z') { + path = Character.toUpperCase(driveLetter) + path.substring(1); + } + } + + // Convert any backslashes to forward slashes. + path = path.replace('\\', '/'); + + return path; + } + + /** + * Compute the nearest common parent for two files. + * + * @return The most nested directory which is both a parent of {@code a} and a parent of {@code b} + * . + * @throws ResourceError if no common parent can be found. This can happen if, for example, the + * two files are on different Windows drive letters. + */ + public static File commonParent (File a, File b) + { + Set parents = new LinkedHashSet<>(); + for (File cur = a; cur != null; cur = cur.getParentFile()) + parents.add(cur); + for (File cur = b; cur != null; cur = cur.getParentFile()) + if (!parents.add(cur)) + return cur; + throw new ResourceError("Could not determine a common parent for " + a + " and " + b + "."); + } + + /** + * Compute the nearest common parent for a set of files. + * + * @return The most nested directory which is a parent of all files in {@code fs}. + * @throws ResourceError if no common parent can be found. This can happen if, for example, two of + * the files are on different Windows drive letters, or the list is empty. + */ + public static File commonParent (List fs) + { + if (fs.isEmpty()) { + throw new CatastrophicError("No files to find common parent of"); + } + File cur = fs.get(0); + for (File f : fs) { + cur = commonParent(cur, f); + } + return cur; + } + + /** + * Compute the relative path to file from baseDirectory. Fails with a + * {@link ResourceError} if file is not a child of baseDirectory. Tries + * canonical paths and comparing normal file paths. The returned path is relative (ie does not + * start with a slash). + * + * @throws ResourceError if a relative path cannot be determined + */ + public static String relativePath (File file, File baseDirectory) + { + String candidate = relativePathOpt(file, baseDirectory); + if (candidate != null) + return candidate; + + throw new ResourceError("Could not determine a relative path to " + file + " from " + baseDirectory); + } + + /** + * Compute the relative path to file from baseDirectory. Returns + * file.getAbsolutePath() if file is not a child of + * baseDirectory. Tries canonical paths and comparing normal file paths. + */ + public static String tryMakeRelativePath (File file, File baseDirectory) + { + String candidate = relativePathOpt(file, baseDirectory); + return candidate != null ? candidate : file.getAbsolutePath(); + } + + public static String relativePathOpt (File file, File baseDirectory) + { + try { + File canonicalToFile = file.getCanonicalFile(); + File canonicalToBase = baseDirectory.getCanonicalFile(); + String canonicalRelative = relativePathAsIsOpt(canonicalToFile, canonicalToBase); + if (canonicalRelative != null) + return canonicalRelative; + } + catch (IOException ignored) { + Exceptions.ignore(ignored, "Fall through to comparing standard paths"); + } + + String relative = relativePathAsIsOpt(file, baseDirectory); + return relative; + } + + /** + * The same as {@link #relativePathOpt(File, File)}, but it does not canonicalize its arguments. + */ + public static String relativePathAsIsOpt (File childFile, File parentFile) + { + String child = childFile.getPath(); + String parent = parentFile.getPath(); + int parentLength = parent.length(); + // Is the child too short? + if (child.length() <= parentLength) + return null; + // Is the parent not even a prefix? + if (!child.startsWith(parent)) + return null; + // Is the parent prefix a full dir name? (catches child=/home and parent=/ho) + if (child.charAt(parentLength) == File.separatorChar) + return child.substring(parentLength + 1); + // We also need to check the previous character to handle + // cases like parent=/ or parent=c:\ + if (parentLength > 0 && child.charAt(parentLength - 1) == File.separatorChar) + return child.substring(parentLength); + return null; + } + + public static boolean isWithin (File file, File baseDirectory) + { + return (relativePathOpt(file, baseDirectory) != null) + || (tryMakeCanonical(file).equals(tryMakeCanonical(baseDirectory))); + } + + /** + * Constructs the relative path to the target {@code f} from {@code base}. The base file or + * directory does not need to be a parent of the target but a common parent needs to exist. + * + * @param f the target file + * @param base the working directory (or a file within the working directory) + * @return the relative path from {@code base} to {@code f} + * @throws ResourceError if there is no common parent according to {@link FileUtil#commonParent} + */ + public static String relativePathLink (File f, File base) + { + f = f.getAbsoluteFile(); + base = base.getAbsoluteFile(); + File parent = commonParent(f, base); + StringBuilder path = new StringBuilder(); + for (File cur = base; !cur.equals(parent); cur = cur.getParentFile()) + path.append(".." + File.separator); + path.append(relativePath(f, parent)); + return path.toString(); + } + + /** + * Try to convert a file into a canonical file. Handles the possible IO exception by just making + * the path absolute. + */ + public static File tryMakeCanonical (File f) + { + try { + return f.getCanonicalFile(); + } + catch (IOException ignored) { + Exceptions.ignore(ignored, "Can't log error: Could be too verbose."); + return new File(simplifyPath(f)); + } + } + + + private static class FileFilterImpl implements FileFilter + { + private final String[] names; + private final boolean caseSensitive; + private final boolean extensionOnly; + + + private FileFilterImpl (String[] names, boolean caseSensitive, boolean extensionOnly) + { + this.names = Arrays.copyOf(names, names.length); + this.caseSensitive = caseSensitive; + this.extensionOnly = extensionOnly; + + if (!caseSensitive) + for (int i = 0; i < this.names.length; i++) + this.names[i] = StringUtil.lc(this.names[i]); + } + + @Override + public boolean accept (File pathname) + { + if (!pathname.isFile()) + return false; + + String nameToMatch = caseSensitive ? pathname.getName() : StringUtil.lc(pathname.getName()); + for (String s : names) { + if (extensionOnly && nameToMatch.endsWith(s)) + return true; + if (!extensionOnly && nameToMatch.equals(s)) + return true; + } + return false; + } + } + + private static class PatternFilter implements FileFilter + { + private final PathMatcher matcher; + private final boolean exclude; + private final File cwd; + + + private PatternFilter (File cwd, Mode mode, boolean exclude, String ... patterns) + { + this.cwd = cwd; + this.exclude = exclude; + matcher = new PathMatcher(mode, Arrays.asList(patterns)); + } + + @Override + public boolean accept (File f) + { + String path = relativePathOpt(f, cwd); + if (path == null) + return exclude; + else + return exclude ^ matcher.matches(path); + } + } + + private static class RegexFileFilter implements FileFilter + { + private final Pattern pattern; + private final boolean matchFiles; + + + private RegexFileFilter (Pattern pattern, boolean matchFiles) + { + this.pattern = pattern; + this.matchFiles = matchFiles; + } + + @Override + public boolean accept (File pathname) + { + if (pathname.isFile() != matchFiles) { + return false; + } + return pattern.matcher(pathname.getName()).matches(); + } + } + + + /** + * Is parent a (recursive) parent of child? The case where parent is the + * same as child is not counted. This could return the wrong result in odd + * situations involving links, but tries to make paths canonical if possible. + */ + public static boolean recursiveParentOf (File parent, File child) + { + return relativePathOpt(child, parent) != null; + } + + /** + * Delete the given file, even if it is a non-empty directory -- in which case it is traversed and + * all contents are removed. This method makes a best-effort attempt to avoid infinite loops + * through symlinks, though that part is largely untested at the time of writing. + * + * @deprecated use FileUtil8.recursiveDelete instead. + * @param file The file or directory that should be deleted + * @return true, if the file or directory was successfully deleted, or false otherwise. + */ + @Deprecated + public static boolean recursiveDelete (File file) + { + return recursiveDelete(file, falseFilter) == DeletionResult.Deleted; + } + + + public enum DeletionResult + { + Deleted, SkippedSomeFiles, Failed + }; + + + /** + * Delete the given file, even if it is a non-empty directory; however, preserve files (or + * directories) that match the given {@link FileFilter} for exceptions. This method makes a + * best-effort attempt to avoid infinite loops through symlinks, though that part is largely + * untested at time of writing [in particular, a loop of read-only symlinks may lead to infinite + * recursion]. + * + * @deprecated use FileUtil8.recursiveDelete instead. + * @param file the file or folder to delete + * @param exceptions a {@link FileFilter} that will be consulted before attempting to delete + * anything; if it accepts, the current file is not deleted (which means its parent + * directories will be preserved, too). + * @return A {@link DeletionResult} indicating the status: Deleted if the file has been + * successfully deleted (and no longer exists), SkippedSomeFiles if the file exists either + * because it was skipped or because it's the parent directory of some skipped files, and + * Failed if a file we tried to delete could not be removed. + */ + @Deprecated + public static DeletionResult recursiveDelete (File file, FileFilter exceptions) + { + if (exceptions.accept(file)) + return DeletionResult.SkippedSomeFiles; + if (file.delete()) + return DeletionResult.Deleted; + if (file.isDirectory()) { + File[] children = file.listFiles(); + if (children == null) + return DeletionResult.Failed; + boolean skippedSomeChildren = false; + for (File f : children) { + DeletionResult childResult = recursiveDelete(f, exceptions); + if (childResult == DeletionResult.Failed) + return DeletionResult.Failed; + else if (childResult == DeletionResult.SkippedSomeFiles) + skippedSomeChildren = true; + } + if (file.delete()) + return DeletionResult.Deleted; + else + return skippedSomeChildren ? DeletionResult.SkippedSomeFiles : DeletionResult.Failed; + } + else + return DeletionResult.Failed; + } + + /** + * Takes a File path and return another File path that has a different extension. Notice that this + * operation works on the path name represented by the File object. No attempt to resolve + * symlinks, or otherwise canonicalize the path name, is made. The file referenced by the file + * path may or may not exist. + * + * @param file the path from which to generate the new path + * @param newExtension the desired extension + * @return a file path with a different extension + */ + public static File replaceFileExtension (File file, String newExtension) + { + String name = file.getName(); + int index = name.lastIndexOf('.'); + if (index != -1) { + name = name.substring(0, index); + } + return new File(file.getParentFile(), name + newExtension); + } + + /** + * Return the SHA-1 hash for a file. + * + * @param file the file to hash + * @return a String representation of the hash in hexadecimal + */ + public static String sha1 (File file) + { + byte[] buf = new byte[4096]; + FileInputStream fio = null; + try { + fio = new FileInputStream(file); + MessageDigest digest = MessageDigest.getInstance("SHA-1"); + int read; + while ((read = fio.read(buf)) > 0) + digest.update(buf, 0, read); + return StringUtil.toHex(digest.digest()); + } catch (IOException e) { + throw new ResourceError("Could not read file for hashing: " + file, e); + } + catch (NoSuchAlgorithmException e) { + throw new ResourceError("Could not find SHA-1 algorithm", e); + } + finally { + close(fio); + } + } + + /** + * Append the given set of properties to a specified file, taking care of proper escaping of + * special characters. + * + * @param target the file to write -- content is appended + * @param extraVariables A string/value mapping of properties + * @param extraComment A properties-file commend to prepend to the new values, to trace + * provenance. Can be {@code null} to disable. + */ + public static void appendProperties (File target, Set> extraVariables, String extraComment) + { + if (extraVariables.isEmpty()) + return; + + Properties props = new Properties(); + + for (Pair var : extraVariables) + props.put(var.fst(), var.snd()); + + StringWriter writer = new StringWriter(); + writer.append('\n'); + try { + props.store(writer, extraComment); + } + catch (IOException e) { + throw new ResourceError("Failed to convert properties to string while appending to file " + target, e); + } + + append(target, writer.toString()); + } + + /** + * Append the given set of properties to a specified file, taking care of proper escaping of + * special characters. + * + * @param target the file to write -- content is appended + * @param extraVariables A string/value mapping of properties + * @param extraComment A properties-file commend to prepend to the new values, to trace + * provenance. Can be {@code null} to disable. + */ + public static void appendProperties (File target, Map extraVariables, String extraComment) + { + Set> vars = new LinkedHashSet<>(); + for (Entry e : extraVariables.entrySet()) + vars.add(Pair.make(e.getKey(), e.getValue())); + appendProperties(target, vars, extraComment); + } + + + /** + * A pattern matching "logical" path separators, i.e. runs of forward or back slashes, possibly + * interspersed with single dots. A string matched by this pattern is equivalent to a single + * occurrence of the separator. + */ + private static final Pattern ANY_PATH_SEPARATOR = Pattern.compile("[\\\\/]+(\\.[\\\\/]+)*"); + + /** + * Normalise a string representing a path by replacing sequences that match + * {@link #ANY_PATH_SEPARATOR} with a single forward slash. The transformation makes no + * reference to a file system. Example transformations: + *

    + *
  • a/./b to a/b
    • + *
  • a\b to a/b
    • + *
  • a\b/c to a/b/c
    • + *
  • C:\a\b to C:/a/b
    • + *
  • /a/b to /a/b
    • + *
  • . to . + *
  • ./ to ./ + *
+ */ + public static String normalizePathSeparators(String path) + { + return ANY_PATH_SEPARATOR.matcher(path).replaceAll("/"); + } + + /** + * Normalise a file name without incurring the cost of filesystem access that + * {@link File#getCanonicalFile()} would. In particular, remove redundant "./" + * components from the path, and simplify "foo/.." to nothing. The path is made + * absolute in the process. + * + * @param file + * the file path to normalise. + * @return A string representing the path of the file, with obviously redundant + * components stripped off. + */ + public static String simplifyPath (File file) + { + return file.toPath().toAbsolutePath().normalize().toString(); + } + + /** + * Read properties from a CSV file into a Map + * + * @param file CSV file, each row containing a key and value + * @return Map containing key-value bindings + */ + public static Map readPropertiesCSV (File file) + { + Map result = new LinkedHashMap<>(); + Reader reader = null; + InputStream input = null; + CSVReader csv = null; + try { + input = new FileInputStream(file); + reader = new InputStreamReader(input, "UTF-8"); + csv = new CSVReader(reader); + for (String[] line : csv.readAll()) { + if (line.length >= 1) { + String key = line[0]; + String value = null; + if (line.length >= 2) { + value = line[1]; + } + result.put(key, value); + } + } + return Collections.unmodifiableMap(result); + } + catch (UnsupportedEncodingException e) { + throw new CatastrophicError(e); + } + catch (IOException e) { + throw new ResourceError("Could not read data from " + file, e); + } + finally { + FileUtil.close(reader); + FileUtil.close(input); + FileUtil.close(csv); + } + } + + /** + * Write properties to a CSV file + * + * @param file CSV file + * @param data Map containing key-value bindings + */ + public static void writePropertiesCSV (File file, Map data) throws IOException + { + OutputStream out = null; + Writer writer = null; + CSVWriter csvWriter = null; + try { + out = new FileOutputStream(file); + writer = new OutputStreamWriter(out, "UTF-8"); + csvWriter = new CSVWriter(writer); + + for (Entry e : data.entrySet()) { + csvWriter.writeNext(e.getKey(), e.getValue()); + } + } + catch (FileNotFoundException e) { + throw new ResourceError("Could not find file '" + file + "'", e); + } + catch (UnsupportedEncodingException e) { + throw new CatastrophicError(e); + } + finally { + FileUtil.close(csvWriter); + FileUtil.close(writer); + FileUtil.close(out); + } + } + + /** + * Obtain a File that does not exist, returning the given {@code path} if it does not exist, or + * the given {@code path} with a randomly generated numerical suffix if it does. + * + * @throws ResourceError if the {@code path} could not be accessed. + */ + public static final File ensureUnique (File path) + { + try { + if (path.exists()) { + Random random = new Random(); + File uniquePath; + do { + uniquePath = new File(path.toString() + "." + random.nextInt()); + } + while (uniquePath.exists()); + return uniquePath; + } + return path; + } + catch (SecurityException se) { + throw new ResourceError("Could not access file " + path, se); + } + } + + /** + * Open the specified {@code file} with a new {@link Writer}. + * + * @param file The file to open + * @param overwrite Flag indicating whether the file may be overwritten if it already exists. + * @param append Flag indicating whether the file should be appended if it already exists ( + * {@code overwrite} is ignored if this flag is true). + */ + public static Writer openWriterUTF8 (File file, boolean overwrite, boolean append) + { + try { + FileOutputStream ostream; + if (file.exists()) { + if (append) { + ostream = new FileOutputStream(file, true); + } + else if (overwrite) { + if (!file.delete()) { + throw new ResourceError("Could not delete existing file: " + file); + } + ostream = new FileOutputStream(file, false); + } + else { + throw new ResourceError("File already exists: " + file); + } + } + else { + ostream = new FileOutputStream(file, false); + } + return new OutputStreamWriter(ostream, UTF8); + } + catch (SecurityException se) { + throw new ResourceError("Could not access file " + file, se); + } + catch (IOException ioe) { + throw new ResourceError("Failed to open FileWriter for " + file, ioe); + } + } + + /** + * Rename a file, creating any directories as needed. If the destination file (or directory) + * exists, it is overwritten. + * + * @param src The file to be renamed. Must be non-null and must exist. + * @param dest The file's new name. Must be non-null. Will be overwritten if it already exists. + */ + public static void forceRename (File src, File dest) + { + final String errorPrefix = "FileUtil.forceRename: "; + if (src == null) + throw new CatastrophicError(errorPrefix + "source File is null."); + if (dest == null) + throw new CatastrophicError(errorPrefix + "destination File is null."); + if (!src.exists()) + throw new ResourceError(errorPrefix + "source File '" + src.toString() + "' does not exist."); + + // File.renameTo(foo) requires that foo's parent directory exists. + mkdirs(dest.getAbsoluteFile().getParentFile()); + if (dest.exists() && !recursiveDelete(dest)) + throw new ResourceError(errorPrefix + "Couldn't overwrite destination file '" + dest.toString() + "'."); + if (!src.renameTo(dest)) + throw new ResourceError(errorPrefix + "Couldn't rename file '" + src.toString() + "' to '" + dest.toString() + + "'."); + } + + /** + * Query whether a {@link File} is non-null, and represents an existing file that can be + * read. + */ + public static boolean isReadableFile (File path) + { + return path != null && path.isFile() && path.canRead(); + } + + /** + * Compare a pair of paths using their canonical form to determine if they resolve to identical + * paths. Returns false if either is null. + */ + public static boolean isSamePath (File path1, File path2) + { + // Quick break-out if either path is null + if (path1 == null || path2 == null) { + return false; + } + // Compare the canonical paths + return ObjectUtil.equals(tryMakeCanonical(path1), tryMakeCanonical(path2)); + } + + /** + * Add the extension {@code extension} to the {@link File} {@code file}. + * + * @param file - the File to which we want to add the {@code extension}. + * @param extension - the extension, without the dot, which will be appended. + * @return a copy of {@code file} with the extension added. + */ + public static File addExtension (File file, String extension) + { + return new File(file.getPath() + "." + extension); + } + + /** + * Ensures the existence of a given file. The method does nothing if the file already exists and creates a + * new one if it does not. + * @throws ResourceError if f already exists and is not a file. + */ + public static void ensureFileExists(File f){ + try{ + if(!f.exists()){ + mkdirs(f.getAbsoluteFile().getParentFile()); + if(!f.createNewFile()) { + throw new ResourceError("Cannot create file '" + f + "', since a directory with the same name already exists!"); + } + }else if(f.isDirectory()){ + throw new ResourceError("Cannot create file '" + f + "', since a directory with the same name already exists!"); + } + }catch(IOException ioe){ + throw new ResourceError("Could not create file '" + f + "'", ioe); + } + } + + /** + * Copy a resource within a class into a file. + * @param clazz The class responsible for the resource. + * @param resourceName The resource's name. + * @param target The file to copy the resource to. + */ + /* + public static void resourceToFile(Class clazz, String resourceName, File target){ + try { + Files.createDirectories(target.toPath().getParent()); + try (InputStream is = clazz.getResourceAsStream(resourceName); + AtomicFileOutputStream afos = new AtomicFileOutputStream(target.toPath()); + BufferedOutputStream bos = new BufferedOutputStream(afos)) { + StreamUtil.copy(is, bos); + } + } catch (IOException e) { + throw new ResourceError("Error copying resource '" + clazz.getName() + "' '" + resourceName + "' to '" + target + "'", e); + } + } + */ + + /** + * Attempts to create a temporary directory. + */ + public static File createTempDir(){ + File globalTempDir = new File(System.getProperty("java.io.tmpdir")); + return createUniqueDirectory(globalTempDir, "semmleTempDir"); + } + + /** + * Magic constant: the maximum number of file operation attempts in + * {@link #performWithRetries}. + */ + private static final int FILE_OPERATION_ATTEMPTS_LIMIT = 30; + /** + * Magic constant: the delay between file operation attempts in + * {@link #performWithRetries}. Chosen to overapproximate the time Windows + * Defender takes to scan directories our code creates. + */ + private static final int FILE_OPERATION_ATTEMPTS_DELAY_MS = 1000; + + /** + * Functional interface resembling a {@link BiConsumer} of {@link Path}s, + * but whose {@code accept} method may throw an {@link IOException}. + */ + private static interface RetryablePathConsumer { + void accept(Path source, Path target) throws IOException; + } + + /** + * In Java 8, this would just be BiFunction. + */ + private static interface ErrorMessageCreator { + String apply(Path source, Path target); + } + + /** + * Attempts to perform the given {@code operation} on the {@code source} and + * {@code target} paths. + * + * If the operation fails, it is attempted again after a + * {@value #FILE_OPERATION_ATTEMPTS_DELAY_MS} ms delay. This process is + * repeated until the operation succeeds, or a total of + * {@value #FILE_OPERATION_ATTEMPTS_LIMIT} attempts are made, at which point + * an error is thrown. The given {@code errorMessageCreator} is used to + * construct suitable messages upon retries or failure. + *

+ * This is useful when either the source or the target has been created just + * before the attempted operation. (For example, atomic directory creation + * usually involves creating a temporary directory with the desired contents + * and then moving it to the desired location, or temporary directories may + * be created, populated, and then deleted after use.) + * + * Aggressive variants of Windows Defender (ATP) tend to scan such newly-created + * files immediately, and the operation will only succeed if attempted again + * after Defender releases the files. + * + * @throws IOException + * if the operation does not succeed after + * {@value #FILE_OPERATION_ATTEMPTS_LIMIT} attempts + */ + private static void performWithRetries(RetryablePathConsumer operation, Path source, Path target, + ErrorMessageCreator errorMessageCreator) throws IOException { + for (int i = 1;; i++) { + try { + operation.accept(source, target); + return; + } catch (AtomicMoveNotSupportedException e) { + // Allow this to propagate, since retrying won't help. + throw e; + } catch (IOException e) { + String message = errorMessageCreator.apply(source, target); + logger.warn(message + " (attempt " + i + ")", e); + if (i == FILE_OPERATION_ATTEMPTS_LIMIT) + throw new IOException(message + "(" + FILE_OPERATION_ATTEMPTS_LIMIT + " attempts made)", e); + } + try { + logger.trace("Waiting for " + FILE_OPERATION_ATTEMPTS_DELAY_MS + " ms before making another attempt."); + Thread.sleep(FILE_OPERATION_ATTEMPTS_DELAY_MS); + } catch (InterruptedException e) { + logger.warn("Thread interrupted before making another attempt.", e); + } + } + } + + /** + * Attempts to move the {@code source} file to the {@code target} file. + * Wraps {@link #performWithRetries} around + * {@link java.nio.file.Files#move} to retry the move if it fails. + * + * @see #performWithRetries + */ + public static void moveWithRetries(Path source, Path target, final CopyOption... options) throws IOException { + performWithRetries( + new RetryablePathConsumer() { + @Override + public void accept(Path source, Path target) throws IOException { Files.move(source, target, options); } + }, + source, target, + new ErrorMessageCreator() { + @Override + public String apply(Path s, Path t) { return "Failed to perform move from " + s.toAbsolutePath() + " to " + t.toAbsolutePath(); } + }); + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/files/PathMatcher.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/files/PathMatcher.java new file mode 100644 index 00000000000..56563960144 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/files/PathMatcher.java @@ -0,0 +1,160 @@ +package com.semmle.util.files; + +import java.util.regex.Pattern; + +import com.semmle.util.data.StringUtil; + +/** + * Utility class to match a string to a pattern, which can either be + * an ant-like include/exclude pattern (with wildcards), or a rsync-like + * pattern. + *

+ * In ant-like mode: + *

    + *
  • '**' matches zero or more characters (most notably including '/'). + *
  • '*' matches zero or more characters except for '/'. + *
  • '?' matches any character (other than '/'). + *
+ *

+ * In rsync-like mode: + *

    + *
  • A pattern is matched only at the root if it starts with '/', and otherwise + * it is matched against each level of the directory tree. + *
  • '**', '*' and '?' have the same meaning as for ant. + *
  • Other rsync features (like [:..:] groups and backslash-escapes) are not supported. + *
+ */ +public class PathMatcher { + + public enum Mode { + Ant, Rsync; + } + + private final Mode mode; + private final Pattern pattern; + private final String originalPattern; + + /** + * Create a {@link PathMatcher}. + * + * @param pattern An ant-like pattern + */ + public PathMatcher(String pattern) { + this(Mode.Ant, pattern); + } + + /** Create a {@link PathMatcher}. + * + * @param mode The {@link Mode} to use + * @param pattern A pattern, interpreted as ant-like or rsync-like depending on + * the value of {@code mode} + */ + public PathMatcher(Mode mode, String pattern) { + this.mode = mode; + this.originalPattern = pattern; + StringBuilder b = new StringBuilder(); + toRegex(b, pattern); + this.pattern = Pattern.compile(b.toString()); + } + + /** Create a {@link PathMatcher}. + * + * @param patterns Several ant-like patterns + */ + public PathMatcher(Iterable patterns) { + this(Mode.Ant, patterns); + } + + /** Create a {@link PathMatcher}. + * + * @param mode The {@link Mode} to use. + * @param patterns Several patterns, interpreted as ant-like or rsync-like depending + * on the value of {@code mode}. + */ + public PathMatcher(Mode mode, Iterable patterns) { + this.mode = mode; + this.originalPattern = patterns.toString(); + StringBuilder b = new StringBuilder(); + for (String pattern : patterns) { + if (b.length() > 0) + b.append('|'); + toRegex(b, pattern); + } + this.pattern = Pattern.compile(b.toString()); + } + + private void toRegex(StringBuilder b, String pattern) { + if (pattern.length() == 0) return; + //normalize pattern path separators + pattern = pattern.replace('\\', '/'); + //replace double slashes + pattern = pattern.replaceAll("//+", "/"); + // escape + pattern = StringUtil.escapeStringLiteralForRegexp(pattern, "*?"); + + // for ant, ending with '/' is shorthand for "/**" + if (mode == Mode.Ant && pattern.endsWith("/")) pattern = pattern + "**"; + + // replace "**/" with (^|.*/)" + // replace "**" with ".*" + // replace "*" with "[^/]* + // replace "?" with "[^/]" + int i = 0; + + // In rsync-mode, a leading slash is an 'anchor' -- the pattern is only matched + // when rooted at the start of the path. This is the default behaviour for ant-like + // patterns. + if (mode == Mode.Rsync) { + if (pattern.charAt(0) == '/') { + // The slash is just anchoring, and may actually be missing + // in the case of a relative path. + b.append("/?"); + i++; + } else { + // Non-anchored rsync pattern: the pattern can match at any level in the tree. + b.append("(.*/)?"); + } + } + + while (i < pattern.length()) { + char c = pattern.charAt(i); + if (c == '*' && i < pattern.length() - 2 && pattern.charAt(i+1) == '*' && pattern.charAt(i+2) == '/') { + b.append("(?:^|.*/)"); + i += 3; + } + else if (c == '*' && i < pattern.length() - 1 && pattern.charAt(i+1) == '*') { + b.append(".*"); + i += 2; + } + else if(c == '*') { + b.append("[^/]*"); + i++; + } + else if(c == '?') { + b.append("[^/]"); + i++; + } + else { + b.append(c); + i++; + } + } + } + + /** + * Match the specified path against a shell pattern. The path is normalised by replacing '\' with '/'. + * @param path The path to match. + */ + public boolean matches(String path) { + // normalise path + path = path.replace('\\', '/'); + if(path.endsWith("/")) + path = path.substring(0, path.length()-1); + return pattern.matcher(path).matches(); + } + + @Override + public String toString() { + return "Matches " + originalPattern + " [" + pattern + "]"; + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/io/BufferedLineReader.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/io/BufferedLineReader.java new file mode 100644 index 00000000000..99c0b7d1569 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/io/BufferedLineReader.java @@ -0,0 +1,103 @@ +package com.semmle.util.io; + +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.IOException; +import java.io.Reader; + +import com.semmle.util.files.FileUtil; + +/** + * A custom buffered reader akin to {@link BufferedReader}, except that it preserves + * line terminators (and so its {@code readLine()} method is called + * {@link #readLineAndTerminator()}). The other {@link Reader} methods should not + * be called, and will throw. + */ +public class BufferedLineReader implements Closeable { + private final char[] buffer = new char[8192]; + private int nextChar = 0, nChars = 0; + private final Reader in; + + public BufferedLineReader(Reader in) { + this.in = in; + } + + /** + * Read the string up to and including the next CRLF or LF terminator. This method + * may return a non-terminated string at EOF, or if a line is too long to fit in the + * internal buffer. Calls will block until enough data has been read to fill the + * buffer or find a line terminator. + * @return The next line (or buffer-full) of text. + * @throws IOException if the underlying stream throws. + */ + public String readLineAndTerminator() throws IOException { + int terminator = findNextLineTerminator(); + if (terminator == -1) + return null; + String result = new String(buffer, nextChar, terminator - nextChar + 1); + nextChar = terminator + 1; + return result; + } + + /** + * Get the index of the last character that should be included in the next line. + * Usually, this is the LF in a LF or CRLF line terminator, but it might be the + * end of the buffer (if it is full, and no newlines are present), or it may be + * -1 (but only if EOF has been reached, and the buffer is currently empty). + * The first character of the line is pointed to by {@link #nextChar}, which + * may be modified by this method if the buffer is refilled. + */ + private int findNextLineTerminator() throws IOException { + int alreadyChecked = 0; + do { + for (int i = nextChar + alreadyChecked; i < nChars; i++) { + if (buffer[i] == '\r' && i+1 < nChars && buffer[i+1] == '\n') + return i+1; // CRLF + else if (buffer[i] == '\n') + return i; // LF + } + + // We didn't find a full newline in the existing buffer: Try to fill. + alreadyChecked = nChars - nextChar; + int newlyRead = fill(); + if (newlyRead <= 0) + return nChars - 1; + } while (true); + } + + /** + * Block until at least one character from the underlying stream is read, + * or EOF is reached. + */ + private int fill() throws IOException { + if (nextChar >= nChars) { + // No unread characters. + nextChar = 0; + nChars = 0; + } else if (nextChar > 0) { + // Some unread characters. + System.arraycopy(buffer, nextChar, buffer, 0, nChars - nextChar); + nChars = nChars - nextChar; + nextChar = 0; + } + + // Is the buffer full? + if (nChars == buffer.length) + return 0; + + int read; + do { + read = in.read(buffer, nChars, buffer.length - nChars); + } while (read == 0); + + if (read > 0) { + nChars += read; + } + return read; + } + + @Override + public void close() { + FileUtil.close(in); + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/io/RawStreamMuncher.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/io/RawStreamMuncher.java new file mode 100644 index 00000000000..29ca3538530 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/io/RawStreamMuncher.java @@ -0,0 +1,34 @@ +package com.semmle.util.io; + +import com.semmle.util.exception.Exceptions; +import com.semmle.util.files.FileUtil; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * A thread that copies data from an input stream to an output stream. When + * the input stream runs out, it closes both the input and output streams. + */ +public class RawStreamMuncher extends Thread { + private final InputStream in; + private final OutputStream out; + + public RawStreamMuncher(InputStream in, OutputStream out) { + this.in = in; + this.out = out; + } + + @Override + public void run() { + try { + StreamUtil.copy(in, out); + } catch (IOException e) { + Exceptions.ignore(e, "When the process exits, a harmless IOException will occur here"); + } finally { + FileUtil.close(in); + FileUtil.close(out); + } + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/io/StreamMuncher.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/io/StreamMuncher.java new file mode 100644 index 00000000000..d9c557690f0 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/io/StreamMuncher.java @@ -0,0 +1,49 @@ +package com.semmle.util.io; + +import com.semmle.util.exception.Exceptions; +import com.semmle.util.files.FileUtil; +import com.semmle.util.io.BufferedLineReader; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.PrintStream; + +/** + * A thread that forwards data from one stream to another. It waits for + * entire lines of input from one stream before writing data to the next + * stream, and it flushes as it goes. + */ +public class StreamMuncher extends Thread { + private final InputStream is; + private PrintStream output; + private BufferedLineReader reader; + + public StreamMuncher(InputStream is, OutputStream output) { + this.is = is; + if (output != null) + this.output = new PrintStream(output); + } + + @Override + public void run() { + InputStreamReader isr = null; + try { + isr = new InputStreamReader(is); + reader = new BufferedLineReader(isr); + String line; + while ((line = reader.readLineAndTerminator()) != null) { + if (output != null) { + output.print(line); + output.flush(); + } + } + } catch (IOException e) { + Exceptions.ignore(e, "When the process exits, a harmless IOException will occur here"); + } finally { + FileUtil.close(reader); + FileUtil.close(isr); + } + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/io/StreamUtil.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/io/StreamUtil.java new file mode 100644 index 00000000000..8071b2803fc --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/io/StreamUtil.java @@ -0,0 +1,201 @@ +package com.semmle.util.io; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Reader; +import java.io.Writer; +import java.nio.charset.StandardCharsets; + +import com.semmle.util.exception.CatastrophicError; + +/** + * Utility methods concerning {@link InputStream}s and {@link OutputStream}s. + */ +public class StreamUtil +{ + /** + * Copy all bytes that can be read from an {@link InputStream}, into an {@link OutputStream}. + * + * @param inputStream The InputStream from which to read, until an + * {@link InputStream#read(byte[])} operation returns indicating that the input stream + * has reached its end. + * @param outputStream The OutputStream to which all bytes read from {@code inputStream} should be + * written. + * @return The number of bytes copied. + * @throws IOException from {@link InputStream#read(byte[])} or + * {@link OutputStream#write(byte[], int, int)} + * @throws CatastrophicError if either of the streams is {@code null} + */ + public static long copy(InputStream inputStream, OutputStream outputStream) throws IOException + { + nullCheck(inputStream, outputStream); + + // Copy byte data + long total = 0; + byte[] bytes = new byte[1024]; + int read; + while ((read = inputStream.read(bytes)) > 0) { + outputStream.write(bytes, 0, read); + total += read; + } + return total; + } + + /** + * Copy all chars that can be read from a {@link Reader}, into a {@link Writer}. + * + * @param reader The Reader from which to read, until a {@link Reader#read(char[])} operation + * returns indicating that the reader has reached its end. + * @param writer The Writer to which all characters read from {@code reader} should be written. + * @return The number of bytes copied. + * @throws IOException from {@link Reader#read(char[])} or + * {@link Writer#write(char[], int, int)} + * @throws CatastrophicError if either of the streams is {@code null} + */ + public static long copy(Reader reader, Writer writer) throws IOException + { + nullCheck(reader, writer); + + // Copy byte data + long total = 0; + char[] chars = new char[1024]; + int read; + while ((read = reader.read(chars)) > 0) { + writer.write(chars, 0, read); + total += read; + } + return total; + } + + /** + * Copy at most {@code length} bytes from an {@link InputStream}, into an {@link OutputStream}. + *

+ * Note that this method will busy-wait during periods for which the {@code inputStream} cannot + * supply any data, but has not reached its end. + *

+ * + * @param inputStream The InputStream from which to read, until {@code length} bytes have + * been read or {@link InputStream#read(byte[], int, int)} operation returns + * indicating that the input stream has reached its end. + * @param outputStream The OutputStream to which all bytes read from {@code inputStream} should be + * written. + * @param length The maximum number of bytes to copy + * @return The number of bytes copied. + * @throws IOException from {@link InputStream#read(byte[], int, int)} or + * {@link OutputStream#write(byte[], int, int)} + * @throws CatastrophicError if either of the streams is {@code null} + */ + public static long limitedCopy(InputStream inputStream, OutputStream outputStream, long length) throws IOException + { + nullCheck(inputStream, outputStream); + + // Copy byte data + long total = 0; + byte[] bytes = new byte[1024]; + int read; + while ((read = inputStream.read(bytes, 0, (int) Math.min(bytes.length, length))) > 0) { + outputStream.write(bytes, 0, read); + length -= read; + total += read; + } + return total; + } + + private static void nullCheck(Object input, Object output) { + CatastrophicError.throwIfAnyNull(input, output); + } + + /** + * Skips over and discards n bytes of data from an input stream. If n is negative then no bytes are skipped. + * @param stream the InputStream + * @param n the number of bytes to be skipped. + * @return false if the end-of-file was reached before successfully skipping n bytes + */ + public static boolean skip(InputStream stream, long n) throws IOException { + if (n <= 0) + return true; + long toSkip = n - 1; + + while (toSkip > 0) { + long skipped = stream.skip(toSkip); + if (skipped == 0) { + if(stream.read() == -1) + return false; + else + skipped++; + } + toSkip -= skipped; + } + if(stream.read() == -1) + return false; + else + return true; + } + + /** + * Reads n bytes from the input stream and returns them. This method will block + * until all n bytes are available. If the end of the stream is reached before n bytes are + * read it returns just the read bytes. + * + * @param stream the InputStream + * @param n the number of bytes to read + * @return the read bytes + * @throws IOException if an IOException occurs when accessing the stream + * @throws IllegalArgumentException if n is negative + */ + public static byte[] readN(InputStream stream, int n) throws IOException { + if (n < 0) throw new IllegalArgumentException("n must be positive"); + + ByteArrayOutputStream bOut = new ByteArrayOutputStream(); + limitedCopy(stream, bOut, n); + return bOut.toByteArray(); + } + + /** + * Reads bytes from the input stream into the given buffer. This method will block + * until all bytes are available. If the end of the stream is reached before enough bytes are + * read it reads as much as it can. + * + * @param stream the InputStream + * @param buf the buffer to read into + * @param offset the offset to read into + * @param length the number of bytes to read + * @return the total number of read bytes + * @throws IOException if an IOException occurs when accessing the stream + * @throws IllegalArgumentException if n is negative + */ + public static int read(InputStream stream, byte[] buf, int offset, int length) throws IOException { + if (length < 0) throw new IllegalArgumentException("length must be positive"); + + // Copy byte data + int total = 0; + int read; + while ((read = stream.read(buf, offset, length)) > 0) { + length -= read; + total += read; + } + + return total; + } + + /** + * Convenience method for constructing a buffered reader with a UTF8 charset. + */ + public static BufferedReader newUTF8BufferedReader(InputStream inputStream) { + return new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + } + + /** + * Convenience method for constructing a buffered writer with a UTF8 charset. + */ + public static BufferedWriter newUTF8BufferedWriter(OutputStream outputStream) { + return new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8)); + } + +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/io/WholeIO.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/io/WholeIO.java new file mode 100644 index 00000000000..d45ea9c5daa --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/io/WholeIO.java @@ -0,0 +1,548 @@ +package com.semmle.util.io; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; +import java.io.Writer; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.regex.Pattern; + +import com.semmle.util.array.ArrayUtil; +import com.semmle.util.data.IntRef; +import com.semmle.util.exception.ResourceError; +import com.semmle.util.files.FileUtil; + +/** + * A class that allows bulk operations on entire files, + * reading or writing them as {@link String} values. + * + * This is intended to address the woeful inadequacy of + * the Java standard libraries in this area. + */ +public class WholeIO { + private IOException e; + + /** + * Regular expression {@link Pattern} + */ + private final static Pattern rpLineEndingCRLF = Pattern.compile("\r\n"); + + /** + * The default encoding to use for writing, and for reading if no + * encoding can be detected. + */ + private final String defaultEncoding; + + /** + * Construct a new {@link WholeIO} instance using ODASA's default + * charset ({@code "UTF-8"}) for all input and output (unless a + * different encoding is detected for a file being read). + */ + public WholeIO() { + this("UTF-8"); + } + + /** + * Construct a new {@link WholeIO} instance using the specified + * encoding for all input and output (unless a different encoding + * is detected for a file being read). + * + * @param encoding The encoding name, e.g. {@code "UTF-8"}. + */ + public WholeIO(String encoding) { + defaultEncoding = encoding; + } + + /** + * Open the given file for reading, get the entire content + * and return it as a {@link String}. Returns null + * on error, in which case you can check the getLastException() + * method for the exception that occurred. + * + * Warning: This method trims the content of the file, removing + * leading and trailing whitespace. Do not use it if you care about file + * locations being preserved; use 'read' instead. + * + * @param file The file to read + * @return The trimmed contents of the file, or null on error. + */ + public String readAndTrim(File file) { + e = null; + FileInputStream f = null; + try { + f = new FileInputStream(file); + String contents = readString(f); + return contents == null ? null : contents.trim(); + } catch (IOException e) { + this.e = e; + return null; + } finally { + FileUtil.close(f); + } + } + + /** + * Open the given filename for writing and dump the given + * {@link String} into it. Returns false + * on error, in which case you can check the getLastException() + * method for the exception that occurred. Tries to create any + * enclosing directories that do not exist. + * + * @param filename The name of the file to write to + * @param contents the string to write out + * @return the success state + */ + public boolean write(String filename, String contents) { + return write(new File(filename), contents); + } + + /** + * Open the given filename for writing and dump the given + * {@link String} into it. Returns false + * on error, in which case you can check the getLastException() + * method for the exception that occurred. Tries to create any + * enclosing directories that do not exist. + * + * @param file The file to write to + * @param contents the string to write out + * @return the success state + */ + public boolean write(File file, String contents) { + return write(file, contents, false); + } + + /** + * Open the given path for writing and dump the given + * {@link String} into it. Returns false + * on error, in which case you can check the getLastException() + * method for the exception that occurred. Tries to create any + * enclosing directories that do not exist. + * + * @param path The path to write to + * @param contents the string to write out + * @return the success state + */ + public boolean write(Path path, String contents) { + return write(path, contents, false); + } + + /** + * Open the given filename for writing and dump the given + * {@link String} into it. Throws {@link ResourceError} + * if we fail. + * + * @param file The file to write to + * @param contents the string to write out + */ + public void strictwrite(File file, String contents) { + strictwrite(file, contents, false); + } + + /** + * Open the given path for writing and dump the given + * {@link String} into it. Throws {@link ResourceError} + * if we fail. + * + * @param path The path to write to + * @param contents the string to write out + */ + public void strictwrite(Path path, String contents) { + strictwrite(path, contents, false); + } + + /** + * This is the same as {@link #write(File,String)}, + * except that this method allows appending to an existing file. + * + * @param file the file to write to + * @param contents the string to write out + * @param append whether or not to append to any existing file + * @return the success state + */ + public boolean write(File file, String contents, boolean append) { + if (file.getParentFile() != null) + file.getParentFile().mkdirs(); + + FileOutputStream fos = null; + try { + fos = new FileOutputStream(file, append); + Writer writer = new OutputStreamWriter(fos, Charset.forName(defaultEncoding)); + writer.append(contents); + writer.close(); + return true; + } catch (IOException e) { + this.e = e; + return false; + } finally { + FileUtil.close(fos); + } + } + + /** + * This is the same as {@link #write(Path,String)}, + * except that this method allows appending to an existing file. + * + * @param path the path to write to + * @param contents the string to write out + * @param append whether or not to append to any existing file + * @return the success state + */ + public boolean write(Path path, String contents, boolean append) { + try { + if (path.getParent() != null) + Files.createDirectories(path.getParent()); + + try (Writer writer = Files.newBufferedWriter(path, Charset.forName(defaultEncoding), + StandardOpenOption.CREATE, StandardOpenOption.WRITE, + append ? StandardOpenOption.APPEND : StandardOpenOption.TRUNCATE_EXISTING)) { + writer.append(contents); + } + } catch (IOException e) { + this.e = e; + return false; + } + return true; + } + + /** + * This is the same as {@link #strictwrite(File,String)}, + * except that this method allows appending to an existing file. + */ + public void strictwrite(File file, String contents, boolean append) { + if (!write(file, contents, append)) + throw new ResourceError("Failed to write file " + file, getLastException()); + } + + /** + * This is the same as {@link #strictwrite(Path,String)}, + * except that this method allows appending to an existing file. + */ + public void strictwrite(Path path, String contents, boolean append) { + if (!write(path, contents, append)) + throw new ResourceError("Failed to write path " + path, getLastException()); + } + + /** + * Get the exception that occurred during the last call to + * read(), if any. If the last read() call completed normally, + * this returns null. + * @return The last caught exception, or null if N/A. + */ + public IOException getLastException() { + return e; + } + + public String read(File file) { + InputStream is = null; + try { + is = new FileInputStream(file); + return readString(is); + } + catch (IOException e) { + this.e = e; + return null; + } + finally { + FileUtil.close(is); + } + } + + public String read(Path path) { + InputStream is = null; + try { + is = Files.newInputStream(path); + return readString(is); + } + catch (IOException e) { + this.e = e; + return null; + } + finally { + FileUtil.close(is); + } + } + + /** + * Read the contents of the given {@link File} as text (line endings are normalised to "\n" in the output). + * + * @param file The file to read. + * @return The text contents of the file, if possible, or null if the file cannot be read. + */ + public String readText(File file) { + String result = read(file); + return result != null ? result.replaceAll("\r\n", "\n") : null; + } + + /** + * Read the contents of the given {@link Path} as text (line endings are normalised to "\n" in the output). + * + * @param path The path to read. + * @return The text contents of the path, if possible, or null if the file cannot be read. + */ + public String readText(Path path) { + String result = read(path); + return result != null ? result.replaceAll("\r\n", "\n") : null; + } + + + /** + * Read the contents of the given {@link File}, throwing a {@link ResourceError} + * if we fail. + */ + public String strictread(File f) { + String content = read(f); + if (content == null) + throw new ResourceError("Failed to read file " + f, getLastException()); + return content; + } + + /** + * Read the contents of the given {@link Path}, throwing a {@link ResourceError} + * if we fail. + */ + public String strictread(Path f) { + String content = read(f); + if (content == null) + throw new ResourceError("Failed to read path " + f, getLastException()); + return content; + } + + /** + * Read the contents of the given {@link File} as text (line endings are normalised to "\n" in the output). + * + * @param file The file to read. + * @return The text contents of the file, if possible. + * @throws ResourceError If the file cannot be read. + */ + public String strictreadText(File file) { + return rpLineEndingCRLF.matcher(strictread(file)).replaceAll("\n"); + } + + /** + * Read the contents of the given {@link Path} as text (line endings are normalised to "\n" in the output). + * + * @param path The path to read. + * @return The text contents of the path, if possible. + * @throws ResourceError If the path cannot be read. + */ + public String strictreadText(Path path) { + return rpLineEndingCRLF.matcher(strictread(path)).replaceAll("\n"); + } + + /** + * Get the entire content of an {@link InputStream} + * and interpret it as a {@link String} trying to detect its character set. + * Returns null on error, in which case you can check + * the getLastException() method for the exception that occurred. + * + * @param stream the stream to read from + * @return The contents of the file, or null on error. + */ + public String readString(InputStream stream) { + IntRef length = new IntRef(0); + byte[] bytes = readBinary(stream, length); + + if (bytes == null) return null; + + try { + IntRef start = new IntRef(0); + String charset = determineCharset(bytes, length.get(), start); + return new String(bytes, start.get(), length.get() - start.get(), charset); + } catch (UnsupportedEncodingException e) { + this.e = e; + return null; + } + } + + /** + * Get the entire content of an {@link InputStream} + * and interpret it as a {@link String} trying to detect its character set. + * Throws a {@link ResourceError} on error. + * + * @param stream the stream to read from + * @return the contents of the input stream + */ + public String strictReadString(InputStream stream) { + String content = readString(stream); + if (content == null) + throw new ResourceError("Could not read from stream", getLastException()); + return content; + } + + /** + * Get the entire content of an {@link InputStream}, interpreting it + * as a sequence of bytes. This removes restrictions regarding invalid + * code points that would potentially prevent reading a file's contents + * as a String. + * + * This method returns null on error, in which case you can + * check {@link #getLastException()} for the exception that occurred. + * + * @param stream the stream to read from + * @return The binary contents of the file, or null on error. + */ + public byte[] readBinary(InputStream stream) { + IntRef length = new IntRef(0); + byte[] bytes = readBinary(stream, length); + return bytes == null ? null : Arrays.copyOf(bytes, length.get()); + } + + /** + * Get the entire content of an {@link InputStream}, interpreting it + * as a sequence of bytes. This removes restrictions regarding invalid + * code points that would potentially prevent reading a file's contents + * as a String. + * + * @param stream the stream to read from + * @return The binary contents of the file -- always non-null. + * @throws ResourceError if an exception occurs during IO. + */ + public byte[] strictReadBinary(InputStream stream) { + byte[] result = readBinary(stream); + if (result == null) + throw new ResourceError("Couldn't read from stream", e); + return result; + } + + /** + * Get the entire binary contents of a {@link File} as a sequence of bytes. + * + * @param file the file to read + * @return the file's contents as a byte[] -- always non-null. + * @throws ResourceError if an exception occurs during IO. + */ + public byte[] strictReadBinary(File file) { + FileInputStream stream = null; + try { + stream = new FileInputStream(file); + byte[] result = readBinary(stream); + if (result == null) + throw new ResourceError("Couldn't read from file " + file + ".", e); + return result; + } catch (FileNotFoundException e) { + throw new ResourceError("Couldn't read from file " + file + ".", e); + } finally { + FileUtil.close(stream); + } + } + + /** + * Get the entire binary contents of a {@link Path} as a sequence of bytes. + * + * @param path the path to read + * @return the file's contents as a byte[] -- always non-null. + * @throws ResourceError if an exception occurs during IO. + */ + public byte[] strictReadBinary(Path path) { + InputStream stream = null; + try { + stream = Files.newInputStream(path); + byte[] result = readBinary(stream); + if (result == null) + throw new ResourceError("Couldn't read from path " + path + ".", e); + return result; + } catch (IOException e) { + throw new ResourceError("Couldn't read from path " + path + ".", e); + } finally { + FileUtil.close(stream); + } + } + + /** + * Get the entire binary contents of a {@link Path} as a sequence of bytes. + * + * @param path the path to read + * @return the file's contents as a byte[] -- always non-null. + */ + public byte[] readBinary(Path path) throws IOException { + InputStream stream = null; + try { + stream = Files.newInputStream(path); + byte[] result = readBinary(stream); + if (result == null) + throw new ResourceError("Couldn't read from path " + path + ".", e); + return result; + } finally { + FileUtil.close(stream); + } + } + + private byte[] readBinary(InputStream stream, IntRef offsetHolder) { + try { + byte[] bytes = new byte[16384]; + int offset = 0; + int readThisTime; + do { + readThisTime = stream.read(bytes, offset, bytes.length - offset); + if (readThisTime > 0) { + offset += readThisTime; + if (offset == bytes.length) + bytes = safeArrayDouble(bytes); + } + } while (readThisTime > 0); + offsetHolder.set(offset); + return bytes; + } catch (IOException e) { + this.e = e; + return null; + } + } + + /** + * Safely attempt to double the length of an array. + * @param array The array which want to be doubled + * @return a new array that is longer than array + */ + private byte[] safeArrayDouble(byte[] array) { + if (array.length >= ArrayUtil.MAX_ARRAY_LENGTH) { + throw new ResourceError("Cannot stream into array as it exceed the maximum array size"); + } + // Compute desired capacity + long newCapacity = array.length * 2L; + // Ensure it is at least as large as minCapacity + if (newCapacity < 16) + newCapacity = 16; + // Ensure it is at most MAX_ARRAY_LENGTH + if (newCapacity > ArrayUtil.MAX_ARRAY_LENGTH) { + newCapacity = ArrayUtil.MAX_ARRAY_LENGTH; + } + return Arrays.copyOf(array, (int)newCapacity); + } + + /** + * Try to determine the encoding of a byte[] using a byte-order mark (if present) + * Defaults to UTF-8 if none found. + */ + private String determineCharset(byte[] bom, int length, IntRef start) { + start.set(0); + String ret = defaultEncoding; + if(length < 2) + return ret; + if (length >= 3 && byteToInt(bom[0]) == 0xEF && byteToInt(bom[1]) == 0xBB && byteToInt(bom[2]) == 0xBF) { + ret = "UTF-8"; + start.set(3); + } else if (byteToInt(bom[0]) == 0xFE && byteToInt(bom[1]) == 0xFF) { + ret = "UTF-16BE"; + start.set(2); + } else if (byteToInt(bom[0]) == 0xFF && byteToInt(bom[1]) == 0xFE) { + ret = "UTF-16LE"; + start.set(2); + } + return ret; + } + + private static int byteToInt(byte b) { + return b & 0xFF; + } + +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/io/csv/CSVParser.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/io/csv/CSVParser.java new file mode 100644 index 00000000000..749b1bee5df --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/io/csv/CSVParser.java @@ -0,0 +1,207 @@ +package com.semmle.util.io.csv; + +/** + Copyright 2005 Bytecode Pty Ltd. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * A very simple CSV parser released under a commercial-friendly license. + * This just implements splitting a single line into fields. + * + * @author Glen Smith + * @author Rainer Pruy + * + */ +public class CSVParser { + + private final char separator; + + private final char quotechar; + + private final char escape; + + private final boolean strictQuotes; + + private StringBuilder buf = new StringBuilder(INITIAL_READ_SIZE); + + /** The default separator to use if none is supplied to the constructor. */ + public static final char DEFAULT_SEPARATOR = ','; + + private static final int INITIAL_READ_SIZE = 128; + + /** + * The default quote character to use if none is supplied to the + * constructor. + */ + public static final char DEFAULT_QUOTE_CHARACTER = '"'; + + + /** + * The default escape character to use if none is supplied to the + * constructor. + */ + public static final char DEFAULT_ESCAPE_CHARACTER = '"'; + + /** + * The default strict quote behavior to use if none is supplied to the + * constructor + */ + public static final boolean DEFAULT_STRICT_QUOTES = false; + + /** + * Constructs CSVReader with supplied separator and quote char. + * Allows setting the "strict quotes" flag + * @param separator + * the delimiter to use for separating entries + * @param quotechar + * the character to use for quoted elements + * @param escape + * the character to use for escaping a separator or quote + * @param strictQuotes + * if true, characters outside the quotes are ignored + */ + CSVParser(char separator, char quotechar, char escape, boolean strictQuotes) { + this.separator = separator; + this.quotechar = quotechar; + this.escape = escape; + this.strictQuotes = strictQuotes; + } + + /** + * + * @return true if something was left over from last call(s) + */ + public boolean isPending() { + return buf.length() != 0; + } + + public String[] parseLineMulti(String nextLine) throws IOException { + return parseLine(nextLine, true); + } + + public String[] parseLine(String nextLine) throws IOException { + return parseLine(nextLine, false); + } + /** + * Parses an incoming String and returns an array of elements. + * + * @param nextLine + * the string to parse + * @return the comma-tokenized list of elements, or null if nextLine is null + * @throws IOException if bad things happen during the read + */ + private String[] parseLine(String nextLine, boolean multi) throws IOException { + + if (!multi && isPending()) { + clear(); + } + + if (nextLine == null) { + if (isPending()) { + String s = buf.toString(); + clear(); + return new String[] {s}; + } else { + return null; + } + } + + ListtokensOnThisLine = new ArrayList(); + boolean inQuotes = isPending(); + for (int i = 0; i < nextLine.length(); i++) { + + char c = nextLine.charAt(i); + if (c == this.escape && isNextCharacterEscapable(nextLine, inQuotes, i)) { + buf.append(nextLine.charAt(i+1)); + i++; + } else if (c == quotechar) { + if( isNextCharacterEscapedQuote(nextLine, inQuotes, i) ){ + buf.append(nextLine.charAt(i+1)); + i++; + }else{ + inQuotes = !inQuotes; + // the tricky case of an embedded quote in the middle: a,bc"d"ef,g + if (!strictQuotes) { + if(i>2 //not on the beginning of the line + && nextLine.charAt(i-1) != this.separator //not at the beginning of an escape sequence + && nextLine.length()>(i+1) && + nextLine.charAt(i+1) != this.separator //not at the end of an escape sequence + ){ + buf.append(c); + } + } + } + } else if (c == separator && !inQuotes) { + tokensOnThisLine.add(buf.toString()); + clear(); // start work on next token + } else { + if (!strictQuotes || inQuotes) + buf.append(c); + } + } + // line is done - check status + if (inQuotes) { + if (multi) { + // continuing a quoted section, re-append newline + buf.append('\n'); + // this partial content is not to be added to field list yet + } else { + throw new IOException("Un-terminated quoted field at end of CSV line"); + } + } else { + tokensOnThisLine.add(buf.toString()); + clear(); + } + return tokensOnThisLine.toArray(new String[tokensOnThisLine.size()]); + + } + + /** + * precondition: the current character is a quote or an escape + * @param nextLine the current line + * @param inQuotes true if the current context is quoted + * @param i current index in line + * @return true if the following character is a quote + */ + private boolean isNextCharacterEscapedQuote(String nextLine, boolean inQuotes, int i) { + return inQuotes // we are in quotes, therefore there can be escaped quotes in here. + && nextLine.length() > (i+1) // there is indeed another character to check. + && nextLine.charAt(i+1) == quotechar; + } + + /** + * precondition: the current character is an escape + * @param nextLine the current line + * @param inQuotes true if the current context is quoted + * @param i current index in line + * @return true if the following character is a quote + */ + protected boolean isNextCharacterEscapable(String nextLine, boolean inQuotes, int i) { + return inQuotes // we are in quotes, therefore there can be escaped quotes in here. + && nextLine.length() > (i+1) // there is indeed another character to check. + && ( nextLine.charAt(i+1) == quotechar || nextLine.charAt(i+1) == this.escape); + } + + /** + * Reset the buffer used for storing the current field's value + */ + private void clear() { + buf.setLength(0); + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/io/csv/CSVReader.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/io/csv/CSVReader.java new file mode 100644 index 00000000000..cbecb552c56 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/io/csv/CSVReader.java @@ -0,0 +1,192 @@ +package com.semmle.util.io.csv; + +/** + Copyright 2005 Bytecode Pty Ltd. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.List; + +/** + * A very simple CSV reader released under a commercial-friendly license. + * + * @author Glen Smith + * + */ +public class CSVReader implements Closeable { + + private final BufferedReader br; + + private boolean hasNext = true; + + private final CSVParser parser; + + private final int skipLines; + + private boolean linesSkipped; + + /** The line number of the last physical line read (one-based). */ + private int curline = 0; + + /** The physical line number at which the last logical line read started (one-based). */ + private int startLine = 0; + + /** + * The default line to start reading. + */ + private static final int DEFAULT_SKIP_LINES = 0; + + /** + * Constructs CSVReader using a comma for the separator. + * + * @param reader + * the reader to an underlying CSV source. + */ + public CSVReader(Reader reader) { + this(reader, + CSVParser.DEFAULT_SEPARATOR, CSVParser.DEFAULT_QUOTE_CHARACTER, + CSVParser.DEFAULT_ESCAPE_CHARACTER, DEFAULT_SKIP_LINES, + CSVParser.DEFAULT_STRICT_QUOTES); + } + + /** + * Constructs CSVReader with supplied separator and quote char. + * + * @param reader + * the reader to an underlying CSV source. + * @param separator + * the delimiter to use for separating entries + * @param quotechar + * the character to use for quoted elements + * @param escape + * the character to use for escaping a separator or quote + * @param line + * the line number to skip for start reading + * @param strictQuotes + * sets if characters outside the quotes are ignored + */ + private CSVReader(Reader reader, char separator, char quotechar, char escape, int line, boolean strictQuotes) { + this.br = new BufferedReader(reader); + this.parser = new CSVParser(separator, quotechar, escape, strictQuotes); + this.skipLines = line; + } + + + /** + * Reads the entire file into a List with each element being a String[] of + * tokens. + * + * @return a List of String[], with each String[] representing a line of the + * file. + * + * @throws IOException + * if bad things happen during the read + */ + public List readAll() throws IOException { + + List allElements = new ArrayList(); + while (hasNext) { + String[] nextLineAsTokens = readNext(); + if (nextLineAsTokens != null) + allElements.add(nextLineAsTokens); + } + return allElements; + + } + + /** + * Reads the next line from the buffer and converts to a string array. + * + * @return a string array with each comma-separated element as a separate + * entry, or null if there are no more lines to read. + * + * @throws IOException + * if bad things happen during the read + */ + public String[] readNext() throws IOException { + boolean first = true; + String[] result = null; + do { + String nextLine = getNextLine(); + + if (first) { + startLine = curline; + first = false; + } + + if (!hasNext) { + return result; // should throw if still pending? + } + String[] r = parser.parseLineMulti(nextLine); + if (r.length > 0) { + if (result == null) { + result = r; + } else { + String[] t = new String[result.length+r.length]; + System.arraycopy(result, 0, t, 0, result.length); + System.arraycopy(r, 0, t, result.length, r.length); + result = t; + } + } + } while (parser.isPending()); + return result; + } + + /** + * Reads the next line from the file. + * + * @return the next line from the file without trailing newline + * @throws IOException + * if bad things happen during the read + */ + private String getNextLine() throws IOException { + if (!this.linesSkipped) { + for (int i = 0; i < skipLines; i++) { + br.readLine(); + ++curline; + } + this.linesSkipped = true; + } + String nextLine = br.readLine(); + if (nextLine == null) { + hasNext = false; + } else { + ++curline; + } + return hasNext ? nextLine : null; + } + + /** + * Closes the underlying reader. + * + * @throws IOException if the close fails + */ + @Override + public void close() throws IOException{ + br.close(); + } + + /** + * Return the physical line number (one-based) at which the last logical line read started, + * or zero if no line has been read yet. + */ + public int getStartLine() { + return startLine; + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/io/csv/CSVWriter.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/io/csv/CSVWriter.java new file mode 100644 index 00000000000..e1fee41154a --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/io/csv/CSVWriter.java @@ -0,0 +1,226 @@ +package com.semmle.util.io.csv; + +/** + Copyright 2005 Bytecode Pty Ltd. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +import java.io.Closeable; +import java.io.IOException; +import java.io.Writer; +import java.util.List; + +/** + * A very simple CSV writer released under a commercial-friendly license. + * + * @author Glen Smith + * + */ +public class CSVWriter implements Closeable { + + private static final int INITIAL_STRING_SIZE = 128; + + private Writer rawWriter; + + private char separator; + + private char quotechar; + + private char escapechar; + + private String lineEnd; + + /** The quote constant to use when you wish to suppress all quoting. */ + public static final char NO_QUOTE_CHARACTER = '\u0000'; + + /** The escape constant to use when you wish to suppress all escaping. */ + private static final char NO_ESCAPE_CHARACTER = '\u0000'; + + /** Default line terminator uses platform encoding. */ + private static final String DEFAULT_LINE_END = "\n"; + + private boolean[] eagerQuotingFlags = {}; + + /** + * Constructs CSVWriter using a comma for the separator. + * + * @param writer + * the writer to an underlying CSV source. + */ + public CSVWriter(Writer writer) { + this(writer, + CSVParser.DEFAULT_SEPARATOR, + CSVParser.DEFAULT_QUOTE_CHARACTER, + CSVParser.DEFAULT_ESCAPE_CHARACTER + ); + } + + /** + * Constructs CSVWriter with supplied separator and quote char. + * + * @param writer + * the writer to an underlying CSV source. + * @param separator + * the delimiter to use for separating entries + * @param quotechar + * the character to use for quoted elements + * @param escapechar + * the character to use for escaping quotechars or escapechars + */ + public CSVWriter(Writer writer, char separator, char quotechar, char escapechar) { + this(writer, separator, quotechar, escapechar, DEFAULT_LINE_END); + } + + /** + * Constructs CSVWriter with supplied separator, quote char, escape char and line ending. + * + * @param writer + * the writer to an underlying CSV source. + * @param separator + * the delimiter to use for separating entries + * @param quotechar + * the character to use for quoted elements + * @param escapechar + * the character to use for escaping quotechars or escapechars + * @param lineEnd + * the line feed terminator to use + */ + private CSVWriter(Writer writer, char separator, char quotechar, char escapechar, String lineEnd) { + this.rawWriter = writer; + this.separator = separator; + this.quotechar = quotechar; + this.escapechar = escapechar; + this.lineEnd = lineEnd; + } + + /** + * Call with an array of booleans, corresponding to columns, where columns that have + * false will not be quoted unless they contain special characters. + *

+ * If there are more columns to print than have been configured here, any additional + * columns will be treated as if true was passed. + */ + public void setEagerQuotingColumns(boolean... flags) { + eagerQuotingFlags = flags; + } + + /** + * Writes the entire list to a CSV file. The list is assumed to be a + * String[] + * + * @param allLines + * a List of String[], with each String[] representing a line of + * the file. + */ + public void writeAll(List allLines) throws IOException { + for (String[] line : allLines) { + writeNext(line); + } + } + + /** + * Writes the next line to the file. + * + * @param nextLine + * a string array with each comma-separated element as a separate + * entry. + */ + public void writeNext(String... nextLine) throws IOException { + + if (nextLine == null) + return; + + StringBuilder sb = new StringBuilder(INITIAL_STRING_SIZE); + for (int i = 0; i < nextLine.length; i++) { + + if (i != 0) { + sb.append(separator); + } + + String nextElement = nextLine[i]; + if (nextElement == null) + continue; + boolean hasSpecials = stringContainsSpecialCharacters(nextElement); + + if (hasSpecials || i >= eagerQuotingFlags.length || eagerQuotingFlags[i] + || stringContainsSomewhatSpecialCharacter(nextElement)) { + if (quotechar != NO_QUOTE_CHARACTER) + sb.append(quotechar); + sb.append(hasSpecials ? processLine(nextElement) : nextElement); + if (quotechar != NO_QUOTE_CHARACTER) + sb.append(quotechar); + } else { + sb.append(nextElement); + } + } + + sb.append(lineEnd); + rawWriter.write(sb.toString()); + + } + + /** + * Return true if there are characters that need to be escaped in addition to + * being quoted. + */ + private boolean stringContainsSpecialCharacters(String line) { + return line.indexOf(quotechar) != -1 || line.indexOf(escapechar) != -1; + } + + /** + * Return true if there are characters that should not appear in a completely + * unquoted field. + */ + private boolean stringContainsSomewhatSpecialCharacter(String s) { + return s.indexOf('"') != -1 || s.indexOf('\'') != -1 || s.indexOf('\t') != -1 || s.indexOf(separator) != -1; + } + + protected StringBuilder processLine(String nextElement) + { + StringBuilder sb = new StringBuilder(INITIAL_STRING_SIZE); + for (int j = 0; j < nextElement.length(); j++) { + char nextChar = nextElement.charAt(j); + if (escapechar != NO_ESCAPE_CHARACTER && nextChar == quotechar) { + sb.append(escapechar).append(nextChar); + } else if (escapechar != NO_ESCAPE_CHARACTER && nextChar == escapechar) { + sb.append(escapechar).append(nextChar); + } else { + sb.append(nextChar); + } + } + + return sb; + } + + /** + * Flush underlying stream to writer. + * + * @throws IOException if bad things happen + */ + public void flush() throws IOException { + rawWriter.flush(); + } + + /** + * Close the underlying stream writer flushing any buffered content. + * + * @throws IOException if bad things happen + * + */ + @Override + public void close() throws IOException { + rawWriter.close(); + } + +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/logging/Streams.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/logging/Streams.java new file mode 100644 index 00000000000..9fee4edda49 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/logging/Streams.java @@ -0,0 +1,101 @@ +package com.semmle.util.logging; + +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.Stack; + +import com.semmle.util.exception.CatastrophicError; + +/** + * A class to wrap around accesses to {@link System#out} and + * {@link System#err}, so that tools can behave consistently when + * run in-process or out-of-process. + */ +public class Streams { + private static final InheritableThreadLocal out = + new InheritableThreadLocal() { + @Override + protected PrintStream initialValue() { + return System.out; + } + }; + + private static final InheritableThreadLocal err = + new InheritableThreadLocal() { + @Override + protected PrintStream initialValue() { + return System.err; + } + }; + + private static final InheritableThreadLocal in = + new InheritableThreadLocal() { + @Override + protected InputStream initialValue() { + return System.in; + } + }; + + private static class SavedContext { + public PrintStream out, err; + public InputStream in; + } + + private static final ThreadLocal> contexts = + new ThreadLocal>() { + @Override + protected Stack initialValue() { + return new Stack(); + } + }; + + public static PrintStream out() { + return out.get(); + } + + public static PrintStream err() { + return err.get(); + } + + public static InputStream in() { + return in.get(); + } + + public static void pushContext(OutputStream stdout, OutputStream stderr, InputStream stdin) { + SavedContext context = new SavedContext(); + context.out = out.get(); + context.err = err.get(); + context.in = in.get(); + // When we run in-process, we don't benefit from + // a clean slate like we do when starting a new + // process. We need to reset anything that we care + // about manually. + // In particular, the parent VM may well have set + // showAllLogs=True, and we don't want the extra + // noise when executing the child, so we set a + // fresh log state for the duration of the child. + + contexts.get().push(context); + out.set(asPrintStream(stdout)); + err.set(asPrintStream(stderr)); + in.set(stdin); + } + + private static PrintStream asPrintStream(OutputStream stdout) { + return stdout instanceof PrintStream ? + (PrintStream)stdout : new PrintStream(stdout); + } + + public static void popContext() { + Stack context = contexts.get(); + out.get().flush(); + err.get().flush(); + if (context.isEmpty()) + throw new CatastrophicError("Popping logging context without preceding push."); + SavedContext old = context.pop(); + out.set(old.out); + err.set(old.err); + in.set(old.in); + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/process/AbstractProcessBuilder.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/process/AbstractProcessBuilder.java new file mode 100644 index 00000000000..98b7794f452 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/process/AbstractProcessBuilder.java @@ -0,0 +1,398 @@ +package com.semmle.util.process; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Timer; +import java.util.TimerTask; + +import com.github.codeql.Logger; +import com.github.codeql.Severity; + +import com.semmle.util.data.StringUtil; +import com.semmle.util.exception.CatastrophicError; +import com.semmle.util.exception.Exceptions; +import com.semmle.util.exception.InterruptedError; +import com.semmle.util.exception.ResourceError; +import com.semmle.util.files.FileUtil; +import com.semmle.util.io.RawStreamMuncher; + +/** + * A builder for an external process. This class wraps {@link ProcessBuilder}, + * adding support for spawning threads to manage the input and output streams of + * the created process. + */ +public abstract class AbstractProcessBuilder { + public static Logger logger = null; + + // timeout for the muncher threads in seconds + protected static final long MUNCH_TIMEOUT = 20; + private final ProcessBuilder builder; + private boolean logFailure = true; + private InputStream in; + private LeakPrevention leakPrevention; + + private volatile boolean interrupted = false; + private volatile Thread threadToInterrupt = null; + + private volatile boolean hitTimeout = false; + + private final Map canonicalEnvVarNames = new LinkedHashMap<>(); + + private RawStreamMuncher inMuncher; + + + public AbstractProcessBuilder (List args, File cwd, Map env) + { + // Sanity checks + CatastrophicError.throwIfNull(args); + for (int i = 0; i < args.size(); ++i) + CatastrophicError.throwIfNull(args.get(i)); + + leakPrevention = LeakPrevention.NONE; + builder = new ProcessBuilder(new ArrayList<>(args)); + if (cwd != null) { + builder.directory(cwd); + } + // Make sure that values that have been explicitly removed from Env.systemEnv() + // -- such as the variables representing command-line arguments -- + // are not taken over by the new ProcessBuilder. + Map keepThese = Env.systemEnv().getenv(); + for (Iterator it = builder.environment().keySet().iterator(); it.hasNext();) { + String name = it.next(); + if (!keepThese.containsKey(name)) + it.remove(); + } + if (env != null) { + addEnvironment(env); + } + + } + + public void setLeakPrevention(LeakPrevention leakPrevention) { + CatastrophicError.throwIfNull(leakPrevention); + this.leakPrevention = leakPrevention; + } + + /** + * See {@link ProcessBuilder#redirectErrorStream(boolean)}. + */ + public void setRedirectErrorStream(boolean redirectErrorStream) { + this.builder.redirectErrorStream(redirectErrorStream); + } + + public final boolean hasEnvVar(String name) { + return builder.environment().containsKey(getCanonicalVarName(name)); + } + + /** + * Add the specified key/value pair to the environment of the builder, + * overriding any previous environment entry of that name. This method + * provides additional logic to handle systems where environment + * variable names are case-insensitive, ensuring the last-added value + * for a name ends up in the final environment regardless of case. + * @param name The name of the environment variable. Whether case matters + * is OS-dependent. + * @param value The value for the environment variable. + */ + public final void addEnvVar(String name, String value) { + builder.environment().put(getCanonicalVarName(name), value); + } + + /** + * Prepend a specified set of arguments to this process builder's command line. + * This only makes sense before the builder is started. + */ + public void prependArgs(List args) { + builder.command().addAll(0, args); + } + + /** + * Compute a canonical environment variable name relative to this process + * builder. + * + * The need for this method arises on platforms where the environment is + * case-insensitive -- any inspection of it in such a situation needs to + * canonicalise the variable name to have well-defined behaviour. This is + * builder-specific, because it depends on its existing environment. For + * example, if it already defines a variable called Path, and the + * environment is case-insensitive, then setting a variable called + * PATH should overwrite this, and checking whether a variable + * called PATH is already defined should return true. + */ + public String getCanonicalVarName(String name) { + if (!Env.getOS().isEnvironmentCaseSensitive()) { + // We need to canonicalise the variable name to work around Java API limitations. + if (canonicalEnvVarNames.isEmpty()) + for (String var : builder.environment().keySet()) + canonicalEnvVarNames.put(StringUtil.lc(var), var); + String canonical = canonicalEnvVarNames.get(StringUtil.lc(name)); + if (canonical == null) + canonicalEnvVarNames.put(StringUtil.lc(name), name); + else + name = canonical; + } + return name; + } + + /** + * Get a snapshot of this builder's environment, using canonical variable names + * (as per {@link #getCanonicalVarName(String)}) as keys. Modifications to this + * map do not propagate back to the builder; use + * {@link #addEnvVar(String, String)} or {@link #addEnvironment(Map)} to extend + * its environment. + */ + public Map getCanonicalCurrentEnv() { + Map result = new LinkedHashMap<>(); + for (Entry e : builder.environment().entrySet()) + result.put(getCanonicalVarName(e.getKey()), e.getValue()); + return result; + } + + /** + * Specify an input stream of data that will be piped to the process's + * standard input. + * + * CAUTION: if this stream is the current process' standard in and no + * input is ever received, then we will leak an uninterruptible thread + * waiting for some input. This will terminate only when the standard in + * is closed, i.e. when the current process terminates. + */ + public final void setIn(InputStream in) { + this.in = in; + } + + /** + * Set the environment of this builder to the given map. Any + * existing environment entries (either from the current process + * environment or from previous calls to {@link #addEnvVar(String, String)}, + * {@link #addEnvironment(Map)} or {@link #setEnvironment(Map)}) + * are discarded. + * @param env The environment to use. + */ + public final void setEnvironment(Map env) { + builder.environment().clear(); + canonicalEnvVarNames.clear(); + addEnvironment(env); + } + + /** + * Add the specified set of environment variables to the environment for + * the builder. This leaves existing variable definitions in place, but + * can override them. + * @param env The environment to merge into the current environment. + */ + public final void addEnvironment(Map env) { + for (Entry entry : env.entrySet()) + addEnvVar(entry.getKey(), entry.getValue()); + } + + public final int execute() { + return execute(0); + } + + /** + * Set the flag indicating that a non-zero exit code may be expected. This + * will suppress the log of failed commands. + */ + public final void expectFailure() { + logFailure = false; + } + + public final int execute(long timeout) { + Process process = null; + boolean processStopped = true; + Timer timer = null; + try { + synchronized (this) { + // Handle the case where we called kill() too early to use + // Thread.interrupt() + if (interrupted) + throw new InterruptedException(); + threadToInterrupt = Thread.currentThread(); + } + + processStopped = false; + String directory; + if (builder.directory() == null) { + directory = "current directory ('" + System.getProperty("user.dir") + "')"; + } else { + directory = "'" + builder.directory().toString() + "'"; + } + logger.debug("Running command: '" + toString() + "' in " + directory); + process = builder.start(); + setupInputHandling(process.getOutputStream()); + setupOutputHandling(process.getInputStream(), + process.getErrorStream()); + if (timeout != 0) { + // create the timer's thread as a "daemon" thread, so it does not + // prevent the jvm from terminating + timer = new Timer(true); + final Thread current = Thread.currentThread(); + timer.schedule(new TimerTask() { + @Override + public void run() { + hitTimeout = true; + current.interrupt(); + } + }, timeout); + } + + int result = process.waitFor(); + processStopped = true; + if (result != 0 && logFailure) + logger.error("Spawned process exited abnormally (code " + result + + "; tried to run: " + getBuilderCommand() + ")"); + return result; + } catch (IOException e) { + throw new ResourceError( + "IOException while executing process with args: " + + getBuilderCommand(), e); + } catch (InterruptedException e) { + throw new InterruptedError( + "InterruptedException while executing process with args: " + + getBuilderCommand(), e); + } finally { + // cancel the timer + if (timer != null) { + timer.cancel(); + } + // clear the interrupted flag of the current thread + // in case it was set earlier (ie by the Timer or a call to kill()) + synchronized (this) { + threadToInterrupt = null; + Thread.interrupted(); + } + // get rid of the process, in case it is still running. + if (process != null && !processStopped) { + killProcess(process); + } + try { + cleanupInputHandling(); + cleanupOutputHandling(); + } finally { + + if (process != null) { + FileUtil.close(process.getErrorStream()); + FileUtil.close(process.getInputStream()); + FileUtil.close(process.getOutputStream()); + } + } + } + } + + /** + * Provides the implementation of actually stopping the child + * process. Provided as an extension point so that this can + * be customised for later Java versions or for other reasons. + */ + protected void killProcess(Process process) { + process.destroy(); + } + + /** + * Setup handling of the process input stream (stdin). + * + * @param outputStream OutputStream connected to the process's standard input. + */ + protected void setupInputHandling(OutputStream outputStream) { + if (in == null) { + FileUtil.close(outputStream); + return; + } + inMuncher = new RawStreamMuncher(in, outputStream); + inMuncher.start(); + } + + /** + * Setup handling of the process' output streams (stdout and stderr). + * + * @param stdout + * InputStream connected to the process' standard output stream. + * @param stderr + * InputStream connected to the process' standard error stream. + */ + protected abstract void setupOutputHandling(InputStream stdout, InputStream stderr); + + /** + * Cleanup resources related to output handling. The method is always called, either after the process + * has exited normally, or after an abnormal termination due to an exception. As a result cleanupOutputHandling() + * might be called, without a previous call to setupOutputHandling. The implementation of this method should + * handle this case. + */ + protected abstract void cleanupOutputHandling(); + + private void cleanupInputHandling() { + if (inMuncher != null && inMuncher.isAlive()) { + // There's no real need to wait for the muncher to terminate -- on the contrary, + // if it's still alive it will typically be waiting for a closing action that + // will only happen after execute() returns anyway. + // The best we can do is try to interrupt it. + inMuncher.interrupt(); + } + } + + protected void waitForMuncher(String which, Thread muncher, long timeout) { + // wait for termination of the muncher until a deadline is reached + try { + muncher.join(timeout); + + } catch (InterruptedException e) { + Exceptions.ignore(e,"Further interruption attempts are ineffective --" + + " we're already waiting for termination."); + } + // if muncher is still alive, report an error + if(muncher.isAlive()){ + muncher.interrupt(); + logger.error(String.format("Standard %s stream hasn't closed %s seconds after termination of subprocess '%s'.", which, MUNCH_TIMEOUT, this)); + } + } + + public final void kill() { + synchronized (this) { + interrupted = true; + if (threadToInterrupt != null) + threadToInterrupt.interrupt(); + } + } + + public boolean processTimedOut() { + return hitTimeout; + } + + @Override + public String toString() { + return commandLineToString(getBuilderCommand()); + } + + private List getBuilderCommand() { + return leakPrevention.cleanUpArguments(builder.command()); + } + + private static String commandLineToString(List commandLine) { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (String s : commandLine) { + boolean tricky = s.isEmpty() || s.contains(" ") ; + + if (!first) + sb.append(" "); + first = false; + if (tricky) + sb.append("\""); + + sb.append(s.replace("\"", "\\\"")); + + if (tricky) + sb.append("\""); + } + return sb.toString(); + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/process/Builder.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/process/Builder.java new file mode 100644 index 00000000000..b54a8fe88f8 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/process/Builder.java @@ -0,0 +1,81 @@ +package com.semmle.util.process; + +import java.io.File; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import com.semmle.util.io.StreamMuncher; +import com.semmle.util.logging.Streams; + +public class Builder extends AbstractProcessBuilder { + + private final OutputStream err; + private final OutputStream out; + protected StreamMuncher errMuncher; + protected StreamMuncher outMuncher; + + public Builder(OutputStream out, OutputStream err, File cwd, String... args) { + this(out, err, cwd, null, args); + } + + public Builder(OutputStream out, OutputStream err, File cwd, + Map env, String... args) { + this(Arrays.asList(args), out, err, env, cwd); + } + + public Builder(List args, OutputStream out, OutputStream err) { + this(args, out, err, null, null); + } + + public Builder(List args, OutputStream out, OutputStream err, + File cwd) { + this(args, out, err, null, cwd); + } + + public Builder(List args, OutputStream out, OutputStream err, + Map env) { + this(args, out, err, env, null); + } + + public Builder(List args, OutputStream out, OutputStream err, + Map env, File cwd) { + super(args, cwd, env); + this.out = out; + this.err = err; + } + + /** + * Convenience method that executes the given command line in the current + * working directory with the current environment, blocking until + * completion. The process's output stream is redirected to System.out, and + * its error stream to System.err. It returns the exit code of the command. + */ + public static int run(List commandLine) { + return new Builder(commandLine, Streams.out(), Streams.err()).execute(); + } + + @Override + protected void cleanupOutputHandling() { + // wait for munchers to finish munching. + long deadline = 1000*MUNCH_TIMEOUT; + // note: check that munchers are not null, in case setupOutputHandling was + // not called to initialize them + if(outMuncher != null) { + waitForMuncher("output", outMuncher,deadline); + } + if(errMuncher != null) { + waitForMuncher("error", errMuncher,deadline); + } + } + + @Override + protected void setupOutputHandling(InputStream stdout, InputStream stderr) { + errMuncher = new StreamMuncher(stderr, err); + errMuncher.start(); + outMuncher = new StreamMuncher(stdout, out); + outMuncher.start(); + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/process/Env.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/process/Env.java new file mode 100644 index 00000000000..564887f5fda --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/process/Env.java @@ -0,0 +1,699 @@ +package com.semmle.util.process; + +import java.io.Serializable; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Stack; +import java.util.TreeMap; + +import com.semmle.util.exception.Exceptions; +import com.semmle.util.expansion.ExpansionEnvironment; + +/** + * Helper methods for finding out environment properties like the OS type. + */ +public class Env { + /** + * Enum for commonly used environment variables. + * + *

+ * The intention is that the name of the enum constant is the same as the environment + * variable itself. This means that the toString method does the right thing, + * as does calling {@link Enum#name() }. + *

+ * + *

+ * Should you wish to rename an environment variable (which you're unlikely to, due to the + * fact that there are many non-Java consumers), you can do a rename refactoring to make the + * Java consumers do the right thing. + *

+ */ + public enum Var { + /* + * Core toolchain variables + */ + /** + * The location of the user's configuration files, including project configurations, + * dashboard configurations, team insight configurations, licenses etc. + */ + SEMMLE_HOME, + /** + * The location of the user's data, including snapshots, built dashboards, team + * insight data, etc. + */ + SEMMLE_DATA, + /** + * The location of any caches used by the toolchain, including compilation caches, trap caches, etc. + */ + SEMMLE_CACHE, + /** + * If running from a git tree, the root of the tree. + */ + SEMMLE_GIT_ROOT, + /** + * The root from which relative paths in a DOOD file are resolved. + */ + SEMMLE_QUERY_ROOT, + /** + * The directory where lock files are kept. + */ + SEMMLE_LOCK_DIR, + /** + * The directory which will be checked for licenses. + */ + SEMMLE_LICENSE_DIR, + /** + * The location where our queries are kept. + */ + ODASA_QUERIES, + /** + * Whether we are running in 'prototyping mode'. + */ + ODASA_PROTOTYPE_MODE, + /** + * The location of the default compilation cache, as a space-separated list of URIs. + * + * Multiple entries are tried in sequence. + */ + SEMMLE_COMPILATION_CACHE, + /** + * Override the versions used in compilation caching. + * + * This is useful for testing without modifying the version manually. + */ + SEMMLE_OVERRIDE_OPTIMISER_VERSION, + /** + * If set, do not use compilation caching. + */ + SEMMLE_NO_COMPILATION_CACHING, + /** + * If set, use this as the size of compilation caches, in bytes. If set to 'INFINITY', no + * limit will be placed on the size. + */ + SEMMLE_COMPILATION_CACHE_SIZE, + + /* + * Other toolchain variables + */ + SEMMLE_JAVA_HOME, + ODASA_JAVA_HOME, + ODASA_TRACER_CONFIGURATION, + /** + * The Java tracer agent to propagate to JVM processes. + */ + SEMMLE_JAVA_TOOL_OPTIONS, + /** + * Whether to run jar-based subprocesses in-process instead. + */ + ODASA_IN_PROCESS, + /** + * The executable to use for importing trap files. + */ + SEMMLE_TRAP_IMPORTER, + SEMMLE_PRESERVE_SYMLINKS, + SEMMLE_PATH_TRANSFORMER, + + /* + * Environment variables for password for credential stores. + * Either is accepted to allow a single entry point in the code + * while documenting as appropriate for the audience. + */ + SEMMLE_CREDENTIALS_PASSWORD, + LGTM_CREDENTIALS_PASSWORD, + + /* + * + * Internal config variables + */ + /** + * Extra arguments to pass to JVMs launched by Semmle tools. + */ + SEMMLE_JAVA_ARGS, + /** + * A list of log levels to set, of the form: + * "foo.bar=TRACE,bar.baz=DEBUG" + */ + SEMMLE_LOG_LEVELS, + /** + * The default heap size for commands that accept a ram parameter. + */ + SEMMLE_DEFAULT_HEAP_SIZE, + SEMMLE_MAX_RAM_MB, + /** + * Whether to disable asynchronous logging in the query server (otherwise it may drop messages). + */ + SEMMLE_SYNCHRONOUS_LOGGING, + /** + * Whether or not to use memory mapping + */ + SEMMLE_MEMORY_MAPPING, + SEMMLE_METRICS_DIR, + /** + * Whether we are running in our own unit tests. + */ + SEMMLE_UNIT_TEST_MODE, + /** + * Whether to include the source QL in a QLO. + */ + SEMMLE_DEBUG_QL_IN_QLO, + /** + * Whether to enable extra assertions + */ + ODASA_ASSERTIONS, + /** + * A file containing extra variables for ExpansionEnvironments. + */ + ODASA_EXTRA_VARIABLES, + ODASA_TUNE_GC, + /** + * Whether to run PI in hosted mode. + */ + SEMMLE_ODASA_DEBUG, + /** + * The python executable to use for Qltest. + */ + SEMMLE_PYTHON, + /** + * The platform we are running on; one of "linux", "osx" and "win". + */ + SEMMLE_PLATFORM, + /** + * PATH to use to look up tooling required by macOS Relocator scripts. + */ + CODEQL_TOOL_PATH, + /** + * This can override the heuristics for BDD factory resetting. Most useful for measurements + * and debugging. + */ + CODEQL_BDD_RESET_FRACTION, + + /** + * How many TRAPLinker errors to report. + */ + SEMMLE_MAX_TRAP_ERRORS, + + /** + * How many tuples to accumulate in memory before pushing to disk. + */ + SEMMLE_MAX_TRAP_INMEMORY_TUPLES, + /** + * How many files to merge at each merge step. + */ + SEMMLE_MAX_TRAP_MERGE, + + /* + * Variables used by extractors. + */ + /** + * Whether the C++ extractor should copy executables before + * running them (works around System Integrity Protection + * on OS X 10.11+). + */ + SEMMLE_COPY_EXECUTABLES, + /** + * When SEMMLE_COPY_EXECUTABLES is in operation, where to + * create the directory to copy the executables to. + */ + SEMMLE_COPY_EXECUTABLES_SUPER_ROOT, + /** + * When SEMMLE_COPY_EXECUTABLES is in operation, the + * directory we are copying executables to. + */ + SEMMLE_COPY_EXECUTABLES_ROOT, + /** + * The executable which should be used as an implicit runner on Windows. + */ + SEMMLE_WINDOWS_RUNNER_BINARY, + /** + * Verbosity level for the Java interceptor. + */ + SEMMLE_INTERCEPT_VERBOSITY, + /** + * Verbosity level for the Java extractor. + */ + ODASA_JAVAC_VERBOSE, + /** + * Whether to use class origin tracking for the Java extractor. + */ + ODASA_JAVA_CLASS_ORIGIN_TRACKING, + ODASA_JAVAC_CORRECT_EXCEPTIONS, + ODASA_JAVAC_EXTRA_CLASSPATH, + ODASA_NO_ECLIPSE_BUILD, + + /* + * Variables set during snapshot builds + */ + /** + * The location of the project being built. + */ + ODASA_PROJECT, + /** + * The location of the snapshot being built. + */ + ODASA_SRC, + ODASA_DB, + ODASA_OUTPUT, + ODASA_SUBPROJECT_THREADS, + + /* + * Layout variables + */ + ODASA_CPP_LAYOUT, + ODASA_CSHARP_LAYOUT, + ODASA_PYTHON_LAYOUT, + ODASA_JAVASCRIPT_LAYOUT, + + /* + * External variables + */ + JAVA_HOME, + PATH, + LINUX_VARIANT, + + /* + * If set, use this proxy for HTTP requests + */ + HTTP_PROXY, + http_proxy, + + /* + * If set, use this proxy for HTTPS requests + */ + HTTPS_PROXY, + https_proxy, + + /* + * If set, ignore the variables above and do not use any proxies for requests + */ + NO_PROXY, + no_proxy, + + /* + * Variables set by the codeql-action. All variables will + * be unset if the CLI is not in the context of the + * codeql-action. + */ + + /** + * Either {@code actions} or {@code runner}. + */ + CODEQL_ACTION_RUN_MODE, + + /** + * Semantic version of the codeql-action. + */ + CODEQL_ACTION_VERSION, + /* + * tracer variables + */ + /** + * Colon-separated list of enabled tracing languages + */ + CODEQL_TRACER_LANGUAGES, + /** + * Path to the build-tracer log file + */ + CODEQL_TRACER_LOG, + /** + * Prefix to a language-specific root directory + */ + CODEQL_TRACER_ROOT_, + + ; + } + + private static final int DEFAULT_RAM_MB_32 = 1024; + private static final int DEFAULT_RAM_MB = 4096; + private static final Env instance = new Env(); + + private final Stack> envVarContexts; + + public static synchronized Env systemEnv() { + return instance; + } + + /** + * Create an instance of Env containing no variables. Intended for use in + * testing to isolate the test from the local machine environment. + */ + public static Env emptyEnv() { + Env env = new Env(); + env.envVarContexts.clear(); + env.envVarContexts.push(Collections.unmodifiableMap(makeContext())); + return env; + } + + private static Map makeContext() { + if (getOS().equals(OS.WINDOWS)) { + // We want to compare in the same way Windows does, which means + // upper-casing. For example, '_' needs to come after 'Z', but + // would come before 'z'. + return new TreeMap<>((a, b) -> a.toUpperCase(Locale.ENGLISH).compareTo(b.toUpperCase(Locale.ENGLISH))); + } else { + return new LinkedHashMap<>(); + } + } + + public Env() { + envVarContexts = new Stack<>(); + Map env = makeContext(); + try { + env.putAll(System.getenv()); + } catch (SecurityException ex) { + Exceptions.ignore(ex, "Treat an inaccessible environment variable as not existing"); + } + envVarContexts.push(Collections.unmodifiableMap(env)); + } + + public synchronized void unsetAll(Collection names) { + if (!names.isEmpty()) { + Map map = envVarContexts.pop(); + map = new LinkedHashMap<>(map); + for (String name : names) + map.remove(name); + envVarContexts.push(Collections.unmodifiableMap(map)); + } + } + + public synchronized Map getenv() { + return envVarContexts.peek(); + } + + /** + * Get the value of an environment variable, or null if + * the environment variable is not set. WARNING: not all systems may + * make a difference between an empty variable or null, + * so don't rely on that behavior. + */ + public synchronized String get(Var var) { + return get(var.name()); + } + + /** + * Get the value of an environment variable, or null if + * the environment variable is not set. WARNING: not all systems may + * make a difference between an empty variable or null, + * so don't rely on that behavior. + */ + public synchronized String get(String envVarName) { + return getenv().get(envVarName); + } + + /** + * Get the non-empty value of an environment variable, or null + * if the environment variable is not set or set to an empty value. + */ + public synchronized String getNonEmpty(Var var) { + return getNonEmpty(var.name()); + } + + /** + * Get the value of an environment variable, or the empty string if it is not + * set. + */ + public synchronized String getPossiblyEmpty(String envVarName) { + String got = getenv().get(envVarName); + return got != null ? got : ""; + } + + /** + * Get the non-empty value of an environment variable, or null + * if the environment variable is not set or set to an empty value. + */ + public synchronized String getNonEmpty(String envVarName) { + String s = get(envVarName); + return s == null || s.isEmpty() ? null : s; + } + + /** + * Gets the value of the first environment variable among envVarNames + * whose value is non-empty, or null if all variables have empty values. + */ + public synchronized String getFirstNonEmpty(String... envVarNames) { + for (String envVarName : envVarNames) { + String s = getNonEmpty(envVarName); + if (s != null) + return s; + } + return null; + } + + /** + * Gets the value of the first environment variable among envVars + * whose value is non-empty, or null if all variables have empty values. + */ + public synchronized String getFirstNonEmpty(Var... envVars) { + String[] envVarNames = new String[envVars.length]; + for (int i = 0; i < envVars.length; ++i) + envVarNames[i] = envVars[i].name(); + return getFirstNonEmpty(envVarNames); + } + + /** + * Read a boolean from the given environment variable. If the variable + * is not set, then return false. Otherwise, interpret the + * environment variable using {@link Boolean#parseBoolean(String)}. + */ + public boolean getBoolean(Var var) { + return getBoolean(var.name()); + } + + /** + * Read a boolean from the given environment variable name. If the variable + * is not set, then return false. Otherwise, interpret the + * environment variable using {@link Boolean#parseBoolean(String)}. + */ + public boolean getBoolean(String envVarName) { + return getBoolean(envVarName, false); + } + + /** + * Read a boolean from the given environment variable. If the variable + * is not set, then return def. Otherwise, interpret the + * environment variable using {@link Boolean#parseBoolean(String)}. + */ + public boolean getBoolean(Var var, boolean def) { + return getBoolean(var.name(), def); + } + + /** + * Read a boolean from the given environment variable name. If the variable + * is not set, then return def. Otherwise, interpret the + * environment variable using {@link Boolean#parseBoolean(String)}. + */ + public boolean getBoolean(String envVarName, boolean def) { + String v = get(envVarName); + return v == null ? def : Boolean.parseBoolean(v); + } + + /** + * Read an integer setting from the given environment variable name. If the + * variable is not set, or fails to parse, return the supplied default value. + */ + public int getInt(Var var, int defaultValue) { + return getInt(var.name(), defaultValue); + } + + /** + * Read an integer setting from the given environment variable name. If the + * variable is not set, or fails to parse, return the supplied default value. + */ + public int getInt(String envVarName, int defaultValue) { + String value = get(envVarName); + if (value == null) + return defaultValue; + + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + Exceptions.ignore(e, "We'll just use the default value."); + return defaultValue; + } + } + + /** + * Enter a new context for environment variables, with the given + * new variable values. The values will override the current environment + * values if they define the same variables. + */ + public synchronized void pushEnvironmentContext(Map addedValues) { + Map newValues = makeContext(); + newValues.putAll(envVarContexts.peek()); + newValues.putAll(addedValues); + envVarContexts.push(Collections.unmodifiableMap(newValues)); + } + + /** + * Leave a context for environment variables that was created with + * pushEnvironmentContext + */ + public synchronized void popEnvironmentContext() { + envVarContexts.pop(); + } + + /** + * Add all the custom environment variables to a process builder, so that + * they are passed on to the child process. + */ + public synchronized void addEnvironmentToNewProcess(ProcessBuilder builder) { + if (envVarContexts.size() > 1) + builder.environment().putAll(envVarContexts.peek()); + } + + public synchronized void addEnvironmentToNewEnv(ExpansionEnvironment env) { + if (envVarContexts.size() > 1) + env.defineVars(envVarContexts.peek()); + } + + /** + * Get a string representing the OS type. This + * is not guaranteed to have any particular form, and + * is for displaying to users. Might return null if + * the property is not defined by the JVM. + */ + public static String getOSName() { + return System.getProperty("os.name"); + } + + /** + * Determine which OS is currently being run (somewhat best-effort). + * Does not determine whether a program is being run under Cygwin + * or not - Windows will be the OS even under Cygwin. + */ + public static OS getOS() { + String name = getOSName(); + if (name == null) + return OS.UNKNOWN; + if (name.contains("Windows")) + return OS.WINDOWS; + else if (name.contains("Mac OS X")) + return OS.MACOS; + else if (name.contains("Linux")) + return OS.LINUX; + else + // Guess that we are probably some Unix flavour + return OS.UNKNOWN_UNIX; + } + + /** + * Kinds of operating systems. A notable absence is Cygwin: this just + * gets reported as Windows. + */ + public static enum OS { + WINDOWS(false, false), LINUX(true, true), MACOS(false, true), UNKNOWN_UNIX(true, true), UNKNOWN(true, true),; + + private final boolean fileSystemCaseSensitive; + private final boolean envVarsCaseSensitive; + + private OS(boolean fileSystemCaseSensitive, boolean envVarsCaseSensitive) { + this.fileSystemCaseSensitive = fileSystemCaseSensitive; + this.envVarsCaseSensitive = envVarsCaseSensitive; + } + + /** + * Get an OS value from the short display name. Acceptable + * inputs (case insensitive) are: Windows, Linux, MacOS or + * Mac OS. + * + * @throws IllegalArgumentException if the given name does not + * correspond to an OS + */ + public static OS fromDisplayName(String name) { + if (name != null) { + name = name.toUpperCase(); + if ("WINDOWS".equals(name)) + return WINDOWS; + if ("LINUX".equals(name)) + return LINUX; + if ("MACOS".equals(name.replace(" ", ""))) + return MACOS; + } + throw new IllegalArgumentException("No OS type found with name " + name); + } + + public boolean isFileSystemCaseSensitive() { + return fileSystemCaseSensitive; + } + + public boolean isEnvironmentCaseSensitive() { + return envVarsCaseSensitive; + } + + /** The short name of this operating system, in the style of {@link Var#SEMMLE_PLATFORM}. */ + public String getShortName() { + switch (this) { + case WINDOWS: + return "win"; + case LINUX: + return "linux"; + case MACOS: + return "osx"; + default: + return "unknown"; + } + } + } + + public static enum Architecture { + X86(true, false), X64(false, true), UNDETERMINED(false, false); + + private final boolean is32Bit; + private final boolean is64Bit; + + private Architecture(boolean is32Bit, boolean is64Bit) { + this.is32Bit = is32Bit; + this.is64Bit = is64Bit; + } + + /** Is this definitely a 32-bit architecture? */ + public boolean is32Bit() { + return is32Bit; + } + + /** Is this definitely a 64-bit architecture? */ + public boolean is64Bit() { + return is64Bit; + } + } + + /** + * Try to detect whether the JVM is 32-bit or 64-bit. Since there is no documented, + * portable way to do this it is best effort. + */ + public Architecture tryDetermineJvmArchitecture() { + String value = System.getProperty("sun.arch.data.model"); + if ("32".equals(value)) + return Architecture.X86; + else if ("64".equals(value)) + return Architecture.X64; + + // Look at the max heap value - if >= 4G we *must* be in 64-bit + long maxHeap = Runtime.getRuntime().maxMemory(); + if (maxHeap < Long.MAX_VALUE && maxHeap >= 4096L << 20) + return Architecture.X64; + + // Try to get the OS arch - it *appears* to give JVM bitness + String osArch = System.getProperty("os.arch"); + if ("x86".equals(osArch) || "i386".equals(osArch)) + return Architecture.X86; + else if ("x86_64".equals(osArch) || "amd64".equals(osArch)) + return Architecture.X64; + + return Architecture.UNDETERMINED; + } + + /** + * Get the default amount of ram to use for new JVMs, depending on the + * current architecture. If it looks like we're running on a 32-bit + * machine, the result is sufficiently small to be representable. + */ + public int defaultRamMb() { + return getInt( + Var.SEMMLE_DEFAULT_HEAP_SIZE, + tryDetermineJvmArchitecture().is32Bit() ? DEFAULT_RAM_MB_32 : DEFAULT_RAM_MB); + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/process/LeakPrevention.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/process/LeakPrevention.java new file mode 100644 index 00000000000..d27bde39430 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/process/LeakPrevention.java @@ -0,0 +1,95 @@ +package com.semmle.util.process; + +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collections; +import java.util.List; + +public abstract class LeakPrevention { + + public abstract List cleanUpArguments(List args); + + /** + * What to put in place of any suppressed arguments. + */ + static final String REPLACEMENT_STRING = "*****"; + + /** + * Hides all arguments. Will only show the command name. + * e.g. "foo bar baz" is changed to "foo" + */ + public static final LeakPrevention ALL = new LeakPrevention() { + @Override + public List cleanUpArguments(List args) { + return args.isEmpty() ? args : Collections.singletonList(args.get(0)); + } + }; + + /** + * Does not hide any arguments. + */ + public static final LeakPrevention NONE = new LeakPrevention() { + @Override + public List cleanUpArguments(List args) { + return args; + } + }; + + /** + * Hides the arguments at the given indexes. + */ + public static LeakPrevention suppressedArguments(int... args) { + if (args.length == 0) + return NONE; + + final BitSet suppressed = new BitSet(); + for (int index : args) { + suppressed.set(index); + } + + return new LeakPrevention() { + @Override + public List cleanUpArguments(List args) { + List result = new ArrayList<>(args.size()); + int index = 0; + for (String arg : args) { + if (suppressed.get(index)) + result.add(REPLACEMENT_STRING); + else + result.add(arg); + index++; + } + return result; + } + }; + } + + /** + * Hides the given string from any arguments that it appears in. + * The substring will be replaced while leaving the rest of the + * argument unmodified. + *

+ * There are some potential pitfalls to be aware of when using this + * method. + *

    + *
  • This only suppresses exact textual matches. If the argument that + * appears is only derived from the secret instead of being an exact + * copy then it will not be suppressed. + *
  • If the secret value appears elsewhere in a known string, then it + * could leak the contents of the secret because the viewer knows what + * should have been there in the known case. + *
+ */ + public static LeakPrevention suppressSubstring(final String substringToSuppress) { + return new LeakPrevention() { + @Override + public List cleanUpArguments(List args) { + List result = new ArrayList<>(args.size()); + for (String arg : args) { + result.add(arg.replace(substringToSuppress, REPLACEMENT_STRING)); + } + return result; + } + }; + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/projectstructure/ProjectLayout.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/projectstructure/ProjectLayout.java new file mode 100644 index 00000000000..4d414c65632 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/projectstructure/ProjectLayout.java @@ -0,0 +1,529 @@ +package com.semmle.util.projectstructure; + +import java.io.File; +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.semmle.util.data.StringUtil; +import com.semmle.util.exception.CatastrophicError; +import com.semmle.util.exception.UserError; +import com.semmle.util.io.WholeIO; + +/** + * A project-layout file optionally begins with an '@' + * followed by the name the project should be renamed to. + * Optionally, it can then be followed by a list of + * include/exclude patterns (see below) which are kept + * as untransformed paths. This is followed by one or + * more clauses. Each clause has the following form: + * + * #virtual-path + * path/to/include + * another/path/to/include + * -/path/to/include/except/this + * + * i.e. one or more paths (to include) and zero or more paths + * prefixed by minus-signs (to exclude). + */ +public class ProjectLayout +{ + public static final char PROJECT_NAME_PREFIX = '@'; + + private String project; + + /** + * Map from virtual path prefixes (following the '#' in the project-layout) + * to the sequence of patterns that fall into that section. Declared as a + * {@link LinkedHashMap} since iteration order matters -- we process blocks in + * the same order as they occur in the project-layout. + */ + private final LinkedHashMap sections = new LinkedHashMap(); + + /** + * A file name, or similar string, to use in error messages so that the + * user knows what to fix. + */ + private String source; + + /** + * Load a project-layout file. + * + * @param file the project-layout to load + */ + public ProjectLayout(File file) { + this(StringUtil.lines(new WholeIO().strictread(file)), file.toString()); + } + + /** + * Construct a project-layout object from an array of strings, each + * corresponding to one line of the project-layout. This constructor + * is for testing. For other uses see {@link ProjectLayout#ProjectLayout(File)}. + * + * @param lines the lines of the project-layout + */ + public ProjectLayout(String... lines) { + this(lines, null); + } + + private ProjectLayout(String[] lines, String source) { + this.source = source; + String virtual = ""; + Section section = new Section(""); + sections.put("", section); + int num = 0; + for (String line : lines) { + num++; + line = line.trim(); + if (line.isEmpty()) + continue; + switch (line.charAt(0)) { + case PROJECT_NAME_PREFIX: + if (project != null) + throw error("Only one project name is allowed", source, num); + project = tail(line); + break; + case '#': + virtual = tail(line); + if (sections.containsKey(virtual)) + throw error("Duplicate virtual path prefix " + virtual, source, num); + section = new Section(virtual); + sections.put(virtual, section); + break; + case '-': + section.add(new Rewrite(tail(line), source, num)); + break; + default: + section.add(new Rewrite(line, virtual, source, num)); + } + } + } + + private static String tail(String line) { + return line.substring(1).trim(); + } + + /** + * Get the project name, if specified by the project-layout. This + * method should only be called if it is guaranteed that the + * project-layout will contain a project name, and it throws + * a {@link UserError} if it doesn't. + * @return the project name -- guaranteed not null. + * @throws UserError if the project-layout file did not specify a + * project name. + */ + public String projectName() { + if (project == null) + throw error("No project name is defined", source); + return project; + } + + /** + * Get the project name, if specified by the project-layout file. + * If the file contains no renaming specification, return the + * given default value. + * @param defaultName The name to use if the project-layout doesn't + * specify a target project name. + * @return the specified name or default value. + */ + public String projectName(String defaultName) { + return project == null ? defaultName : project; + } + + /** + * @return the section headings (aka virtual paths) + */ + public List sections() { + List result = new ArrayList(); + result.addAll(sections.keySet()); + return result; + } + + /** + * Determine whether or not a particular section in this + * project-layout is empty (has no include/exclude patterns). + * + * @param section the name of the section + * @return true if the section is empty + */ + public boolean sectionIsEmpty(String section) { + if (!sections.containsKey(section)) + throw new CatastrophicError("Section does not exist: " + section); + return sections.get(section).isEmpty(); + } + + /** + * Reaname a section in this project-layout. + * + * @param oldName the old name of the section + * @param newName the new name + */ + public void renameSection(String oldName, String newName) { + if (!sections.containsKey(oldName)) + throw new CatastrophicError("Section does not exist: " + oldName); + Section section = sections.remove(oldName); + section.rename(newName); + sections.put(newName, section); + } + + /** + * Return a project-layout file for just one of the sections in this + * project-layout. This is done by copying all the rules from the + * section, and changing the section heading (beginning with '#') + * to a project name (beginning with '@'). + * + * @param sectionName the section to create a project-layout from + * @return the text of the newly created project-layout + */ + public String subLayout(String sectionName) { + Section section = sections.get(sectionName); + if (section == null) + throw new CatastrophicError("Section does not exist: " + section); + return section.toLayout(); + } + + /** + * Maps a path to its corresponding artificial path according to the + * rules in this project-layout. If the path is excluded (either + * explicitly, or because it is not mentioned in the project-layout) + * then null is returned. + *

+ * Paths should start with a leading forward-slash + * + * @param path the path to map + * @return the artificial path, or null if the path is excluded + */ + public String artificialPath(String path) { + // If there is no leading slash, the path does not conform to the expected + // format and there is no match. (An exception is made for a completely + // empty string, which will get the sole prefix '/' and be mapped as usual). + if (path.length() > 0 && path.charAt(0) != '/') + return null; + List prefixes = Section.prefixes(path); + for (Section section : sections.values()) { + Rewrite rewrite = section.match(prefixes); + String rewritten = null; + if (rewrite != null) + rewritten = rewrite.rewrite(path); + if (rewritten != null) + return rewritten; + } + return null; + } + + /** + * Checks whether a path should be included in the project specified by + * this file. A file is included if it is mapped to some location. + *

+ * Paths should start with a leading forward-slash + * + * @param path the path to check + * @return true if the path should be included + */ + public boolean includeFile(String path) { + return artificialPath(path) != null; + } + + public void writeTo(Writer writer) throws IOException { + if (project != null) { + writer.write(PROJECT_NAME_PREFIX); + writer.write(project); + writer.write("\n"); + } + for (Section section : sections.values()) { + if (!section.virtual.isEmpty()) { + writer.write("#"); + writer.write(section.virtual); + writer.write("\n"); + } + section.outputRules(writer); + } + } + + public void addPattern(String section, String pattern) { + if (pattern == null || pattern.isEmpty()) { + throw new IllegalArgumentException("ProjectLayout.addPattern: pattern must be a non-empty string"); + } + boolean exclude = pattern.charAt(0) == '-'; + Rewrite rewrite = exclude ? + new Rewrite(pattern.substring(1), null, 0) : + new Rewrite(pattern, section, null, 0); + Section s = sections.get(section); + if (s == null) { + s = new Section(section); + sections.put(section, s); + } + s.add(rewrite); + } + + private static UserError error(String message, String source) { + return error(message, source, 0); + } + + private static UserError error(String message, String source, int line) { + if (source == null) + return new UserError(message); + StringBuilder sb = new StringBuilder(message); + sb.append(" ("); + if (line > 0) + sb.append("line ").append(line).append(" of "); + sb.append(source).append(")"); + return new UserError(sb.toString()); + } + + /** + * Each section corresponds to a block beginning with '#some/path'. There + * is also an initial section for any include/exclude patterns before the + * first '#'. + */ + private static class Section { + private String virtual; + private final Map simpleRewrites; + private final List complexRewrites; + + public Section(String virtual) { + this.virtual = virtual; + simpleRewrites = new LinkedHashMap(); + complexRewrites = new ArrayList(); + } + + public String toLayout() { + StringWriter result = new StringWriter(); + result.append('@').append(virtual).append('\n'); + try { + outputRules(result); + } catch (IOException e) { + throw new CatastrophicError("StringWriter.append threw an IOException", e); + } + return result.toString(); + } + + private void outputRules(Writer writer) throws IOException { + List all = new ArrayList(); + all.addAll(simpleRewrites.values()); + all.addAll(complexRewrites); + Collections.sort(all, Rewrite.COMPARATOR); + for (Rewrite rewrite : all) + writer.append(rewrite.toString()).append('\n'); + } + + public void rename(String newName) { + virtual = newName; + for (Rewrite rewrite : simpleRewrites.values()) + rewrite.virtual = newName; + for (Rewrite rewrite : complexRewrites) + rewrite.virtual = newName; + } + + public void add(Rewrite rewrite) { + int index = simpleRewrites.size() + complexRewrites.size(); + rewrite.setIndex(index); + if (rewrite.isSimple()) + simpleRewrites.put(rewrite.simplePrefix(), rewrite); + else + complexRewrites.add(rewrite); + } + + public boolean isEmpty() { + return simpleRewrites.isEmpty() && complexRewrites.isEmpty(); + } + + private static List prefixes(String path) { + List result = new ArrayList(); + result.add(path); + int i = path.length(); + while (i > 1) { + i = path.lastIndexOf('/', i - 1); + result.add(path.substring(0, i)); + } + result.add("/"); + return result; + } + + public Rewrite match(List prefixes) { + Rewrite best = null; + for (String prefix : prefixes) { + Rewrite match = simpleRewrites.get(prefix); + if (match != null) + if (best == null || best.index < match.index) + best = match; + } + // Last matching rewrite 'wins' + for (int i = complexRewrites.size() - 1; i >= 0; i--) { + Rewrite rewrite = complexRewrites.get(i); + if (rewrite.matches(prefixes.get(0))) { + if (best == null || best.index < rewrite.index) + best = rewrite; + // no point continuing + break; + } + } + return best; + } + } + + /** + * Each Rewrite corresponds to a single include or exclude line in the project-layout. + * For example, for following clause there would be three Rewrite objects: + * + * #Source + * /src + * /lib + * -/src/tests + * + * For includes use the two-argument constructor; for excludes the one-argument constructor. + */ + private static class Rewrite { + + private static final Comparator COMPARATOR = new Comparator() { + + @Override + public int compare(Rewrite t, Rewrite o) { + if (t.index < o.index) + return -1; + if (t.index == o.index) + return 0; + return 1; + } + }; + + private int index; + private final String original; + private final Pattern pattern; + private String virtual; + private final String simple; + + /** + * The intention is to allow the ** wildcard when followed by a slash only. The + * following should be invalid: + * - a / *** / b (too many stars) + * - a / ** (** at the end should be omitted) + * - a / **b (illegal) + * - a / b** (illegal) + * - ** (the same as a singleton '/') + * This regex matches ** when followed by a non-/ character, or the end of string. + */ + private static final Pattern verifyStars = Pattern.compile(".*(?:\\*\\*[^/].*|\\*\\*$|[^/]\\*\\*.*)"); + + public Rewrite(String exclude, String source, int line) { + original = '-' + exclude; + if (!exclude.startsWith("/")) + exclude = '/' + exclude; + if (exclude.indexOf("//") != -1) + throw error("Illegal '//' in exclude path", source, line); + if (verifyStars.matcher(exclude).matches()) + throw error("Illegal use of '**' in exclude path", source, line); + if (exclude.endsWith("/")) + exclude = exclude.substring(0, exclude.length() - 1); + pattern = compilePrefix(exclude); + exclude = exclude.replace("//", "/"); + if (exclude.length() > 1 && exclude.endsWith("/")) + exclude = exclude.substring(0, exclude.length() - 1); + simple = exclude.contains("*") ? null : exclude; + } + + public void setIndex(int index) { + this.index = index; + } + + public Rewrite(String include, String virtual, String source, int line) { + original = include; + if (!include.startsWith("/")) + include = '/' + include; + int doubleslash = include.indexOf("//"); + if (doubleslash != include.lastIndexOf("//")) + throw error("More than one '//' in include path", source, line); + if (verifyStars.matcher(include).matches()) + throw error("Illegal use of '**' in include path", source, line); + if (!virtual.startsWith("/")) + virtual = "/" + virtual; + if (virtual.endsWith("/")) + virtual = virtual.substring(0, virtual.length() - 1); + this.virtual = virtual; + this.pattern = compilePrefix(include); + include = include.replace("//", "/"); + if (include.length() > 1 && include.endsWith("/")) + include = include.substring(0, include.length() - 1); + simple = include.contains("*") ? null : include; + } + + /** + * Patterns are matched by translation to regex. The following invariants + * are assumed to hold: + * + * - The pattern starts with a '/'. + * - There are no occurrences of '**' that is not surrounded by slashes + * (unless it is at the start of a pattern). + * - There is at most one double slash. + * + * The result of the translation has precisely one capture group, which + * (after successful matching) will contain the part of the path that + * should be glued to the virtual prefix. + * + * It proceeds by starting the capture group either after the double + * slash or at the start of the pattern, and then replacing '*' with + * '[^/]*' (meaning any number of non-slash characters) and '/**' with + * '(?:|/.*)' (meaning empty string or a slash followed by any number of + * characters including '/'). + * + * The pattern is terminated by the term '(?:/.*|$)', saying 'either the + * next character is a '/' or the string ends' -- this avoids accidental + * matching of partial directory/file names. + * + * IMPORTANT: Run the ProjectLayoutTests when changing this! + */ + private static Pattern compilePrefix(String pattern) { + pattern = StringUtil.escapeStringLiteralForRegexp(pattern, "*"); + if (pattern.contains("//")) + pattern = pattern.replace("//", "(/"); + else + pattern = "(" + pattern; + if (pattern.endsWith("/")) + pattern = pattern.substring(0, pattern.length() - 1); + pattern = pattern.replace("/**", "-///-") + .replace("*", "[^/]*") + .replace("-///-", "(?:|/.*)"); + return Pattern.compile(pattern + "(?:/.*|$))"); + } + + /** Is this rewrite simple? (i.e. contains no wildcards) */ + public boolean isSimple() { + return simple != null; + } + + /** Returns the path included/excluded by this rewrite, if it is + * simple, or null if it is not. + * + * @return included/excluded path, or null + */ + public String simplePrefix() { + return simple; + } + + public boolean matches(String path) { + return pattern.matcher(path).matches(); + } + + public String rewrite(String path) { + if (virtual == null) + return null; + Matcher matcher = pattern.matcher(path); + if (!matcher.matches()) + return null; + return virtual + matcher.group(1); + } + + @Override + public String toString() { + return original; + } + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/CompressedFileInputStream.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/CompressedFileInputStream.java new file mode 100644 index 00000000000..0a6fc1c2915 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/CompressedFileInputStream.java @@ -0,0 +1,29 @@ +package com.semmle.util.trap; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import com.semmle.util.zip.MultiMemberGZIPInputStream; + +public class CompressedFileInputStream { + /** + * Create an input stream for reading the uncompressed data from a (possibly) compressed file, with + * the decompression method chosen based on the file extension. + * + * @param f The compressed file to read + * @return An input stream from which you can read the file's uncompressed data. + * @throws IOException From the underlying decompression input stream. + */ + public static InputStream fromFile(Path f) throws IOException { + InputStream fileInputStream = Files.newInputStream(f); + if (f.getFileName().toString().endsWith(".gz")) { + return new MultiMemberGZIPInputStream(fileInputStream, 8192); + //} else if (f.getFileName().toString().endsWith(".br")) { + // return new BrotliInputStream(fileInputStream); + } else { + return fileInputStream; + } + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/dependencies/TextFile.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/dependencies/TextFile.java new file mode 100644 index 00000000000..c896827ee2f --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/dependencies/TextFile.java @@ -0,0 +1,125 @@ +package com.semmle.util.trap.dependencies; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.semmle.util.exception.ResourceError; +import com.semmle.util.io.StreamUtil; +import com.semmle.util.io.WholeIO; +import com.semmle.util.trap.CompressedFileInputStream; + +public abstract class TextFile { + static final String TRAPS = "TRAPS"; + private static final Pattern HEADER = Pattern.compile("([^\r\n]+?) (\\d\\.\\d)"); + + protected String version; + protected final Set traps = new LinkedHashSet(); + + protected abstract Set getSet(Path path, String label); + protected abstract void parseError(Path path); + + public TextFile(String version) { + this.version = version; + } + + /** + * Load the current text file, checking that it matches the expected header. + * + *

+ * This method is somewhat performance-sensitive, as at least our C++ extractors + * can generate very large input files. The format is therefore parsed by hand. + *

+ * + *

+ * The accepted format consists of: + *

    + *
  • Zero or more EOL comments, marked with {@code //}. + *
  • Precisely one header line, of the form {@code $HEADER $VERSION}; this is + * checked against {@code expected_header}. + *
  • Zero or more "file lists", each beginning with the name of a set (see + * {@link #getSet(File, String)}) on a line by itself, followed by file paths, + * one per line. + *
+ * + *

+ * Empty lines are permitted throughout. + *

+ */ + protected void load(String expected_header, Path path) { + try (InputStream is = CompressedFileInputStream.fromFile(path); + BufferedReader lines = StreamUtil.newUTF8BufferedReader(is)) { + boolean commentsPermitted = true; + Set currentSet = null; + for (String line = lines.readLine(); line != null; line = lines.readLine()) { + // Skip empty lines. + if (line.isEmpty()) + continue; + // If comments are still permitted, skip comment lines. + if (commentsPermitted && line.startsWith("//")) + continue; + // If comments are still permitted, the first non-comment line is the header. + // In addition, we allow no further comments. + if (commentsPermitted) { + Matcher matcher = HEADER.matcher(line); + if (!matcher.matches() || !matcher.group(1).equals(expected_header)) + parseError(path); + commentsPermitted = false; + version = matcher.group(2); + continue; + } + // We have a non-blank line; this either names the new set, or is a line that + // should be put into the current set. + Set newSet = getSet(path, line); + if (newSet != null) { + currentSet = newSet; + } else { + if (currentSet == null) + parseError(path); + else + currentSet.add(line); + } + } + } catch (IOException e) { + throw new ResourceError("Couldn't read " + path, e); + } + } + + /** + * @return the format version of the loaded file + */ + public String version() { + return version; + } + + /** + * Save this object to a file (or throw a ResourceError on failure) + * + * @param file the file in which to save this object + */ + public void save(Path file) { + new WholeIO().strictwrite(file, toString()); + } + + protected void appendHeaderString(StringBuilder sb, String header, String version) { + sb.append(header).append(' ').append(version).append('\n'); + } + + protected void appendSet(StringBuilder sb, String title, Set set) { + sb.append('\n').append(title).append('\n'); + for (String s : set) + sb.append(s).append('\n'); + } + + protected void appendSingleton(StringBuilder sb, String title, String s) { + sb.append('\n').append(title).append('\n'); + sb.append(s).append('\n'); + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/dependencies/TrapDependencies.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/dependencies/TrapDependencies.java new file mode 100644 index 00000000000..ff6880ea80e --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/dependencies/TrapDependencies.java @@ -0,0 +1,109 @@ +package com.semmle.util.trap.dependencies; + +import java.io.File; +import java.nio.file.Path; +import java.util.AbstractSet; +import java.util.Collections; +import java.util.Iterator; +import java.util.Set; + +import com.semmle.util.exception.ResourceError; + +/** + * The immediate dependencies of a particular TRAP file + */ +public class TrapDependencies extends TextFile +{ + static final String TRAP = "TRAP"; + + private String trap; + + /** + * Create an empty dependencies node for a TRAP file + */ + public TrapDependencies(String trap) { + super(TrapSet.LATEST_VERSION); + this.trap = trap; + } + + /** + * Load a TRAP dependencies (.dep) file + * + * @param file the file to load + */ + public TrapDependencies(Path file) { + super(null); + load(TrapSet.HEADER, file); + if(trap == null) + parseError(file); + } + + @Override + protected Set getSet(final Path file, String label) { + if(label.equals(TRAP)) { + return new AbstractSet() { + @Override + public Iterator iterator() { + return null; + } + @Override + public int size() { + return 0; + } + @Override + public boolean add(String s) { + if(trap != null) + parseError(file); + trap = s; + return true; + } + }; + } + if(label.equals(TRAPS)) return traps; + return null; + } + + @Override + protected void parseError(Path file) { + throw new ResourceError("Corrupt TRAP dependencies: " + file); + } + + /** + * @return the path of the TRAP with the dependencies stored in this object + * (relative to the source location) + */ + public String trapFile() { + return trap; + } + + /** + * @return the paths of the TRAP file dependencies + * (relative to the trap directory) + * + */ + public Set dependencies() { + return Collections.unmodifiableSet(traps); + } + + /** + * Add a path to a TRAP file (relative to the trap directory). + * + * @param trap the path to the trap file to add + */ + public void addDependency(String trap) { + traps.add(trap); + } + + /* + * (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + appendHeaderString(sb, TrapSet.HEADER, TrapSet.LATEST_VERSION); + appendSingleton(sb, TRAP, trap); + appendSet(sb, TRAPS, traps); + return sb.toString(); + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/dependencies/TrapSet.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/dependencies/TrapSet.java new file mode 100644 index 00000000000..d1d6760fdbf --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/dependencies/TrapSet.java @@ -0,0 +1,196 @@ +package com.semmle.util.trap.dependencies; + +import java.nio.file.Path; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Set; + +import com.semmle.util.exception.ResourceError; + +/** + * A set of source files and the TRAP files that were generated when + * compiling them. + *

+ * The set of TRAP files is not necessarily sufficient to create a + * consistent database, unless combined with inter-TRAP dependency + * information from .dep files (see {@link TrapDependencies}). + */ +public class TrapSet extends TextFile +{ + static final String HEADER = "TRAP dependencies"; + static final String LATEST_VERSION = "1.2"; + static final String SOURCES = "SOURCES"; + static final String INCLUDES = "INCLUDES"; + static final String OBJECTS = "OBJECTS"; + static final String INPUT_OBJECTS = "INPUT_OBJECTS"; + + // state + private final Set sources = new LinkedHashSet(); + private final Set includes = new LinkedHashSet(); + private final Set objects = new LinkedHashSet(); + private final Set inputObjects = new LinkedHashSet(); + + private Path file; + + /** + * Create an empty TRAP set + */ + public TrapSet() { + super(LATEST_VERSION); + } + + @Override + protected Set getSet(Path file, String label) { + if (label.equals(SOURCES)) return sources; + if (label.equals(INCLUDES)) return includes; + if (label.equals(OBJECTS)) return objects; + if (label.equals(INPUT_OBJECTS)) return inputObjects; + if (label.equals(TRAPS)) return traps; + return null; + } + + /** + * Load a TRAP set (.set) file + * + * @param path the file to load + */ + public TrapSet(Path path) { + super(null); + load(HEADER, path); + this.file = path; + } + + /** + * Return the most recent file used when loading or saving this + * trap set. If this set was constructed, rather than loaded, and + * has not been saved then the result is null. + * + * @return the file or null + */ + public Path getFile() { + return file; + } + + @Override + protected void parseError(Path file) { + throw new ResourceError("Corrupt TRAP set: " + file); + } + + /** + * @return the paths of the source files contained in this TRAP set + */ + public Set sourceFiles() { + return Collections.unmodifiableSet(sources); + } + + /** + * @return the paths to the include files contained in this TRAP set + */ + public Set includeFiles() { + return Collections.unmodifiableSet(includes); + } + + /** + * @return the paths of the TRAP files contained in this TRAP set + * (relative to the trap directory) + * + */ + public Set trapFiles() { + return Collections.unmodifiableSet(traps); + } + + /** + * @return the object names in this TRAP set + * + */ + public Set objectNames() { + return Collections.unmodifiableSet(objects); + } + + /** + * @return the object names in this TRAP set + * + */ + public Set inputObjectNames() { + return Collections.unmodifiableSet(inputObjects); + } + + /** + * Add a fully-qualified path to a source-file. + * + * @param source the path to the source file to add + */ + public void addSource(String source) { + sources.add(source); + } + + /** + * Add a fully-qualified path to an include-file. + * + * @param include the path to the include file to add + */ + public void addInclude(String include) { + includes.add(include); + } + + /** + * Add a path to a TRAP file (relative to the trap directory). + * + * @param trap the path to the trap file to add + * @return true if the path was not already present + */ + public boolean addTrap(String trap) { + return traps.add(trap); + } + + /** + * Check if this set contains a TRAP path + * + * @param trap the path to check + * @return true if this set contains the path + */ + public boolean containsTrap(String trap) { + return trap.contains(trap); + } + + /** + * Are the sources mentioned in this TRAP set disjoint from the given + * set of paths? + * + * @param paths the set of paths to check disjointness with + * @return true if and only if the paths are disjoint + */ + public boolean sourcesDisjointFrom(Set paths) { + for (String source : sources) + if (paths.contains(source)) + return false; + return true; + } + + /** + * Save this TRAP set to a .set file (or throw a ResourceError on failure) + * + * @param file the file in which to save this set + */ + @Override + public void save(Path file) { + super.save(file); + this.file = file; + } + + /* + * (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + appendHeaderString(sb, HEADER, LATEST_VERSION); + appendSet(sb, SOURCES, sources); + appendSet(sb, INCLUDES, includes); + appendSet(sb, OBJECTS, objects); + appendSet(sb, INPUT_OBJECTS, inputObjects); + appendSet(sb, TRAPS, traps); + return sb.toString(); + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/pathtransformers/NoopTransformer.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/pathtransformers/NoopTransformer.java new file mode 100644 index 00000000000..5ee27b81c84 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/pathtransformers/NoopTransformer.java @@ -0,0 +1,8 @@ +package com.semmle.util.trap.pathtransformers; + +public class NoopTransformer extends PathTransformer { + @Override + public String transform(String input) { + return input; + } +} \ No newline at end of file diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/pathtransformers/PathTransformer.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/pathtransformers/PathTransformer.java new file mode 100644 index 00000000000..434f882fae8 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/pathtransformers/PathTransformer.java @@ -0,0 +1,54 @@ +package com.semmle.util.trap.pathtransformers; + +import java.io.File; + +import com.semmle.util.files.FileUtil; +import com.semmle.util.process.Env; +import com.semmle.util.process.Env.Var; + +public abstract class PathTransformer { + public abstract String transform(String input); + + /** + * Convert a file to its path in the (code) database. Turns file paths into + * canonical, absolute, strings and normalises away Unix/Windows differences. + */ + public String fileAsDatabaseString(File file) { + String path = file.getPath(); + // For /!unknown-binary-location/... and /modules/... + // paths, on Windows the standard code wants to + // normalise them to e.g. C:/!unknown-binary-location/... + // which is particularly annoying for cross-platform test + // output. We therefore handle them specially here. + if (path.matches("^[/\\\\](!unknown-binary-location|modules)[/\\\\].*")) { + return path.replace('\\', '/'); + } + if (Boolean.valueOf(Env.systemEnv().get(Var.SEMMLE_PRESERVE_SYMLINKS))) + path = FileUtil.simplifyPath(file); + else + path = FileUtil.tryMakeCanonical(file).getPath(); + return transform(FileUtil.normalisePath(path)); + } + + /** + * Utility method for extractors: Canonicalise the given path as required + * for the current extraction. Unlike {@link FileUtil#tryMakeCanonical(File)}, + * this method is consistent with {@link #fileAsDatabaseString(File)}. + */ + public File canonicalFile(String path) { + return new File(fileAsDatabaseString(new File(path))); + } + + private static final PathTransformer DEFAULT_TRANSFORMER; + static { + String layout = Env.systemEnv().get(Var.SEMMLE_PATH_TRANSFORMER); + if (layout == null) + DEFAULT_TRANSFORMER = new NoopTransformer(); + else + DEFAULT_TRANSFORMER = new ProjectLayoutTransformer(new File(layout)); + } + + public static PathTransformer std() { + return DEFAULT_TRANSFORMER; + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/pathtransformers/ProjectLayoutTransformer.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/pathtransformers/ProjectLayoutTransformer.java new file mode 100644 index 00000000000..b1bd319e150 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/trap/pathtransformers/ProjectLayoutTransformer.java @@ -0,0 +1,37 @@ +package com.semmle.util.trap.pathtransformers; + +import java.io.File; + +import com.semmle.util.projectstructure.ProjectLayout; + +public class ProjectLayoutTransformer extends PathTransformer { + private final ProjectLayout layout; + + public ProjectLayoutTransformer(File file) { + layout = new ProjectLayout(file); + } + + @Override + public String transform(String input) { + if (isWindowsPath(input, 0)) { + String result = layout.artificialPath('/' + input); + if (result == null) { + return input; + } else if (isWindowsPath(result, 1) && result.charAt(0) == '/') { + return result.substring(1); + } else { + return result; + } + } else { + String result = layout.artificialPath(input); + return result != null ? result : input; + } + } + + private static boolean isWindowsPath(String s, int startAt) { + return s.length() >= (3 + startAt) && + s.charAt(startAt) != '/' && + s.charAt(startAt + 1) == ':' && + s.charAt(startAt + 2) == '/'; + } +} \ No newline at end of file diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/unicode/UTF8Util.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/unicode/UTF8Util.java new file mode 100644 index 00000000000..48e44166785 --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/unicode/UTF8Util.java @@ -0,0 +1,52 @@ +package com.semmle.util.unicode; + +public class UTF8Util { + /** + * Get the length (in Unicode code units, not code points) of the longest prefix of + * a string that can be UTF-8 encoded in no more than the given number of bytes. + * + *

+ * Unencodable characters (such as lone surrogate halves or low surrogates + * that do not follow a high surrogate) are treated as being encoded in + * three bytes. This is safe since on encoding they will be replaced by + * a replacement character, which in turn will take at most three bytes to + * encode. + *

+ * + * @param str string to encode + * @param maxEncodedLength maximum number of bytes for the encoded prefix + * @return length of the prefix + */ + public static int encodablePrefixLength(String str, int maxEncodedLength) { + // no character takes more than three bytes to encode + if (str.length() > maxEncodedLength / 3) { + int encodedLength = 0; + for (int i = 0; i < str.length(); ++i) { + int oldI = i; + char c = str.charAt(i); + if (c <= 0x7f) { + encodedLength += 1; + } else if (c <= 0x7ff) { + encodedLength += 2; + } else if (Character.isHighSurrogate(c)) { + // surrogate pairs take four bytes to encode + if (i+1 < str.length() && Character.isLowSurrogate(str.charAt(i+1))) { + encodedLength += 4; + ++i; + } else { + // lone high surrogate, assume length three + encodedLength += 3; + } + } else { + encodedLength += 3; + } + + if (encodedLength > maxEncodedLength) { + return oldI; + } + } + } + + return str.length(); + } +} diff --git a/java/kotlin-extractor2/src/main/java/com/semmle/util/zip/MultiMemberGZIPInputStream.java b/java/kotlin-extractor2/src/main/java/com/semmle/util/zip/MultiMemberGZIPInputStream.java new file mode 100644 index 00000000000..85a081bd41d --- /dev/null +++ b/java/kotlin-extractor2/src/main/java/com/semmle/util/zip/MultiMemberGZIPInputStream.java @@ -0,0 +1,71 @@ +package com.semmle.util.zip; + +import java.io.IOException; +import java.io.InputStream; +import java.io.PushbackInputStream; +import java.util.zip.GZIPInputStream; + +public class MultiMemberGZIPInputStream extends GZIPInputStream { + + public MultiMemberGZIPInputStream(InputStream in, int size) throws IOException { + // Wrap the stream in a PushbackInputStream... + super(new PushbackInputStream(in, size), size); + this.size = size; + } + + public MultiMemberGZIPInputStream(InputStream in) throws IOException { + // Wrap the stream in a PushbackInputStream... + super(new PushbackInputStream(in, 1024)); + this.size = -1; + } + + private MultiMemberGZIPInputStream child; + private int size; + private boolean eos; + + @Override + public int read(byte[] inputBuffer, int inputBufferOffset, int inputBufferLen) throws IOException { + if (eos) { + return -1; + } + else if (child != null) { + return child.read(inputBuffer, inputBufferOffset, inputBufferLen); + } + int charsRead = super.read(inputBuffer, inputBufferOffset, inputBufferLen); + if (charsRead == -1) { + // Push any remaining buffered data back onto the stream + // If the stream is then not empty, use it to construct + // a new instance of this class and delegate this and any + // future calls to it... + int n = inf.getRemaining() - 8; + if (n > 0) { + // More than 8 bytes remaining in deflater + // First 8 are gzip trailer. Add the rest to + // any un-read data... + ((PushbackInputStream) this.in).unread(buf, len - n, n); + } else { + // Nothing in the buffer. We need to know whether or not + // there is unread data available in the underlying stream + // since the base class will not handle an empty file. + // Read a byte to see if there is data and if so, + // push it back onto the stream... + byte[] b = new byte[1]; + int ret = in.read(b, 0, 1); + if (ret == -1) { + eos = true; + return -1; + } else { + ((PushbackInputStream) this.in).unread(b, 0, 1); + } + } + if(size == -1) + child = new MultiMemberGZIPInputStream(in); + else + child = new MultiMemberGZIPInputStream(in, size); + return child.read(inputBuffer, inputBufferOffset, inputBufferLen); + } else { + return charsRead; + } + } + +} diff --git a/java/kotlin-extractor2/src/main/kotlin/ExternalDeclExtractor.kt b/java/kotlin-extractor2/src/main/kotlin/ExternalDeclExtractor.kt new file mode 100644 index 00000000000..43cfad2f621 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/ExternalDeclExtractor.kt @@ -0,0 +1,204 @@ +package com.github.codeql + +import com.github.codeql.utils.isExternalFileClassMember +import com.semmle.extractor.java.OdasaOutput +import com.semmle.util.data.StringDigestor +import java.io.BufferedWriter +import java.io.File +import java.util.ArrayList +import java.util.HashSet +import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.ir.util.isFileClass +import org.jetbrains.kotlin.ir.util.packageFqName + +class ExternalDeclExtractor( + val logger: FileLogger, + val compression: Compression, + val invocationTrapFile: String, + val sourceFilePath: String, + val primitiveTypeMapping: PrimitiveTypeMapping, + val pluginContext: IrPluginContext, + val globalExtensionState: KotlinExtractorGlobalState, + val diagnosticTrapWriter: DiagnosticTrapWriter +) { + + val declBinaryNames = HashMap() + val externalDeclsDone = HashSet>() + val externalDeclWorkList = ArrayList>() + + val propertySignature = ";property" + val fieldSignature = ";field" + + val output = + OdasaOutput(false, compression, logger).also { + it.setCurrentSourceFile(File(sourceFilePath)) + } + + fun extractLater(d: IrDeclarationWithName, signature: String): Boolean { + if (d !is IrClass && !isExternalFileClassMember(d)) { + logger.errorElement( + "External declaration is neither a class, nor a top-level declaration", + d + ) + return false + } + val declBinaryName = declBinaryNames.getOrPut(d) { getIrElementBinaryName(d) } + val ret = externalDeclsDone.add(Pair(declBinaryName, signature)) + if (ret) externalDeclWorkList.add(Pair(d, signature)) + return ret + } + + fun extractLater(c: IrClass) = extractLater(c, "") + + fun writeStubTrapFile(e: IrElement, signature: String = "") { + extractElement(e, signature, true) { trapFileBW, _, _ -> + trapFileBW.write( + "// Trap file stubbed because this declaration was extracted from source in $sourceFilePath\n" + ) + trapFileBW.write("// Part of invocation $invocationTrapFile\n") + } + } + + private fun extractElement( + element: IrElement, + possiblyLongSignature: String, + fromSource: Boolean, + extractorFn: (BufferedWriter, String, OdasaOutput.TrapFileManager) -> Unit + ) { + // In order to avoid excessively long signatures which can lead to trap file names longer + // than the filesystem + // limit, we truncate and add a hash to preserve uniqueness if necessary. + val signature = + if (possiblyLongSignature.length > 100) { + possiblyLongSignature.substring(0, 92) + + "#" + + StringDigestor.digest(possiblyLongSignature).substring(0, 8) + } else { + possiblyLongSignature + } + output.getTrapLockerForDecl(element, signature, fromSource).useAC { locker -> + locker.trapFileManager.useAC { manager -> + val shortName = + when (element) { + is IrDeclarationWithName -> element.name.asString() + is IrFile -> element.name + else -> "(unknown name)" + } + if (manager == null) { + logger.info("Skipping extracting external decl $shortName") + } else { + val trapFile = manager.file + logger.info("Will write TRAP file $trapFile") + val trapTmpFile = + File.createTempFile( + "${trapFile.nameWithoutExtension}.", + ".${trapFile.extension}.tmp", + trapFile.parentFile + ) + logger.debug("Writing temporary TRAP file $trapTmpFile") + try { + compression.bufferedWriter(trapTmpFile).use { + extractorFn(it, signature, manager) + } + + if (!trapTmpFile.renameTo(trapFile)) { + logger.error("Failed to rename $trapTmpFile to $trapFile") + } + logger.info("Finished writing TRAP file $trapFile") + } catch (e: Exception) { + manager.setHasError() + logger.error( + "Failed to extract '$shortName'. Partial TRAP file location is $trapTmpFile", + e + ) + } + } + } + } + } + + fun extractExternalClasses() { + do { + val nextBatch = ArrayList(externalDeclWorkList) + externalDeclWorkList.clear() + nextBatch.forEach { workPair -> + val (irDecl, possiblyLongSignature) = workPair + extractElement(irDecl, possiblyLongSignature, false) { + trapFileBW, + signature, + manager -> + val binaryPath = getIrDeclarationBinaryPath(irDecl) + if (binaryPath == null) { + logger.errorElement("Unable to get binary path", irDecl) + } else { + // We want our comments to be the first thing in the file, + // so start off with a PlainTrapWriter + val tw = + PlainTrapWriter( + logger.loggerBase, + TrapLabelManager(), + trapFileBW, + diagnosticTrapWriter + ) + tw.writeComment( + "Generated by the CodeQL Kotlin extractor for external dependencies" + ) + tw.writeComment("Part of invocation $invocationTrapFile") + if (signature != possiblyLongSignature) { + tw.writeComment( + "Function signature abbreviated; full signature is: $possiblyLongSignature" + ) + } + // Now elevate to a SourceFileTrapWriter, and populate the + // file information if needed: + val ftw = tw.makeFileTrapWriter(binaryPath, true) + + val fileExtractor = + KotlinFileExtractor( + logger, + ftw, + null, + binaryPath, + manager, + this, + primitiveTypeMapping, + pluginContext, + KotlinFileExtractor.DeclarationStack(), + globalExtensionState + ) + + if (irDecl is IrClass) { + // Populate a location and compilation-unit package for the file. This + // is similar to + // the beginning of `KotlinFileExtractor.extractFileContents` but + // without an `IrFile` + // to start from. + val pkg = irDecl.packageFqName?.asString() ?: "" + val pkgId = fileExtractor.extractPackage(pkg) + ftw.writeHasLocation(ftw.fileId, ftw.getWholeFileLocation()) + ftw.writeCupackage(ftw.fileId, pkgId) + + fileExtractor.extractClassSource( + irDecl, + extractDeclarations = !irDecl.isFileClass, + extractStaticInitializer = false, + extractPrivateMembers = false, + extractFunctionBodies = false + ) + } else { + fileExtractor.extractDeclaration( + irDecl, + extractPrivateMembers = false, + extractFunctionBodies = false, + extractAnnotations = true + ) + } + } + } + } + } while (externalDeclWorkList.isNotEmpty()) + output.writeTrapSet() + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/KotlinExtractorCommandLineProcessor.kt b/java/kotlin-extractor2/src/main/kotlin/KotlinExtractorCommandLineProcessor.kt new file mode 100644 index 00000000000..72be6d89dc1 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/KotlinExtractorCommandLineProcessor.kt @@ -0,0 +1,100 @@ +package com.github.codeql + +import org.jetbrains.kotlin.compiler.plugin.AbstractCliOption +import org.jetbrains.kotlin.compiler.plugin.CliOption +import org.jetbrains.kotlin.compiler.plugin.CommandLineProcessor +import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi +import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.config.CompilerConfigurationKey + +@OptIn(ExperimentalCompilerApi::class) +class KotlinExtractorCommandLineProcessor : CommandLineProcessor { + override val pluginId = "kotlin-extractor" + + override val pluginOptions = + listOf( + CliOption( + optionName = OPTION_INVOCATION_TRAP_FILE, + valueDescription = "Invocation TRAP file", + description = "Extractor will append invocation-related TRAP to this file", + required = true, + allowMultipleOccurrences = false + ), + CliOption( + optionName = OPTION_CHECK_TRAP_IDENTICAL, + valueDescription = "Check whether different invocations produce identical TRAP", + description = "Check whether different invocations produce identical TRAP", + required = false, + allowMultipleOccurrences = false + ), + CliOption( + optionName = OPTION_COMPILATION_STARTTIME, + valueDescription = "The start time of the compilation as a Unix timestamp", + description = "The start time of the compilation as a Unix timestamp", + required = false, + allowMultipleOccurrences = false + ), + CliOption( + optionName = OPTION_EXIT_AFTER_EXTRACTION, + valueDescription = + "Specify whether to call exitProcess after the extraction has completed", + description = + "Specify whether to call exitProcess after the extraction has completed", + required = false, + allowMultipleOccurrences = false + ) + ) + + override fun processOption( + option: AbstractCliOption, + value: String, + configuration: CompilerConfiguration + ) = + when (option.optionName) { + OPTION_INVOCATION_TRAP_FILE -> configuration.put(KEY_INVOCATION_TRAP_FILE, value) + OPTION_CHECK_TRAP_IDENTICAL -> + processBooleanOption( + value, + OPTION_CHECK_TRAP_IDENTICAL, + KEY_CHECK_TRAP_IDENTICAL, + configuration + ) + OPTION_EXIT_AFTER_EXTRACTION -> + processBooleanOption( + value, + OPTION_EXIT_AFTER_EXTRACTION, + KEY_EXIT_AFTER_EXTRACTION, + configuration + ) + OPTION_COMPILATION_STARTTIME -> + when (val v = value.toLongOrNull()) { + is Long -> configuration.put(KEY_COMPILATION_STARTTIME, v) + else -> + error( + "kotlin extractor: Bad argument $value for $OPTION_COMPILATION_STARTTIME" + ) + } + else -> error("kotlin extractor: Bad option: ${option.optionName}") + } + + private fun processBooleanOption( + value: String, + optionName: String, + configKey: CompilerConfigurationKey, + configuration: CompilerConfiguration + ) = + when (value) { + "true" -> configuration.put(configKey, true) + "false" -> configuration.put(configKey, false) + else -> error("kotlin extractor: Bad argument $value for $optionName") + } +} + +private val OPTION_INVOCATION_TRAP_FILE = "invocationTrapFile" +val KEY_INVOCATION_TRAP_FILE = CompilerConfigurationKey(OPTION_INVOCATION_TRAP_FILE) +private val OPTION_CHECK_TRAP_IDENTICAL = "checkTrapIdentical" +val KEY_CHECK_TRAP_IDENTICAL = CompilerConfigurationKey(OPTION_CHECK_TRAP_IDENTICAL) +private val OPTION_COMPILATION_STARTTIME = "compilationStartTime" +val KEY_COMPILATION_STARTTIME = CompilerConfigurationKey(OPTION_COMPILATION_STARTTIME) +private val OPTION_EXIT_AFTER_EXTRACTION = "exitAfterExtraction" +val KEY_EXIT_AFTER_EXTRACTION = CompilerConfigurationKey(OPTION_EXIT_AFTER_EXTRACTION) diff --git a/java/kotlin-extractor2/src/main/kotlin/KotlinExtractorComponentRegistrar.kt b/java/kotlin-extractor2/src/main/kotlin/KotlinExtractorComponentRegistrar.kt new file mode 100644 index 00000000000..7fd6416dcd6 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/KotlinExtractorComponentRegistrar.kt @@ -0,0 +1,29 @@ +// For ComponentRegistrar +@file:Suppress("DEPRECATION") + +package com.github.codeql + +import com.intellij.mock.MockProject +import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension +import org.jetbrains.kotlin.config.CompilerConfiguration + +class KotlinExtractorComponentRegistrar : Kotlin2ComponentRegistrar() { + override fun registerProjectComponents( + project: MockProject, + configuration: CompilerConfiguration + ) { + val invocationTrapFile = configuration[KEY_INVOCATION_TRAP_FILE] + if (invocationTrapFile == null) { + throw Exception("Required argument for TRAP invocation file not given") + } + IrGenerationExtension.registerExtension( + project, + KotlinExtractorExtension( + invocationTrapFile, + configuration[KEY_CHECK_TRAP_IDENTICAL] ?: false, + configuration[KEY_COMPILATION_STARTTIME], + configuration[KEY_EXIT_AFTER_EXTRACTION] ?: false + ) + ) + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/KotlinExtractorExtension.kt b/java/kotlin-extractor2/src/main/kotlin/KotlinExtractorExtension.kt new file mode 100644 index 00000000000..3850d690a61 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/KotlinExtractorExtension.kt @@ -0,0 +1,584 @@ +package com.github.codeql + +import com.github.codeql.utils.versions.usesK2 +import com.semmle.util.files.FileUtil +import com.semmle.util.trap.pathtransformers.PathTransformer +import java.io.BufferedInputStream +import java.io.BufferedOutputStream +import java.io.BufferedReader +import java.io.BufferedWriter +import java.io.File +import java.io.FileInputStream +import java.io.FileOutputStream +import java.io.InputStreamReader +import java.io.OutputStreamWriter +import java.lang.management.* +import java.nio.file.Files +import java.nio.file.Paths +import java.util.zip.GZIPInputStream +import java.util.zip.GZIPOutputStream +import kotlin.system.exitProcess +import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension +import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext +import org.jetbrains.kotlin.config.KotlinCompilerVersion +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.ir.util.* + +/* + * KotlinExtractorExtension is the main entry point of the CodeQL Kotlin + * extractor. When the jar is used as a kotlinc plugin, kotlinc will + * call the `generate` method. + */ +class KotlinExtractorExtension( + // The filepath for the invocation TRAP file. + // This TRAP file is for this invocation of the extractor as a + // whole, not tied to a particular source file. It contains + // information about which files this invocation compiled, and + // any warnings or errors encountered during the invocation. + private val invocationTrapFile: String, + // By default, if a TRAP file we want to generate for a source + // file already exists, then we will do nothing. If this is set, + // then we will instead generate the TRAP file, and give a + // warning if we would generate different TRAP to that which + // already exists. + private val checkTrapIdentical: Boolean, + // If non-null, then this is the number of milliseconds since + // midnight, January 1, 1970 UTC (as returned by Java's + // `System.currentTimeMillis()`. If this is given, then it is used + // to record the time taken to compile the source code, which is + // presumed to be the difference between this time and the time + // that this plugin is invoked. + private val compilationStartTime: Long?, + // Under normal conditions, the extractor runs during a build of + // the project, and kotlinc continues after the plugin has finished. + // If the plugin is being used independently of a build, then this + // can be set to true to make the plugin terminate the kotlinc + // invocation when it has finished. This means that kotlinc will not + // write any `.class` files etc. + private val exitAfterExtraction: Boolean +) : IrGenerationExtension { + + // This is the main entry point to the extractor. + // It will be called by kotlinc with the IR for the files being + // compiled in `moduleFragment`, and `pluginContext` providing + // various utility functions. + override fun generate(moduleFragment: IrModuleFragment, pluginContext: IrPluginContext) { + try { + runExtractor(moduleFragment, pluginContext) + // We catch Throwable rather than Exception, as we want to + // continue trying to extract everything else even if we get a + // stack overflow or an assertion failure in one file. + } catch (e: Throwable) { + // If we get an exception at the top level, then something's + // gone very wrong. Don't try to be too fancy, but try to + // log a simple message. + val msg = "[ERROR] CodeQL Kotlin extractor: Top-level exception." + // First, if we can find our log directory, then let's try + // making a log file there: + val extractorLogDir = System.getenv("CODEQL_EXTRACTOR_JAVA_LOG_DIR") + if (extractorLogDir != null && extractorLogDir != "") { + // We use a slightly different filename pattern compared + // to normal logs. Just the existence of a `-top` log is + // a sign that something's gone very wrong. + val logFile = + File.createTempFile("kotlin-extractor-top.", ".log", File(extractorLogDir)) + logFile.writeText(msg) + // Now we've got that out, let's see if we can append a stack trace too + logFile.appendText(e.stackTraceToString()) + } else { + // We don't have much choice here except to print to + // stderr and hope for the best. + System.err.println(msg) + e.printStackTrace(System.err) + } + } + if (exitAfterExtraction) { + exitProcess(0) + } + } + + private fun runExtractor(moduleFragment: IrModuleFragment, pluginContext: IrPluginContext) { + val startTimeMs = System.currentTimeMillis() + val usesK2 = usesK2(pluginContext) + // This default should be kept in sync with + // com.semmle.extractor.java.interceptors.KotlinInterceptor.initializeExtractionContext + val trapDir = + File( + System.getenv("CODEQL_EXTRACTOR_JAVA_TRAP_DIR").takeUnless { it.isNullOrEmpty() } + ?: "kotlin-extractor/trap" + ) + // The invocation TRAP file will already have been started + // before the plugin is run, so we always use no compression + // and we open it in append mode. + FileOutputStream(File(invocationTrapFile), true).bufferedWriter().use { invocationTrapFileBW + -> + val invocationExtractionProblems = ExtractionProblems() + val lm = TrapLabelManager() + val logCounter = LogCounter() + val loggerBase = LoggerBase(logCounter) + val tw = DiagnosticTrapWriter(loggerBase, lm, invocationTrapFileBW) + // The interceptor has already defined #compilation = * + val compilation: Label = StringLabel("compilation") + tw.writeCompilation_started(compilation) + tw.writeCompilation_info( + compilation, + "Kotlin Compiler Version", + KotlinCompilerVersion.getVersion() ?: "" + ) + val extractor_name = + this::class.java.getResource("extractor.name")?.readText() ?: "" + tw.writeCompilation_info(compilation, "Kotlin Extractor Name", extractor_name) + tw.writeCompilation_info(compilation, "Uses Kotlin 2", usesK2.toString()) + if (compilationStartTime != null) { + tw.writeCompilation_compiler_times( + compilation, + -1.0, + (System.currentTimeMillis() - compilationStartTime) / 1000.0 + ) + } + tw.flush() + val logger = Logger(loggerBase, tw) + logger.info("Extraction started") + logger.flush() + logger.info("Extraction for invocation TRAP file $invocationTrapFile") + logger.flush() + logger.info("Kotlin version ${KotlinCompilerVersion.getVersion()}") + logger.flush() + logPeakMemoryUsage(logger, "before extractor") + if (System.getenv("CODEQL_EXTRACTOR_JAVA_KOTLIN_DUMP") == "true") { + logger.info("moduleFragment:\n" + moduleFragment.dump()) + } + val compression = getCompression(logger) + + val primitiveTypeMapping = PrimitiveTypeMapping(logger, pluginContext) + // FIXME: FileUtil expects a static global logger + // which should be provided by SLF4J's factory facility. For now we set it here. + FileUtil.logger = logger + val srcDir = + File( + System.getenv("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR").takeUnless { + it.isNullOrEmpty() + } ?: "kotlin-extractor/src" + ) + srcDir.mkdirs() + val globalExtensionState = KotlinExtractorGlobalState() + moduleFragment.files.mapIndexed { index: Int, file: IrFile -> + val fileExtractionProblems = FileExtractionProblems(invocationExtractionProblems) + val fileTrapWriter = tw.makeSourceFileTrapWriter(file, true) + loggerBase.setFileNumber(index) + fileTrapWriter.writeCompilation_compiling_files( + compilation, + index, + fileTrapWriter.fileId + ) + doFile( + compression, + fileExtractionProblems, + invocationTrapFile, + fileTrapWriter, + checkTrapIdentical, + loggerBase, + trapDir, + srcDir, + file, + primitiveTypeMapping, + pluginContext, + globalExtensionState + ) + fileTrapWriter.writeCompilation_compiling_files_completed( + compilation, + index, + fileExtractionProblems.extractionResult() + ) + } + loggerBase.printLimitedDiagnosticCounts(tw) + logPeakMemoryUsage(logger, "after extractor") + logger.info("Extraction completed") + logger.flush() + val compilationTimeMs = System.currentTimeMillis() - startTimeMs + tw.writeCompilation_finished( + compilation, + -1.0, + compilationTimeMs.toDouble() / 1000, + invocationExtractionProblems.extractionResult() + ) + tw.flush() + loggerBase.close() + } + } + + private fun getCompression(logger: Logger): Compression { + val compression_env_var = "CODEQL_EXTRACTOR_JAVA_OPTION_TRAP_COMPRESSION" + val compression_option = System.getenv(compression_env_var) + val defaultCompression = Compression.GZIP + if (compression_option == null) { + return defaultCompression + } else { + try { + val compression_option_upper = compression_option.uppercase() + if (compression_option_upper == "BROTLI") { + logger.warn( + "Kotlin extractor doesn't support Brotli compression. Using GZip instead." + ) + return Compression.GZIP + } else { + return Compression.valueOf(compression_option_upper) + } + } catch (e: IllegalArgumentException) { + logger.warn( + "Unsupported compression type (\$$compression_env_var) \"$compression_option\". Supported values are ${Compression.values().joinToString()}." + ) + return defaultCompression + } + } + } + + private fun logPeakMemoryUsage(logger: Logger, time: String) { + logger.info("Peak memory: Usage $time") + + val beans = ManagementFactory.getMemoryPoolMXBeans() + var heap: Long = 0 + var nonheap: Long = 0 + for (bean in beans) { + val peak = bean.getPeakUsage().getUsed() + val kind = + when (bean.getType()) { + MemoryType.HEAP -> { + heap += peak + "heap" + } + MemoryType.NON_HEAP -> { + nonheap += peak + "non-heap" + } + else -> "unknown" + } + logger.info("Peak memory: * Peak for $kind bean ${bean.getName()} is $peak") + } + logger.info("Peak memory: * Total heap peak: $heap") + logger.info("Peak memory: * Total non-heap peak: $nonheap") + } +} + +class KotlinExtractorGlobalState { + // These three record mappings of classes, functions and fields that should be replaced wherever + // they are found. + // As of now these are only used to fix IR generated by the Gradle Android Extensions plugin, + // hence e.g. IrProperty + // doesn't have a map as that plugin doesn't generate them. If and when these are used more + // widely additional maps + // should be added here. + val syntheticToRealClassMap = HashMap() + val syntheticToRealFunctionMap = HashMap() + val syntheticToRealFieldMap = HashMap() + val syntheticRepeatableAnnotationContainers = HashMap() +} + +/* +The `ExtractionProblems` class is used to record whether this invocation +had any problems. It distinguish 2 kinds of problem: +* Recoverable problems: e.g. if we check something that we expect to be + non-null and find that it is null. +* Non-recoverable problems: if we catch an exception. +*/ +open class ExtractionProblems { + private var recoverableProblem = false + private var nonRecoverableProblem = false + + open fun setRecoverableProblem() { + recoverableProblem = true + } + + open fun setNonRecoverableProblem() { + nonRecoverableProblem = true + } + + fun extractionResult(): Int { + if (nonRecoverableProblem) { + return 2 + } else if (recoverableProblem) { + return 1 + } else { + return 0 + } + } +} + +/* +The `FileExtractionProblems` is analogous to `ExtractionProblems`, +except it records whether there were any problems while extracting a +particular source file. +*/ +class FileExtractionProblems(val invocationExtractionProblems: ExtractionProblems) : + ExtractionProblems() { + override fun setRecoverableProblem() { + super.setRecoverableProblem() + invocationExtractionProblems.setRecoverableProblem() + } + + override fun setNonRecoverableProblem() { + super.setNonRecoverableProblem() + invocationExtractionProblems.setNonRecoverableProblem() + } +} + +/* +This function determines whether 2 TRAP files should be considered to be +equivalent. It returns `true` iff all of their non-comment lines are +identical. +*/ +private fun equivalentTrap(r1: BufferedReader, r2: BufferedReader): Boolean { + r1.use { br1 -> + r2.use { br2 -> + while (true) { + val l1 = br1.readLine() + val l2 = br2.readLine() + if (l1 == null && l2 == null) { + return true + } else if (l1 == null || l2 == null) { + return false + } else if (l1 != l2) { + if (!l1.startsWith("//") || !l2.startsWith("//")) { + return false + } + } + } + } + } +} + +private fun doFile( + compression: Compression, + fileExtractionProblems: FileExtractionProblems, + invocationTrapFile: String, + fileTrapWriter: FileTrapWriter, + checkTrapIdentical: Boolean, + loggerBase: LoggerBase, + dbTrapDir: File, + dbSrcDir: File, + srcFile: IrFile, + primitiveTypeMapping: PrimitiveTypeMapping, + pluginContext: IrPluginContext, + globalExtensionState: KotlinExtractorGlobalState +) { + val srcFilePath = srcFile.path + val logger = FileLogger(loggerBase, fileTrapWriter) + logger.info("Extracting file $srcFilePath") + logger.flush() + + val context = logger.loggerBase.extractorContextStack + if (!context.empty()) { + logger.warn("Extractor context was not empty. It thought:") + context.clear() + } + + val srcFileRelativePath = PathTransformer.std().fileAsDatabaseString(File(srcFilePath)) + + val dbSrcFilePath = FileUtil.appendAbsolutePath(dbSrcDir, srcFileRelativePath).toPath() + val dbSrcDirPath = dbSrcFilePath.parent + Files.createDirectories(dbSrcDirPath) + val srcTmpFile = + File.createTempFile( + dbSrcFilePath.fileName.toString() + ".", + ".src.tmp", + dbSrcDirPath.toFile() + ) + srcTmpFile.outputStream().use { Files.copy(Paths.get(srcFilePath), it) } + srcTmpFile.renameTo(dbSrcFilePath.toFile()) + + val trapFileName = FileUtil.appendAbsolutePath(dbTrapDir, "$srcFileRelativePath.trap").getAbsolutePath() + val trapFileWriter = getTrapFileWriter(compression, logger, trapFileName) + + if (checkTrapIdentical || !trapFileWriter.exists()) { + trapFileWriter.makeParentDirectory() + + try { + trapFileWriter.getTempWriter().use { trapFileBW -> + // We want our comments to be the first thing in the file, + // so start off with a mere TrapWriter + val tw = + PlainTrapWriter( + loggerBase, + TrapLabelManager(), + trapFileBW, + fileTrapWriter.getDiagnosticTrapWriter() + ) + tw.writeComment("Generated by the CodeQL Kotlin extractor for kotlin source code") + tw.writeComment("Part of invocation $invocationTrapFile") + // Now elevate to a SourceFileTrapWriter, and populate the + // file information + val sftw = tw.makeSourceFileTrapWriter(srcFile, true) + val externalDeclExtractor = + ExternalDeclExtractor( + logger, + compression, + invocationTrapFile, + srcFilePath, + primitiveTypeMapping, + pluginContext, + globalExtensionState, + fileTrapWriter.getDiagnosticTrapWriter() + ) + val linesOfCode = LinesOfCode(logger, sftw, srcFile) + val fileExtractor = + KotlinFileExtractor( + logger, + sftw, + linesOfCode, + srcFilePath, + null, + externalDeclExtractor, + primitiveTypeMapping, + pluginContext, + KotlinFileExtractor.DeclarationStack(), + globalExtensionState + ) + + fileExtractor.extractFileContents(srcFile, sftw.fileId) + externalDeclExtractor.extractExternalClasses() + } + + if (checkTrapIdentical && trapFileWriter.exists()) { + if ( + equivalentTrap(trapFileWriter.getTempReader(), trapFileWriter.getRealReader()) + ) { + trapFileWriter.deleteTemp() + } else { + trapFileWriter.renameTempToDifferent() + } + } else { + trapFileWriter.renameTempToReal() + } + // We catch Throwable rather than Exception, as we want to + // continue trying to extract everything else even if we get a + // stack overflow or an assertion failure in one file. + } catch (e: Throwable) { + logger.error("Failed to extract '$srcFilePath'. " + trapFileWriter.debugInfo(), e) + context.clear() + fileExtractionProblems.setNonRecoverableProblem() + } + } +} + +enum class Compression(val extension: String) { + NONE("") { + override fun bufferedWriter(file: File): BufferedWriter { + return file.bufferedWriter() + } + }, + GZIP(".gz") { + override fun bufferedWriter(file: File): BufferedWriter { + return GZIPOutputStream(file.outputStream()).bufferedWriter() + } + }; + + abstract fun bufferedWriter(file: File): BufferedWriter +} + +private fun getTrapFileWriter( + compression: Compression, + logger: FileLogger, + trapFileName: String +): TrapFileWriter { + return when (compression) { + Compression.NONE -> NonCompressedTrapFileWriter(logger, trapFileName) + Compression.GZIP -> GZipCompressedTrapFileWriter(logger, trapFileName) + } +} + +private abstract class TrapFileWriter( + val logger: FileLogger, + trapName: String, + val extension: String +) { + private val realFile = File(trapName + extension) + private val parentDir = realFile.parentFile + lateinit private var tempFile: File + + fun debugInfo(): String { + if (this::tempFile.isInitialized) { + return "Partial TRAP file location is $tempFile" + } else { + return "Temporary file not yet created." + } + } + + fun makeParentDirectory() { + parentDir.mkdirs() + } + + fun exists(): Boolean { + return realFile.exists() + } + + abstract protected fun getReader(file: File): BufferedReader + + abstract protected fun getWriter(file: File): BufferedWriter + + fun getRealReader(): BufferedReader { + return getReader(realFile) + } + + fun getTempReader(): BufferedReader { + return getReader(tempFile) + } + + fun getTempWriter(): BufferedWriter { + logger.info("Will write TRAP file $realFile") + if (this::tempFile.isInitialized) { + logger.error("Temp writer reinitialized for $realFile") + } + tempFile = File.createTempFile(realFile.getName() + ".", ".trap.tmp" + extension, parentDir) + logger.debug("Writing temporary TRAP file $tempFile") + return getWriter(tempFile) + } + + fun deleteTemp() { + if (!tempFile.delete()) { + logger.warn("Failed to delete $tempFile") + } + } + + fun renameTempToDifferent() { + val trapDifferentFile = + File.createTempFile(realFile.getName() + ".", ".trap.different" + extension, parentDir) + if (tempFile.renameTo(trapDifferentFile)) { + logger.warn("TRAP difference: $realFile vs $trapDifferentFile") + } else { + logger.warn("Failed to rename $tempFile to $realFile") + } + } + + fun renameTempToReal() { + if (!tempFile.renameTo(realFile)) { + logger.warn("Failed to rename $tempFile to $realFile") + } + logger.info("Finished writing TRAP file $realFile") + } +} + +private class NonCompressedTrapFileWriter(logger: FileLogger, trapName: String) : + TrapFileWriter(logger, trapName, "") { + override protected fun getReader(file: File): BufferedReader { + return file.bufferedReader() + } + + override protected fun getWriter(file: File): BufferedWriter { + return file.bufferedWriter() + } +} + +private class GZipCompressedTrapFileWriter(logger: FileLogger, trapName: String) : + TrapFileWriter(logger, trapName, ".gz") { + override protected fun getReader(file: File): BufferedReader { + return BufferedReader( + InputStreamReader(GZIPInputStream(BufferedInputStream(FileInputStream(file)))) + ) + } + + override protected fun getWriter(file: File): BufferedWriter { + return BufferedWriter( + OutputStreamWriter(GZIPOutputStream(BufferedOutputStream(FileOutputStream(file)))) + ) + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/KotlinFileExtractor.kt b/java/kotlin-extractor2/src/main/kotlin/KotlinFileExtractor.kt new file mode 100644 index 00000000000..47288394904 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/KotlinFileExtractor.kt @@ -0,0 +1,9210 @@ +package com.github.codeql + +import com.github.codeql.comments.CommentExtractorLighterAST +import com.github.codeql.comments.CommentExtractorPSI +import com.github.codeql.utils.* +import com.github.codeql.utils.versions.* +import com.semmle.extractor.java.OdasaOutput +import java.io.Closeable +import java.util.* +import kotlin.collections.ArrayList +import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext +import org.jetbrains.kotlin.backend.common.pop +import org.jetbrains.kotlin.builtins.functions.BuiltInFunctionArity +import org.jetbrains.kotlin.config.JvmAnalysisFlags +import org.jetbrains.kotlin.descriptors.* +import org.jetbrains.kotlin.descriptors.java.JavaVisibilities +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.IrStatement +import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI +import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET +import org.jetbrains.kotlin.ir.backend.js.utils.realOverrideTarget +import org.jetbrains.kotlin.ir.builders.declarations.* +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.ir.declarations.lazy.IrLazyFunction +import org.jetbrains.kotlin.ir.expressions.* +import org.jetbrains.kotlin.ir.expressions.impl.* +import org.jetbrains.kotlin.ir.symbols.* +import org.jetbrains.kotlin.ir.types.* +import org.jetbrains.kotlin.ir.types.impl.makeTypeProjection +import org.jetbrains.kotlin.ir.util.companionObject +import org.jetbrains.kotlin.ir.util.constructors +import org.jetbrains.kotlin.ir.util.fqNameWhenAvailable +import org.jetbrains.kotlin.ir.util.hasAnnotation +import org.jetbrains.kotlin.ir.util.hasInterfaceParent +import org.jetbrains.kotlin.ir.util.isAnnotationClass +import org.jetbrains.kotlin.ir.util.isAnonymousObject +import org.jetbrains.kotlin.ir.util.isFakeOverride +import org.jetbrains.kotlin.ir.util.isFunctionOrKFunction +import org.jetbrains.kotlin.ir.util.isInterface +import org.jetbrains.kotlin.ir.util.isLocal +import org.jetbrains.kotlin.ir.util.isNonCompanionObject +import org.jetbrains.kotlin.ir.util.isObject +import org.jetbrains.kotlin.ir.util.isSuspend +import org.jetbrains.kotlin.ir.util.isSuspendFunctionOrKFunction +import org.jetbrains.kotlin.ir.util.isVararg +import org.jetbrains.kotlin.ir.util.kotlinFqName +import org.jetbrains.kotlin.ir.util.packageFqName +import org.jetbrains.kotlin.ir.util.parentAsClass +import org.jetbrains.kotlin.ir.util.parentClassOrNull +import org.jetbrains.kotlin.ir.util.parents +import org.jetbrains.kotlin.ir.util.primaryConstructor +import org.jetbrains.kotlin.ir.util.render +import org.jetbrains.kotlin.ir.util.target +import org.jetbrains.kotlin.load.java.JvmAnnotationNames +import org.jetbrains.kotlin.load.java.NOT_NULL_ANNOTATIONS +import org.jetbrains.kotlin.load.java.NULLABLE_ANNOTATIONS +import org.jetbrains.kotlin.load.java.sources.JavaSourceElement +import org.jetbrains.kotlin.load.java.structure.JavaAnnotation +import org.jetbrains.kotlin.load.java.structure.JavaClass +import org.jetbrains.kotlin.load.java.structure.JavaConstructor +import org.jetbrains.kotlin.load.java.structure.JavaMethod +import org.jetbrains.kotlin.load.java.structure.JavaTypeParameter +import org.jetbrains.kotlin.load.java.structure.JavaTypeParameterListOwner +import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.types.Variance +import org.jetbrains.kotlin.util.OperatorNameConventions +import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull + +open class KotlinFileExtractor( + override val logger: FileLogger, + override val tw: FileTrapWriter, + val linesOfCode: LinesOfCode?, + val filePath: String, + dependencyCollector: OdasaOutput.TrapFileManager?, + externalClassExtractor: ExternalDeclExtractor, + primitiveTypeMapping: PrimitiveTypeMapping, + pluginContext: IrPluginContext, + val declarationStack: DeclarationStack, + globalExtensionState: KotlinExtractorGlobalState, +) : + KotlinUsesExtractor( + logger, + tw, + dependencyCollector, + externalClassExtractor, + primitiveTypeMapping, + pluginContext, + globalExtensionState + ) { + + val usesK2 = usesK2(pluginContext) + val metaAnnotationSupport = MetaAnnotationSupport(logger, pluginContext, this) + + private inline fun with(kind: String, element: IrElement, f: () -> T): T { + val name = + when (element) { + is IrFile -> element.name + is IrDeclarationWithName -> element.name.asString() + else -> "" + } + val loc = tw.getLocationString(element) + val context = logger.loggerBase.extractorContextStack + context.push(ExtractorContext(kind, element, name, loc)) + try { + val depth = context.size + val depthDescription = "${"-".repeat(depth)} (${depth.toString()})" + logger.trace("$depthDescription: Starting a $kind ($name) at $loc") + val result = f() + logger.trace("$depthDescription: Finished a $kind ($name) at $loc") + return result + } catch (exception: Exception) { + throw Exception("While extracting a $kind ($name) at $loc", exception) + } finally { + context.pop() + } + } + + fun extractFileContents(file: IrFile, id: Label) { + with("file", file) { + val locId = tw.getWholeFileLocation() + val pkg = file.packageFqName.asString() + val pkgId = extractPackage(pkg) + tw.writeHasLocation(id, locId) + tw.writeCupackage(id, pkgId) + + val exceptionOnFile = + System.getenv("CODEQL_KOTLIN_INTERNAL_EXCEPTION_WHILE_EXTRACTING_FILE") + if (exceptionOnFile != null) { + if (exceptionOnFile.lowercase() == file.name.lowercase()) { + throw Exception("Internal testing exception") + } + } + + file.declarations.forEach { + extractDeclaration( + it, + extractPrivateMembers = true, + extractFunctionBodies = true, + extractAnnotations = true + ) + if (it is IrProperty || it is IrField || it is IrFunction) { + externalClassExtractor.writeStubTrapFile(it, getTrapFileSignature(it)) + } + } + extractStaticInitializer(file, { extractFileClass(file) }) + val psiCommentsExtracted = CommentExtractorPSI(this, file, tw.fileId).extract() + val lighterAstCommentsExtracted = + CommentExtractorLighterAST(this, file, tw.fileId).extract() + if (psiCommentsExtracted == lighterAstCommentsExtracted) { + if (psiCommentsExtracted) { + logger.warnElement( + "Found both PSI and LighterAST comments in ${file.path}.", + file + ) + } else { + logger.warnElement("Comments could not be processed in ${file.path}.", file) + } + } + + if (!declarationStack.isEmpty()) { + logger.errorElement( + "Declaration stack is not empty after processing the file", + file + ) + } + + linesOfCode?.linesOfCodeInFile(id) + + externalClassExtractor.writeStubTrapFile(file) + } + } + + private fun javaBinaryDeclaresMethod(c: IrClass, name: String) = + ((c.source as? JavaSourceElement)?.javaElement as? BinaryJavaClass)?.methods?.any { + it.name.asString() == name + } + + private fun isJavaBinaryDeclaration(f: IrFunction) = + f.parentClassOrNull?.let { javaBinaryDeclaresMethod(it, f.name.asString()) } ?: false + + private fun isJavaBinaryObjectMethodRedeclaration(d: IrDeclaration) = + when (d) { + is IrFunction -> + when (d.name.asString()) { + "toString" -> d.valueParameters.isEmpty() + "hashCode" -> d.valueParameters.isEmpty() + "equals" -> d.valueParameters.singleOrNull()?.type?.isNullableAny() ?: false + else -> false + } && isJavaBinaryDeclaration(d) + else -> false + } + + private fun FunctionDescriptor.tryIsHiddenToOvercomeSignatureClash(d: IrFunction): Boolean { + try { + return this.isHiddenToOvercomeSignatureClash + } catch (e: NotImplementedError) { + // `org.jetbrains.kotlin.ir.descriptors.IrBasedClassConstructorDescriptor.isHiddenToOvercomeSignatureClash` throws the exception + // TODO: We need a replacement for this for Kotlin 2 + if (!usesK2) { + logger.warnElement("Couldn't query if element is fake, deciding it's not.", d, e) + } + return false + } + } + + @OptIn(ObsoleteDescriptorBasedAPI::class) + fun isFake(d: IrDeclarationWithVisibility): Boolean { + val hasFakeVisibility = + d.visibility.let { + it is DelegatedDescriptorVisibility && it.delegate == Visibilities.InvisibleFake + } || d.isFakeOverride + if (hasFakeVisibility && !isJavaBinaryObjectMethodRedeclaration(d)) return true + return (d as? IrFunction)?.descriptor?.tryIsHiddenToOvercomeSignatureClash(d) == true + } + + private fun shouldExtractDecl(declaration: IrDeclaration, extractPrivateMembers: Boolean) = + extractPrivateMembers || !isPrivate(declaration) + + fun extractDeclaration( + declaration: IrDeclaration, + extractPrivateMembers: Boolean, + extractFunctionBodies: Boolean, + extractAnnotations: Boolean + ) { + with("declaration", declaration) { + if (!shouldExtractDecl(declaration, extractPrivateMembers)) return + when (declaration) { + is IrClass -> { + if (isExternalDeclaration(declaration)) { + extractExternalClassLater(declaration) + } else { + extractClassSource( + declaration, + extractDeclarations = true, + extractStaticInitializer = true, + extractPrivateMembers = extractPrivateMembers, + extractFunctionBodies = extractFunctionBodies + ) + } + } + is IrFunction -> { + val parentId = useDeclarationParentOf(declaration, false)?.cast() + if (parentId != null) { + extractFunction( + declaration, + parentId, + extractBody = extractFunctionBodies, + extractMethodAndParameterTypeAccesses = extractFunctionBodies, + extractAnnotations = extractAnnotations, + null, + listOf() + ) + } + Unit + } + is IrAnonymousInitializer -> { + // Leaving this intentionally empty. init blocks are extracted during class + // extraction. + } + is IrProperty -> { + val parentId = useDeclarationParentOf(declaration, false)?.cast() + if (parentId != null) { + extractProperty( + declaration, + parentId, + extractBackingField = true, + extractFunctionBodies = extractFunctionBodies, + extractPrivateMembers = extractPrivateMembers, + extractAnnotations = extractAnnotations, + null, + listOf() + ) + } + Unit + } + is IrEnumEntry -> { + val parentId = useDeclarationParentOf(declaration, false)?.cast() + if (parentId != null) { + extractEnumEntry( + declaration, + parentId, + extractPrivateMembers, + extractFunctionBodies + ) + } + Unit + } + is IrField -> { + val parentId = useDeclarationParentOf(declaration, false)?.cast() + if (parentId != null) { + // For consistency with the Java extractor, enum entries get type accesses + // only if we're extracting from .kt source (i.e., when + // `extractFunctionBodies` is set) + extractField( + declaration, + parentId, + extractAnnotationEnumTypeAccesses = extractFunctionBodies + ) + } + Unit + } + is IrTypeAlias -> extractTypeAlias(declaration) + else -> + logger.errorElement( + "Unrecognised IrDeclaration: " + declaration.javaClass, + declaration + ) + } + } + } + + private fun extractTypeParameter( + tp: IrTypeParameter, + apparentIndex: Int, + javaTypeParameter: JavaTypeParameter? + ): Label? { + with("type parameter", tp) { + val parentId = getTypeParameterParentLabel(tp) ?: return null + val id = tw.getLabelFor(getTypeParameterLabel(tp)) + + // Note apparentIndex does not necessarily equal `tp.index`, because at least + // constructor type parameters + // have indices offset from the type parameters of the constructed class (i.e. the + // parameter S of + // `class Generic { public Generic(T t, S s) { ... } }` will have `tp.index` 1, + // not 0). + tw.writeTypeVars(id, tp.name.asString(), apparentIndex, 0, parentId) + val locId = tw.getLocation(tp) + tw.writeHasLocation(id, locId) + + // Annoyingly, we have no obvious way to pair up the bounds of an IrTypeParameter and a + // JavaTypeParameter + // because JavaTypeParameter provides a Collection not an ordered list, so we can only + // do our best here: + fun tryGetJavaBound(idx: Int) = + when (tp.superTypes.size) { + 1 -> javaTypeParameter?.upperBounds?.singleOrNull() + else -> (javaTypeParameter?.upperBounds as? List)?.getOrNull(idx) + } + + tp.superTypes.forEachIndexed { boundIdx, bound -> + if (!(bound.isAny() || bound.isNullableAny())) { + tw.getLabelFor("@\"bound;$boundIdx;{$id}\"") { + // Note we don't look for @JvmSuppressWildcards here because it doesn't seem + // to have any impact + // on kotlinc adding wildcards to type parameter bounds. + val boundWithWildcards = + addJavaLoweringWildcards(bound, true, tryGetJavaBound(tp.index)) + tw.writeTypeBounds( + it, + useType(boundWithWildcards).javaResult.id.cast(), + boundIdx, + id + ) + } + } + } + + if (tp.isReified) { + addModifiers(id, "reified") + } + + if (tp.variance == Variance.IN_VARIANCE) { + addModifiers(id, "in") + } else if (tp.variance == Variance.OUT_VARIANCE) { + addModifiers(id, "out") + } + + // extractAnnotations(tp, id) + // TODO: introduce annotations once they can be disambiguated from bounds, which are + // also child expressions. + return id + } + } + + private fun extractVisibility( + elementForLocation: IrElement, + id: Label, + v: DescriptorVisibility + ) { + with("visibility", elementForLocation) { + when (v) { + DescriptorVisibilities.PRIVATE -> addModifiers(id, "private") + DescriptorVisibilities.PRIVATE_TO_THIS -> addModifiers(id, "private") + DescriptorVisibilities.PROTECTED -> addModifiers(id, "protected") + DescriptorVisibilities.PUBLIC -> addModifiers(id, "public") + DescriptorVisibilities.INTERNAL -> addModifiers(id, "internal") + DescriptorVisibilities.LOCAL -> + if (elementForLocation is IrFunction && elementForLocation.isLocalFunction()) { + // The containing class is `private`. + addModifiers(id, "public") + } else { + addVisibilityModifierToLocalOrAnonymousClass(id) + } + is DelegatedDescriptorVisibility -> { + when (v.delegate) { + JavaVisibilities.ProtectedStaticVisibility -> { + addModifiers(id, "protected") + addModifiers(id, "static") + } + JavaVisibilities.PackageVisibility -> { + // default java visibility (top level) + } + JavaVisibilities.ProtectedAndPackage -> { + addModifiers(id, "protected") + } + else -> + logger.errorElement( + "Unexpected delegated visibility: $v", + elementForLocation + ) + } + } + else -> logger.errorElement("Unexpected visibility: $v", elementForLocation) + } + } + } + + private fun extractClassModifiers(c: IrClass, id: Label) { + with("class modifiers", c) { + when (c.modality) { + Modality.FINAL -> addModifiers(id, "final") + Modality.SEALED -> addModifiers(id, "sealed") + Modality.OPEN -> {} // This is the default + Modality.ABSTRACT -> addModifiers(id, "abstract") + else -> logger.errorElement("Unexpected class modality: ${c.modality}", c) + } + extractVisibility(c, id, c.visibility) + } + } + + fun extractClassInstance( + classLabel: Label, + c: IrClass, + argsIncludingOuterClasses: List?, + shouldExtractOutline: Boolean, + shouldExtractDetails: Boolean + ) { + DeclarationStackAdjuster(c).use { + if (shouldExtractOutline) { + extractClassWithoutMembers(c, argsIncludingOuterClasses) + } + + if (shouldExtractDetails) { + val supertypeMode = + if (argsIncludingOuterClasses == null) ExtractSupertypesMode.Raw + else ExtractSupertypesMode.Specialised(argsIncludingOuterClasses) + extractClassSupertypes(c, classLabel, supertypeMode, true) + extractNonPrivateMemberPrototypes(c, argsIncludingOuterClasses, classLabel) + } + } + } + + // `argsIncludingOuterClasses` can be null to describe a raw generic type. + // For non-generic types it will be zero-length list. + private fun extractClassWithoutMembers( + c: IrClass, + argsIncludingOuterClasses: List? + ): Label { + with("class instance", c) { + if (argsIncludingOuterClasses?.isEmpty() == true) { + logger.error("Instance without type arguments: " + c.name.asString()) + } + + val classLabelResults = getClassLabel(c, argsIncludingOuterClasses) + val id = tw.getLabelFor(classLabelResults.classLabel) + val pkg = c.packageFqName?.asString() ?: "" + val cls = classLabelResults.shortName + val pkgId = extractPackage(pkg) + // TODO: There's lots of duplication between this and extractClassSource. + // Can we share it? + val sourceId = useClassSource(c) + tw.writeClasses_or_interfaces(id, cls, pkgId, sourceId) + if (c.isInterfaceLike) { + tw.writeIsInterface(id) + } else { + val kind = c.kind + if (kind == ClassKind.ENUM_CLASS) { + tw.writeIsEnumType(id) + } else if ( + kind != ClassKind.CLASS && + kind != ClassKind.OBJECT && + kind != ClassKind.ENUM_ENTRY + ) { + logger.errorElement("Unrecognised class kind $kind", c) + } + } + + val typeArgs = removeOuterClassTypeArgs(c, argsIncludingOuterClasses) + if (typeArgs != null) { + // From 1.9, the list might change when we call erase, + // so we make a copy that it is safe to iterate over. + val typeArgsCopy = typeArgs.toList() + for ((idx, arg) in typeArgsCopy.withIndex()) { + val argId = getTypeArgumentLabel(arg).id + tw.writeTypeArgs(argId, idx, id) + } + tw.writeIsParameterized(id) + } else { + tw.writeIsRaw(id) + } + + val unbound = useClassSource(c) + tw.writeErasure(id, unbound) + extractClassModifiers(c, id) + extractClassSupertypes( + c, + id, + if (argsIncludingOuterClasses == null) ExtractSupertypesMode.Raw + else ExtractSupertypesMode.Specialised(argsIncludingOuterClasses) + ) + + val locId = getLocation(c, argsIncludingOuterClasses) + tw.writeHasLocation(id, locId) + + // Extract the outer <-> inner class relationship, passing on any type arguments in + // excess to this class' parameters if this is an inner class. + // For example, in `class Outer { inner class Inner { } }`, `Inner` + // nests within `Outer` and raw `Inner<>` within `Outer<>`, + // but for a similar non-`inner` (in Java terms, static nested) class both `Inner` + // and `Inner<>` nest within the unbound type `Outer`. + val useBoundOuterType = + (c.isInner || c.isLocal) && + (c.parents.firstNotNullOfOrNull { + when (it) { + is IrClass -> + when { + it.typeParameters.isNotEmpty() -> + true // Type parameters visible to this class -- extract an + // enclosing bound or raw type. + !(it.isInner || it.isLocal) -> + false // No type parameters seen yet, and this is a static + // class -- extract an enclosing unbound type. + else -> + null // No type parameters seen here, but may be visible + // enclosing type parameters; keep searching. + } + else -> + null // Look through enclosing non-class entities (this may need to + // change) + } + } ?: false) + + extractEnclosingClass( + c.parent, + id, + c, + locId, + if (useBoundOuterType) argsIncludingOuterClasses?.drop(c.typeParameters.size) + else listOf() + ) + + return id + } + } + + private fun getLocation( + decl: IrDeclaration, + typeArgs: List? + ): Label { + return if (typeArgs != null && typeArgs.isNotEmpty()) { + val binaryPath = getIrDeclarationBinaryPath(decl) + if (binaryPath == null) { + tw.getLocation(decl) + } else { + val newTrapWriter = tw.makeFileTrapWriter(binaryPath, true) + newTrapWriter.getWholeFileLocation() + } + } else { + tw.getLocation(decl) + } + } + + private fun makeTypeParamSubstitution( + c: IrClass, + argsIncludingOuterClasses: List? + ) = + when (argsIncludingOuterClasses) { + null -> { x: IrType, _: TypeContext, _: IrPluginContext -> x.toRawType() } + else -> makeGenericSubstitutionFunction(c, argsIncludingOuterClasses) + } + + fun extractDeclarationPrototype( + d: IrDeclaration, + parentId: Label, + argsIncludingOuterClasses: List?, + typeParamSubstitutionQ: TypeSubstitution? = null + ) { + val typeParamSubstitution = + typeParamSubstitutionQ + ?: when (val parent = d.parent) { + is IrClass -> makeTypeParamSubstitution(parent, argsIncludingOuterClasses) + else -> { + logger.warnElement("Unable to extract prototype of local declaration", d) + return + } + } + when (d) { + is IrFunction -> + extractFunction( + d, + parentId, + extractBody = false, + extractMethodAndParameterTypeAccesses = false, + extractAnnotations = false, + typeParamSubstitution, + argsIncludingOuterClasses + ) + is IrProperty -> + extractProperty( + d, + parentId, + extractBackingField = false, + extractFunctionBodies = false, + extractPrivateMembers = false, + extractAnnotations = false, + typeParamSubstitution, + argsIncludingOuterClasses + ) + else -> {} + } + } + + // `argsIncludingOuterClasses` can be null to describe a raw generic type. + // For non-generic types it will be zero-length list. + private fun extractNonPrivateMemberPrototypes( + c: IrClass, + argsIncludingOuterClasses: List?, + id: Label + ) { + with("member prototypes", c) { + val typeParamSubstitution = makeTypeParamSubstitution(c, argsIncludingOuterClasses) + + c.declarations.map { + if (shouldExtractDecl(it, false)) { + extractDeclarationPrototype( + it, + id, + argsIncludingOuterClasses, + typeParamSubstitution + ) + } + } + } + } + + private fun extractLocalTypeDeclStmt( + c: IrClass, + callable: Label, + parent: Label, + idx: Int + ) { + val id = + extractClassSource( + c, + extractDeclarations = true, + extractStaticInitializer = true, + extractPrivateMembers = true, + extractFunctionBodies = true + ) + extractLocalTypeDeclStmt(id, c, callable, parent, idx) + } + + private fun extractLocalTypeDeclStmt( + id: Label, + locElement: IrElement, + callable: Label, + parent: Label, + idx: Int + ) { + val stmtId = tw.getFreshIdLabel() + tw.writeStmts_localtypedeclstmt(stmtId, parent, idx, callable) + tw.writeIsLocalClassOrInterface(id, stmtId) + val locId = tw.getLocation(locElement) + tw.writeHasLocation(stmtId, locId) + } + + private fun extractObinitFunction(c: IrClass, parentId: Label) { + // add method: + val obinitLabel = getObinitLabel(c, parentId) + val obinitId = tw.getLabelFor(obinitLabel) + val returnType = useType(pluginContext.irBuiltIns.unitType, TypeContext.RETURN) + tw.writeMethods( + obinitId, + "", + "()", + returnType.javaResult.id, + parentId, + obinitId + ) + tw.writeMethodsKotlinType(obinitId, returnType.kotlinResult.id) + + val locId = tw.getLocation(c) + tw.writeHasLocation(obinitId, locId) + + addModifiers(obinitId, "private") + + // add body: + val blockId = extractBlockBody(obinitId, locId) + + extractDeclInitializers(c.declarations, false) { Pair(blockId, obinitId) } + } + + private val javaLangDeprecated by lazy { referenceExternalClass("java.lang.Deprecated") } + + private val javaLangDeprecatedConstructor by lazy { + javaLangDeprecated?.constructors?.singleOrNull() + } + + private fun replaceKotlinDeprecatedAnnotation( + annotations: List + ): List { + val shouldReplace = + annotations.any { + (it.type as? IrSimpleType)?.classFqName?.asString() == "kotlin.Deprecated" + } && annotations.none { it.type.classOrNull == javaLangDeprecated?.symbol } + val jldConstructor = javaLangDeprecatedConstructor + if (!shouldReplace || jldConstructor == null) return annotations + return annotations.filter { + (it.type as? IrSimpleType)?.classFqName?.asString() != "kotlin.Deprecated" + } + + // Note we lose any arguments to @java.lang.Deprecated that were written in source. + IrConstructorCallImpl.fromSymbolOwner( + UNDEFINED_OFFSET, + UNDEFINED_OFFSET, + jldConstructor.returnType, + jldConstructor.symbol, + 0 + ) + } + + private fun extractAnnotations( + c: IrAnnotationContainer, + annotations: List, + parent: Label, + extractEnumTypeAccesses: Boolean + ) { + val origin = + (c as? IrDeclaration)?.origin + ?: run { + logger.warn("Unexpected annotation container: $c") + return + } + val replacedAnnotations = + if (origin == IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB) + replaceKotlinDeprecatedAnnotation(annotations) + else annotations + val groupedAnnotations = + metaAnnotationSupport.groupRepeatableAnnotations(replacedAnnotations) + for ((idx, constructorCall: IrConstructorCall) in + groupedAnnotations.sortedBy { v -> v.type.classFqName?.asString() }.withIndex()) { + extractAnnotation(constructorCall, parent, idx, extractEnumTypeAccesses) + } + } + + private fun extractAnnotations( + c: IrAnnotationContainer, + parent: Label, + extractEnumTypeAccesses: Boolean + ) { + extractAnnotations(c, c.annotations, parent, extractEnumTypeAccesses) + } + + private fun extractAnnotation( + constructorCall: IrConstructorCall, + parent: Label, + idx: Int, + extractEnumTypeAccesses: Boolean, + contextLabel: String? = null + ): Label { + // Erase the type here because the JVM lowering erases the annotation type, and so the Java + // extractor will see it in erased form. + val t = useType(erase(constructorCall.type)) + val annotationContextLabel = contextLabel ?: "{${t.javaResult.id}}" + val id = + tw.getLabelFor("@\"annotation;{$parent};$annotationContextLabel\"") + tw.writeExprs_declannotation(id, t.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, t.kotlinResult.id) + + val locId = tw.getLocation(constructorCall) + tw.writeHasLocation(id, locId) + + for (i in 0 until constructorCall.valueArgumentsCount) { + val param = constructorCall.symbol.owner.valueParameters[i] + val prop = + constructorCall.symbol.owner.parentAsClass.declarations + .filterIsInstance() + .first { it.name == param.name } + val v = constructorCall.getValueArgument(i) ?: param.defaultValue?.expression + val getter = prop.getter + if (getter == null) { + logger.warnElement("Expected annotation property to define a getter", prop) + } else { + val getterId = useFunction(getter) + if (getterId == null) { + logger.errorElement("Couldn't get ID for getter", getter) + } else { + val exprId = + extractAnnotationValueExpression( + v, + id, + i, + "{$getterId}", + getter.returnType, + extractEnumTypeAccesses + ) + if (exprId != null) { + tw.writeAnnotValue(id, getterId, exprId) + } + } + } + } + return id + } + + private fun extractAnnotationValueExpression( + v: IrExpression?, + parent: Label, + idx: Int, + contextLabel: String, + contextType: IrType?, + extractEnumTypeAccesses: Boolean + ): Label? { + + fun exprId() = tw.getLabelFor("@\"annotationExpr;{$parent};$idx\"") + + return when (v) { + is IrConst<*> -> { + extractConstant(v, parent, idx, null, null, overrideId = exprId()) + } + is IrGetEnumValue -> { + extractEnumValue( + v, + parent, + idx, + null, + null, + extractTypeAccess = extractEnumTypeAccesses, + overrideId = exprId() + ) + } + is IrClassReference -> { + val classRefId = exprId() + val typeAccessId = + tw.getLabelFor("@\"annotationExpr;{$classRefId};0\"") + extractClassReference( + v, + parent, + idx, + null, + null, + overrideId = classRefId, + typeAccessOverrideId = typeAccessId, + useJavaLangClassType = true + ) + } + is IrConstructorCall -> { + extractAnnotation(v, parent, idx, extractEnumTypeAccesses, contextLabel) + } + is IrVararg -> { + tw.getLabelFor("@\"annotationarray;{$parent};$contextLabel\"").also { + arrayId -> + // Use the context type (i.e., the type the annotation expects, not the actual + // type of the array) + // because the Java extractor fills in array types using the same technique. + // These should only + // differ for generic annotations. + if (contextType == null) { + logger.warnElement( + "Expected an annotation array to have an enclosing context", + v + ) + } else { + val type = useType(kClassToJavaClass(contextType)) + tw.writeExprs_arrayinit(arrayId, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(arrayId, type.kotlinResult.id) + tw.writeHasLocation(arrayId, tw.getLocation(v)) + + v.elements.forEachIndexed { index, irVarargElement -> + run { + val argExpr = + when (irVarargElement) { + is IrExpression -> irVarargElement + is IrSpreadElement -> irVarargElement.expression + else -> { + logger.errorElement( + "Unrecognised IrVarargElement: " + + irVarargElement.javaClass, + irVarargElement + ) + null + } + } + extractAnnotationValueExpression( + argExpr, + arrayId, + index, + "child;$index", + null, + extractEnumTypeAccesses + ) + } + } + } + } + } + // is IrErrorExpression + // null + // Note: emitting an ErrorExpr here would induce an inconsistency if this annotation is + // later seen from source or by the Java extractor, + // in both of which cases the real value will get extracted. + else -> null + } + } + + fun extractClassSource( + c: IrClass, + extractDeclarations: Boolean, + extractStaticInitializer: Boolean, + extractPrivateMembers: Boolean, + extractFunctionBodies: Boolean + ): Label { + with("class source", c) { + DeclarationStackAdjuster(c).use { + val id = useClassSource(c) + val pkg = c.packageFqName?.asString() ?: "" + val cls = if (c.isAnonymousObject) "" else c.name.asString() + val pkgId = extractPackage(pkg) + tw.writeClasses_or_interfaces(id, cls, pkgId, id) + if (c.isInterfaceLike) { + tw.writeIsInterface(id) + if (c.kind == ClassKind.ANNOTATION_CLASS) { + tw.writeIsAnnotType(id) + } + } else { + val kind = c.kind + if (kind == ClassKind.ENUM_CLASS) { + tw.writeIsEnumType(id) + } else if ( + kind != ClassKind.CLASS && + kind != ClassKind.OBJECT && + kind != ClassKind.ENUM_ENTRY + ) { + logger.warnElement("Unrecognised class kind $kind", c) + } + + if (c.origin == IrDeclarationOrigin.FILE_CLASS) { + tw.writeFile_class(id) + } + + if (c.isData) { + tw.writeKtDataClasses(id) + } + } + + val locId = tw.getLocation(c) + tw.writeHasLocation(id, locId) + + extractEnclosingClass(c.parent, id, c, locId, listOf()) + + val javaClass = (c.source as? JavaSourceElement)?.javaElement as? JavaClass + + c.typeParameters.mapIndexed { idx, param -> + extractTypeParameter(param, idx, javaClass?.typeParameters?.getOrNull(idx)) + } + if (extractDeclarations) { + if (c.kind == ClassKind.ANNOTATION_CLASS) { + c.declarations.filterIsInstance().forEach { + val getter = it.getter + if (getter == null) { + logger.warnElement( + "Expected an annotation property to have a getter", + it + ) + } else { + extractFunction( + getter, + id, + extractBody = false, + extractMethodAndParameterTypeAccesses = + extractFunctionBodies, + extractAnnotations = true, + null, + listOf() + ) + ?.also { functionLabel -> + tw.writeIsAnnotElem(functionLabel.cast()) + } + } + } + } else { + try { + c.declarations.forEach { + extractDeclaration( + it, + extractPrivateMembers = extractPrivateMembers, + extractFunctionBodies = extractFunctionBodies, + extractAnnotations = true + ) + } + if (extractStaticInitializer) extractStaticInitializer(c, { id }) + extractJvmStaticProxyMethods( + c, + id, + extractPrivateMembers, + extractFunctionBodies + ) + } catch (e: IllegalArgumentException) { + // A Kotlin bug causes this to throw: https://youtrack.jetbrains.com/issue/KT-63847/K2-IllegalStateException-IrFieldPublicSymbolImpl-for-java.time-Clock.OffsetClock.offset0-is-already-bound + // TODO: This should either be removed or log something, once the bug is fixed + } + } + } + if (c.isNonCompanionObject) { + // For `object MyObject { ... }`, the .class has an + // automatically-generated `public static final MyObject INSTANCE` + // field that may be referenced from Java code, and is used in our + // IrGetObjectValue support. We therefore need to fabricate it + // here. + val instance = useObjectClassInstance(c) + val type = useSimpleTypeClass(c, emptyList(), false) + tw.writeFields(instance.id, instance.name, type.javaResult.id, id, instance.id) + tw.writeFieldsKotlinType(instance.id, type.kotlinResult.id) + tw.writeHasLocation(instance.id, locId) + addModifiers(instance.id, "public", "static", "final") + tw.writeClass_object(id, instance.id) + } + if (c.isObject) { + addModifiers(id, "static") + } + if (extractFunctionBodies && needsObinitFunction(c)) { + extractObinitFunction(c, id) + } + + extractClassModifiers(c, id) + extractClassSupertypes( + c, + id, + inReceiverContext = true + ) // inReceiverContext = true is specified to force extraction of member prototypes + // of base types + + linesOfCode?.linesOfCodeInDeclaration(c, id) + + val additionalAnnotations = + if ( + c.kind == ClassKind.ANNOTATION_CLASS && + c.origin != IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB + ) + metaAnnotationSupport.generateJavaMetaAnnotations(c, extractFunctionBodies) + else listOf() + + extractAnnotations( + c, + c.annotations + additionalAnnotations, + id, + extractFunctionBodies + ) + + if (extractFunctionBodies && !c.isAnonymousObject && !c.isLocal) + externalClassExtractor.writeStubTrapFile(c) + + return id + } + } + } + + val jvmStaticFqName = FqName("kotlin.jvm.JvmStatic") + + private fun extractJvmStaticProxyMethods( + c: IrClass, + classId: Label, + extractPrivateMembers: Boolean, + extractFunctionBodies: Boolean + ) { + + // Add synthetic forwarders for any JvmStatic methods or properties: + val companionObject = c.companionObject() ?: return + + val cType = c.typeWith() + val companionType = companionObject.typeWith() + + fun makeProxyFunction(f: IrFunction) { + // Emit a function in class `c` that delegates to the same function defined on + // `c.CompanionInstance`. + val proxyFunctionId = tw.getLabelFor(getFunctionLabel(f, classId, listOf())) + // We extract the function prototype with its ID overridden to belong to `c` not the + // companion object, + // but suppress outputting the body, which we will replace with a delegating call below. + forceExtractFunction( + f, + classId, + extractBody = false, + extractMethodAndParameterTypeAccesses = extractFunctionBodies, + extractAnnotations = false, + typeSubstitution = null, + classTypeArgsIncludingOuterClasses = listOf(), + extractOrigin = false, + OverriddenFunctionAttributes(id = proxyFunctionId) + ) + addModifiers(proxyFunctionId, "static") + tw.writeCompiler_generated( + proxyFunctionId, + CompilerGeneratedKinds.JVMSTATIC_PROXY_METHOD.kind + ) + if (extractFunctionBodies) { + val realFunctionLocId = tw.getLocation(f) + extractExpressionBody(proxyFunctionId, realFunctionLocId).also { returnId -> + extractRawMethodAccess( + f, + realFunctionLocId, + f.returnType, + proxyFunctionId, + returnId, + 0, + returnId, + f.valueParameters.size, + { argParent, idxOffset -> + f.valueParameters.forEachIndexed { idx, param -> + val syntheticParamId = useValueParameter(param, proxyFunctionId) + extractVariableAccess( + syntheticParamId, + param.type, + realFunctionLocId, + argParent, + idxOffset + idx, + proxyFunctionId, + returnId + ) + } + }, + companionType, + { callId -> + val companionField = + useCompanionObjectClassInstance(companionObject)?.id + extractVariableAccess( + companionField, + companionType, + realFunctionLocId, + callId, + -1, + proxyFunctionId, + returnId + ) + .also { varAccessId -> + extractTypeAccessRecursive( + cType, + realFunctionLocId, + varAccessId, + -1, + proxyFunctionId, + returnId + ) + } + }, + null + ) + } + } + } + + companionObject.declarations.forEach { + if (shouldExtractDecl(it, extractPrivateMembers)) { + val wholeDeclAnnotated = it.hasAnnotation(jvmStaticFqName) + when (it) { + is IrFunction -> { + if (wholeDeclAnnotated) { + makeProxyFunction(it) + if (it.hasAnnotation(jvmOverloadsFqName)) { + extractGeneratedOverloads( + it, + classId, + classId, + extractFunctionBodies, + extractMethodAndParameterTypeAccesses = extractFunctionBodies, + typeSubstitution = null, + classTypeArgsIncludingOuterClasses = listOf() + ) + } + } + } + is IrProperty -> { + it.getter?.let { getter -> + if (wholeDeclAnnotated || getter.hasAnnotation(jvmStaticFqName)) + makeProxyFunction(getter) + } + it.setter?.let { setter -> + if (wholeDeclAnnotated || setter.hasAnnotation(jvmStaticFqName)) + makeProxyFunction(setter) + } + } + } + } + } + } + + /** + * This function traverses the declaration-parent hierarchy upwards, and retrieves the enclosing + * class of a class to extract the `enclInReftype` relation. Additionally, it extracts a + * companion field for a companion object into its parent class. + * + * Note that the nested class can also be a local class declared inside a function, so the + * upwards traversal is skipping the non-class parents. Also, in some cases the file class is + * the enclosing one, which has no IR representation. + */ + private fun extractEnclosingClass( + declarationParent: + IrDeclarationParent, // The declaration parent of the element for which we are + // extracting the enclosing class + innerId: Label, // ID of the inner class + innerClass: + IrClass?, // The inner class, if available. It's not available if the enclosing class of + // a generated class is being extracted + innerLocId: Label, // Location of the inner class + parentClassTypeArguments: + List< + IrTypeArgument + >? // Type arguments of the parent class. If `parentClassTypeArguments` is null, the + // parent class is a raw type + ) { + with("enclosing class", declarationParent) { + var parent: IrDeclarationParent? = declarationParent + while (parent != null) { + if (parent is IrClass) { + val parentId = useClassInstance(parent, parentClassTypeArguments).typeResult.id + tw.writeEnclInReftype(innerId, parentId) + if (innerClass != null && innerClass.isCompanion) { + // If we are a companion then our parent has a + // public static final ParentClass$CompanionObjectClass + // CompanionObjectName; + // that we need to fabricate here + val instance = useCompanionObjectClassInstance(innerClass) + if (instance != null) { + val type = useSimpleTypeClass(innerClass, emptyList(), false) + tw.writeFields( + instance.id, + instance.name, + type.javaResult.id, + parentId, + instance.id + ) + tw.writeFieldsKotlinType(instance.id, type.kotlinResult.id) + tw.writeHasLocation(instance.id, innerLocId) + addModifiers(instance.id, "public", "static", "final") + tw.writeType_companion_object(parentId, instance.id, innerId) + } + } + + break + } else if (parent is IrFile) { + if (innerClass != null && !innerClass.isLocal) { + // We don't have to extract file class containers for classes except for + // local classes + break + } + if (this.filePath != parent.path) { + logger.error("Unexpected file parent found") + } + val fileId = extractFileClass(parent) + tw.writeEnclInReftype(innerId, fileId) + break + } + + parent = (parent as? IrDeclaration)?.parent + } + } + } + + private data class FieldResult(val id: Label, val name: String) + + private fun useCompanionObjectClassInstance(c: IrClass): FieldResult? { + val parent = c.parent + if (!c.isCompanion) { + logger.error("Using companion instance for non-companion class") + return null + } else if (parent !is IrClass) { + logger.error("Using companion instance for non-companion class") + return null + } else { + val parentId = useClassInstance(parent, listOf()).typeResult.id + val instanceName = c.name.asString() + val instanceLabel = "@\"field;{$parentId};$instanceName\"" + val instanceId: Label = tw.getLabelFor(instanceLabel) + return FieldResult(instanceId, instanceName) + } + } + + private fun useObjectClassInstance(c: IrClass): FieldResult { + if (!c.isNonCompanionObject) { + logger.error("Using instance for non-object class") + } + val classId = useClassInstance(c, listOf()).typeResult.id + val instanceName = "INSTANCE" + val instanceLabel = "@\"field;{$classId};$instanceName\"" + val instanceId: Label = tw.getLabelFor(instanceLabel) + return FieldResult(instanceId, instanceName) + } + + @OptIn(ObsoleteDescriptorBasedAPI::class) + private fun hasSynthesizedParameterNames(f: IrFunction) = + f.descriptor.hasSynthesizedParameterNames() + + private fun extractValueParameter( + vp: IrValueParameter, + parent: Label, + idx: Int, + typeSubstitution: TypeSubstitution?, + parentSourceDeclaration: Label, + classTypeArgsIncludingOuterClasses: List?, + extractTypeAccess: Boolean, + locOverride: Label? = null + ): TypeResults { + with("value parameter", vp) { + val location = locOverride ?: getLocation(vp, classTypeArgsIncludingOuterClasses) + val maybeAlteredType = + (vp.parent as? IrFunction)?.let { + if (overridesCollectionsMethodWithAlteredParameterTypes(it)) + eraseCollectionsMethodParameterType(vp.type, it.name.asString(), idx) + else if ( + (vp.parent as? IrConstructor)?.parentClassOrNull?.kind == + ClassKind.ANNOTATION_CLASS + ) + kClassToJavaClass(vp.type) + else null + } ?: vp.type + val javaType = + (vp.parent as? IrFunction)?.let { + getJavaCallable(it)?.let { jCallable -> + getJavaValueParameterType(jCallable, idx) + } + } + val typeWithWildcards = + addJavaLoweringWildcards( + maybeAlteredType, + !getInnermostWildcardSupppressionAnnotation(vp), + javaType + ) + val substitutedType = + typeSubstitution?.let { it(typeWithWildcards, TypeContext.OTHER, pluginContext) } + ?: typeWithWildcards + val id = useValueParameter(vp, parent) + if (extractTypeAccess) { + extractTypeAccessRecursive(substitutedType, location, id, -1) + } + val syntheticParameterNames = + isUnderscoreParameter(vp) || + ((vp.parent as? IrFunction)?.let { hasSynthesizedParameterNames(it) } ?: true) + val javaParameter = + when (val callable = (vp.parent as? IrFunction)?.let { getJavaCallable(it) }) { + is JavaConstructor -> callable.valueParameters.getOrNull(idx) + is JavaMethod -> callable.valueParameters.getOrNull(idx) + else -> null + } + val extraAnnotations = + listOfNotNull( + getNullabilityAnnotation( + vp.type, + vp.origin, + vp.annotations, + javaParameter?.annotations + ) + ) + extractAnnotations(vp, vp.annotations + extraAnnotations, id, extractTypeAccess) + return extractValueParameter( + id, + substitutedType, + vp.name.asString(), + location, + parent, + idx, + useValueParameter(vp, parentSourceDeclaration), + syntheticParameterNames, + vp.isVararg, + vp.isNoinline, + vp.isCrossinline + ) + } + } + + private fun extractValueParameter( + id: Label, + t: IrType, + name: String, + locId: Label, + parent: Label, + idx: Int, + paramSourceDeclaration: Label, + syntheticParameterNames: Boolean, + isVararg: Boolean, + isNoinline: Boolean, + isCrossinline: Boolean + ): TypeResults { + val type = useType(t) + tw.writeParams(id, type.javaResult.id, idx, parent, paramSourceDeclaration) + tw.writeParamsKotlinType(id, type.kotlinResult.id) + tw.writeHasLocation(id, locId) + if (!syntheticParameterNames) { + tw.writeParamName(id, name) + } + if (isVararg) { + tw.writeIsVarargsParam(id) + } + if (isNoinline) { + addModifiers(id, "noinline") + } + if (isCrossinline) { + addModifiers(id, "crossinline") + } + return type + } + + /** + * mkContainerLabel is a lambda so that we get laziness: If the container is a file, then we + * don't want to extract the file class unless something actually needs it. + */ + private fun extractStaticInitializer( + container: IrDeclarationContainer, + mkContainerLabel: () -> Label + ) { + with("static initializer extraction", container) { + extractDeclInitializers(container.declarations, true) { + val containerId = mkContainerLabel() + val clinitLabel = + getFunctionLabel( + container, + containerId, + "", + listOf(), + pluginContext.irBuiltIns.unitType, + extensionParamType = null, + functionTypeParameters = listOf(), + classTypeArgsIncludingOuterClasses = listOf(), + overridesCollectionsMethod = false, + javaSignature = null, + addParameterWildcardsByDefault = false + ) + val clinitId = tw.getLabelFor(clinitLabel) + val returnType = useType(pluginContext.irBuiltIns.unitType, TypeContext.RETURN) + tw.writeMethods( + clinitId, + "", + "()", + returnType.javaResult.id, + containerId, + clinitId + ) + tw.writeMethodsKotlinType(clinitId, returnType.kotlinResult.id) + + tw.writeCompiler_generated( + clinitId, + CompilerGeneratedKinds.CLASS_INITIALISATION_METHOD.kind + ) + + val locId = tw.getWholeFileLocation() + tw.writeHasLocation(clinitId, locId) + + addModifiers(clinitId, "static") + + // add and return body block: + Pair(extractBlockBody(clinitId, locId), clinitId) + } + } + } + + private fun extractInstanceInitializerBlock( + parent: StmtParent, + enclosingConstructor: IrConstructor + ) { + with("object initializer block", enclosingConstructor) { + val constructorId = useFunction(enclosingConstructor) + if (constructorId == null) { + logger.errorElement("Cannot get ID for constructor", enclosingConstructor) + return + } + val enclosingClass = enclosingConstructor.parentClassOrNull + if (enclosingClass == null) { + logger.errorElement("Constructor's parent is not a class", enclosingConstructor) + return + } + + // Don't make this block lazily since we need to insert something at the given + // parent.idx position, + // and in the case where there are no initializers to emit an empty block is an + // acceptable filler. + val initBlockId = + tw.getFreshIdLabel().also { + tw.writeStmts_block(it, parent.parent, parent.idx, constructorId) + val locId = tw.getLocation(enclosingConstructor) + tw.writeHasLocation(it, locId) + } + extractDeclInitializers(enclosingClass.declarations, false) { + Pair(initBlockId, constructorId) + } + } + } + + private fun extractDeclInitializers( + declarations: List, + extractStaticInitializers: Boolean, + makeEnclosingBlock: () -> Pair, Label> + ) { + val blockAndFunctionId by lazy { makeEnclosingBlock() } + + // Extract field initializers and init blocks (the latter can only occur in object + // initializers) + var idx = 0 + + fun extractFieldInitializer(f: IrDeclaration) { + val static: Boolean + val initializer: IrExpressionBody? + val lhsType: TypeResults? + val vId: Label? + val isAnnotationClassField: Boolean + if (f is IrField) { + static = f.isStatic + initializer = f.initializer + isAnnotationClassField = isAnnotationClassField(f) + lhsType = useType(if (isAnnotationClassField) kClassToJavaClass(f.type) else f.type) + vId = useField(f) + } else if (f is IrEnumEntry) { + static = true + initializer = f.initializerExpression + isAnnotationClassField = false + lhsType = getEnumEntryType(f) + if (lhsType == null) { + return + } + vId = useEnumEntry(f) + } else { + return + } + + if (static != extractStaticInitializers || initializer == null) { + return + } + + val expr = initializer.expression + + val declLocId = tw.getLocation(f) + extractExpressionStmt( + declLocId, + blockAndFunctionId.first, + idx++, + blockAndFunctionId.second + ) + .also { stmtId -> + val type = + if (isAnnotationClassField) kClassToJavaClass(expr.type) else expr.type + extractAssignExpr(type, declLocId, stmtId, 0, blockAndFunctionId.second, stmtId) + .also { assignmentId -> + tw.writeKtInitializerAssignment(assignmentId) + extractVariableAccess( + vId, + lhsType, + declLocId, + assignmentId, + 0, + blockAndFunctionId.second, + stmtId + ) + .also { lhsId -> + if (static) { + extractStaticTypeAccessQualifier( + f, + lhsId, + declLocId, + blockAndFunctionId.second, + stmtId + ) + } + } + extractExpressionExpr( + expr, + blockAndFunctionId.second, + assignmentId, + 1, + stmtId + ) + } + } + } + + for (decl in declarations) { + when (decl) { + is IrProperty -> { + decl.backingField?.let { extractFieldInitializer(it) } + } + is IrField -> { + extractFieldInitializer(decl) + } + is IrEnumEntry -> { + extractFieldInitializer(decl) + } + is IrAnonymousInitializer -> { + if (decl.isStatic != extractStaticInitializers) { + continue + } + + for (stmt in decl.body.statements) { + extractStatement( + stmt, + blockAndFunctionId.second, + blockAndFunctionId.first, + idx++ + ) + } + } + else -> continue + } + } + } + + private fun isKotlinDefinedInterface(cls: IrClass?) = + cls != null && + cls.isInterface && + cls.origin != IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB + + private fun needsInterfaceForwarder(f: IrFunction) = + // jvmDefaultModeEnabledIsEnabled means that -Xjvm-default=all or all-compatibility was + // used, in which case real Java default interfaces are used, and we don't need to do + // anything. + // Otherwise, for a Kotlin-defined method inheriting a Kotlin-defined default, we need to + // create a synthetic method like + // `int f(int x) { return super.InterfaceWithDefault.f(x); }`, because kotlinc will generate + // a public method and Java callers may directly target it. + // (NB. kotlinc's actual implementation strategy is different -- it makes an inner class + // called InterfaceWithDefault$DefaultImpls and stores the default methods + // there to allow default method usage in Java < 8, but this is hopefully niche. + !jvmDefaultModeEnabledIsEnabled( + pluginContext.languageVersionSettings + .getFlag(JvmAnalysisFlags.jvmDefaultMode)) && + f.parentClassOrNull.let { + it != null && + it.origin != IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB && + it.modality != Modality.ABSTRACT + } && + f.realOverrideTarget.let { + it != f && + (it as? IrSimpleFunction)?.modality != Modality.ABSTRACT && + isKotlinDefinedInterface(it.parentClassOrNull) + } + + private fun makeInterfaceForwarder( + f: IrFunction, + parentId: Label, + extractBody: Boolean, + extractMethodAndParameterTypeAccesses: Boolean, + typeSubstitution: TypeSubstitution?, + classTypeArgsIncludingOuterClasses: List? + ) = + forceExtractFunction( + f, + parentId, + extractBody = false, + extractMethodAndParameterTypeAccesses, + extractAnnotations = false, + typeSubstitution, + classTypeArgsIncludingOuterClasses, + overriddenAttributes = + OverriddenFunctionAttributes( + visibility = DescriptorVisibilities.PUBLIC, + modality = Modality.OPEN + ) + ) + .also { functionId -> + tw.writeCompiler_generated( + functionId, + CompilerGeneratedKinds.INTERFACE_FORWARDER.kind + ) + if (extractBody) { + val realFunctionLocId = tw.getLocation(f) + val inheritedDefaultFunction = f.realOverrideTarget + val directlyInheritedSymbol = + when (f) { + is IrSimpleFunction -> + f.overriddenSymbols.find { it.owner === inheritedDefaultFunction } + ?: f.overriddenSymbols.find { + it.owner.realOverrideTarget === inheritedDefaultFunction + } + ?: inheritedDefaultFunction.symbol + else -> + inheritedDefaultFunction + .symbol // This is strictly invalid, since we shouldn't use + // A.super.f(...) where A may not be a direct supertype, + // but this path should also be unreachable. + } + val defaultDefiningInterfaceType = + (directlyInheritedSymbol.owner.parentClassOrNull ?: return functionId) + .typeWith() + + extractExpressionBody(functionId, realFunctionLocId).also { returnId -> + extractRawMethodAccess( + f, + realFunctionLocId, + f.returnType, + functionId, + returnId, + 0, + returnId, + f.valueParameters.size, + { argParentId, idxOffset -> + f.valueParameters.mapIndexed { idx, param -> + val syntheticParamId = useValueParameter(param, functionId) + extractVariableAccess( + syntheticParamId, + param.type, + realFunctionLocId, + argParentId, + idxOffset + idx, + functionId, + returnId + ) + } + }, + f.dispatchReceiverParameter?.type, + { callId -> + extractSuperAccess( + defaultDefiningInterfaceType, + functionId, + callId, + -1, + returnId, + realFunctionLocId + ) + }, + null + ) + } + } + } + + private fun extractFunction( + f: IrFunction, + parentId: Label, + extractBody: Boolean, + extractMethodAndParameterTypeAccesses: Boolean, + extractAnnotations: Boolean, + typeSubstitution: TypeSubstitution?, + classTypeArgsIncludingOuterClasses: List? + ) = + if (isFake(f)) { + if (needsInterfaceForwarder(f)) + makeInterfaceForwarder( + f, + parentId, + extractBody, + extractMethodAndParameterTypeAccesses, + typeSubstitution, + classTypeArgsIncludingOuterClasses + ) + else null + } else { + // Work around an apparent bug causing redeclarations of `fun toString(): String` + // specifically in interfaces loaded from Java classes show up like fake overrides. + val overriddenVisibility = + if (f.isFakeOverride && isJavaBinaryObjectMethodRedeclaration(f)) + OverriddenFunctionAttributes(visibility = DescriptorVisibilities.PUBLIC) + else null + forceExtractFunction( + f, + parentId, + extractBody, + extractMethodAndParameterTypeAccesses, + extractAnnotations, + typeSubstitution, + classTypeArgsIncludingOuterClasses, + overriddenAttributes = overriddenVisibility + ) + .also { + // The defaults-forwarder function is a static utility, not a member, so we only + // need to extract this for the unspecialised instance of this class. + if (classTypeArgsIncludingOuterClasses.isNullOrEmpty()) + extractDefaultsFunction( + f, + parentId, + extractBody, + extractMethodAndParameterTypeAccesses + ) + extractGeneratedOverloads( + f, + parentId, + null, + extractBody, + extractMethodAndParameterTypeAccesses, + typeSubstitution, + classTypeArgsIncludingOuterClasses + ) + } + } + + private fun extractDefaultsFunction( + f: IrFunction, + parentId: Label, + extractBody: Boolean, + extractMethodAndParameterTypeAccesses: Boolean + ) { + if (f.valueParameters.none { it.defaultValue != null }) return + + val id = getDefaultsMethodLabel(f) + if (id == null) { + logger.errorElement("Cannot get defaults method label for function", f) + return + } + val locId = getLocation(f, null) + val extReceiver = f.extensionReceiverParameter + val dispatchReceiver = if (f.shouldExtractAsStatic) null else f.dispatchReceiverParameter + val parameterTypes = getDefaultsMethodArgTypes(f) + val allParamTypeResults = + parameterTypes.mapIndexed { i, paramType -> + val paramId = tw.getLabelFor(getValueParameterLabel(id, i)) + extractValueParameter( + paramId, + paramType, + "p$i", + locId, + id, + i, + paramId, + isVararg = false, + syntheticParameterNames = true, + isCrossinline = false, + isNoinline = false + ) + .also { + if (extractMethodAndParameterTypeAccesses) + extractTypeAccess(useType(paramType), locId, paramId, -1) + } + } + val paramsSignature = + allParamTypeResults.joinToString(separator = ",", prefix = "(", postfix = ")") { + signatureOrWarn(it.javaResult, f) + } + val shortName = getDefaultsMethodName(f) + + if (f.symbol is IrConstructorSymbol) { + val constrId = id.cast() + extractConstructor(constrId, shortName, paramsSignature, parentId, constrId) + } else { + val methodId = id.cast() + extractMethod( + methodId, + locId, + shortName, + erase(f.returnType), + paramsSignature, + parentId, + methodId, + origin = null, + extractTypeAccess = extractMethodAndParameterTypeAccesses + ) + addModifiers(id, "static") + if (extReceiver != null) { + val idx = if (dispatchReceiver != null) 1 else 0 + val extendedType = allParamTypeResults[idx] + tw.writeKtExtensionFunctions( + methodId, + extendedType.javaResult.id, + extendedType.kotlinResult.id + ) + } + } + tw.writeHasLocation(id, locId) + if ( + f.visibility != DescriptorVisibilities.PRIVATE && + f.visibility != DescriptorVisibilities.PRIVATE_TO_THIS + ) { + // Private methods have package-private (default) visibility $default methods; all other + // visibilities seem to produce a public $default method. + addModifiers(id, "public") + } + tw.writeCompiler_generated(id, CompilerGeneratedKinds.DEFAULT_ARGUMENTS_METHOD.kind) + + if (extractBody) { + val nonSyntheticParams = listOfNotNull(dispatchReceiver) + f.valueParameters + // This stack entry represents as if we're extracting the 'real' function `f`, giving + // the indices of its non-synthetic parameters + // such that when we extract the default expressions below, any reference to f's nth + // parameter will resolve to f$default's + // n + o'th parameter, where `o` is the parameter offset caused by adding any dispatch + // receiver to the parameter list. + // Note we don't need to add the extension receiver here because `useValueParameter` + // always assumes an extension receiver + // will be prepended if one exists. + val realFunctionId = useFunction(f, parentId, null) + DeclarationStackAdjuster( + f, + OverriddenFunctionAttributes( + id, + id, + locId, + nonSyntheticParams, + typeParameters = listOf(), + isStatic = true + ) + ) + .use { + val realParamsVarId = getValueParameterLabel(id, parameterTypes.size - 2) + val intType = pluginContext.irBuiltIns.intType + val paramIdxOffset = + listOf(dispatchReceiver, f.extensionReceiverParameter).count { it != null } + extractBlockBody(id, locId).also { blockId -> + var nextStmt = 0 + // For each parameter with a default, sub in the default value if the caller + // hasn't supplied a value: + f.valueParameters.forEachIndexed { paramIdx, param -> + val defaultVal = param.defaultValue + if (defaultVal != null) { + extractIfStmt(locId, blockId, nextStmt++, id).also { ifId -> + // if (realParams & thisParamBit == 0) ... + extractEqualsExpression(locId, ifId, 0, id, ifId).also { eqId -> + extractAndbitExpression(intType, locId, eqId, 0, id, ifId) + .also { opId -> + extractConstantInteger( + 1 shl paramIdx, + locId, + opId, + 0, + id, + ifId + ) + extractVariableAccess( + tw.getLabelFor(realParamsVarId), + intType, + locId, + opId, + 1, + id, + ifId + ) + } + extractConstantInteger(0, locId, eqId, 1, id, ifId) + } + // thisParamVar = defaultExpr... + extractExpressionStmt(locId, ifId, 1, id).also { exprStmtId -> + extractAssignExpr( + param.type, + locId, + exprStmtId, + 0, + id, + exprStmtId + ) + .also { assignId -> + extractVariableAccess( + tw.getLabelFor( + getValueParameterLabel( + id, + paramIdx + paramIdxOffset + ) + ), + param.type, + locId, + assignId, + 0, + id, + exprStmtId + ) + extractExpressionExpr( + defaultVal.expression, + id, + assignId, + 1, + exprStmtId + ) + } + } + } + } + } + // Now call the real function: + if (f is IrConstructor) { + tw.getFreshIdLabel().also { thisCallId -> + tw.writeStmts_constructorinvocationstmt( + thisCallId, + blockId, + nextStmt++, + id + ) + tw.writeHasLocation(thisCallId, locId) + f.valueParameters.forEachIndexed { idx, param -> + extractVariableAccess( + tw.getLabelFor(getValueParameterLabel(id, idx)), + param.type, + locId, + thisCallId, + idx, + id, + thisCallId + ) + } + tw.writeCallableBinding(thisCallId, realFunctionId) + } + } else { + tw.getFreshIdLabel().also { returnId -> + tw.writeStmts_returnstmt(returnId, blockId, nextStmt++, id) + tw.writeHasLocation(returnId, locId) + extractMethodAccessWithoutArgs( + f.returnType, + locId, + id, + returnId, + 0, + returnId, + realFunctionId + ) + .also { thisCallId -> + val realFnIdxOffset = + if (f.extensionReceiverParameter != null) 1 else 0 + val paramMappings = + f.valueParameters.mapIndexed { idx, param -> + Triple( + param.type, + idx + paramIdxOffset, + idx + realFnIdxOffset + ) + } + + listOfNotNull( + dispatchReceiver?.let { + Triple(it.type, 0, -1) + }, + extReceiver?.let { + Triple( + it.type, + if (dispatchReceiver != null) 1 else 0, + 0 + ) + } + ) + paramMappings.forEach { (type, fromIdx, toIdx) -> + extractVariableAccess( + tw.getLabelFor( + getValueParameterLabel(id, fromIdx) + ), + type, + locId, + thisCallId, + toIdx, + id, + returnId + ) + } + if (f.shouldExtractAsStatic) + extractStaticTypeAccessQualifier( + f, + thisCallId, + locId, + id, + returnId + ) + else if (f.isLocalFunction()) { + extractNewExprForLocalFunction( + getLocallyVisibleFunctionLabels(f), + thisCallId, + locId, + id, + returnId + ) + } + } + } + } + } + } + } + } + + private val jvmOverloadsFqName = FqName("kotlin.jvm.JvmOverloads") + + private fun extractGeneratedOverloads( + f: IrFunction, + parentId: Label, + maybeSourceParentId: Label?, + extractBody: Boolean, + extractMethodAndParameterTypeAccesses: Boolean, + typeSubstitution: TypeSubstitution?, + classTypeArgsIncludingOuterClasses: List? + ) { + + fun extractGeneratedOverload(paramList: List) { + val overloadParameters = paramList.filterNotNull() + // Note `overloadParameters` have incorrect parents and indices, since there is no + // actual IrFunction describing the required synthetic overload. + // We have to use the `overriddenAttributes` element of `DeclarationStackAdjuster` to + // fix up references to these parameters while we're extracting + // these synthetic overloads. + val overloadId = + tw.getLabelFor( + getFunctionLabel( + f, + parentId, + classTypeArgsIncludingOuterClasses, + overloadParameters + ) + ) + val sourceParentId = + maybeSourceParentId + ?: if (typeSubstitution != null) useDeclarationParentOf(f, false) else parentId + if (sourceParentId == null) { + logger.errorElement("Cannot get source parent ID for function", f) + return + } + val sourceDeclId = + tw.getLabelFor( + getFunctionLabel(f, sourceParentId, listOf(), overloadParameters) + ) + val overriddenAttributes = + OverriddenFunctionAttributes( + id = overloadId, + sourceDeclarationId = sourceDeclId, + valueParameters = overloadParameters + ) + forceExtractFunction( + f, + parentId, + extractBody = false, + extractMethodAndParameterTypeAccesses, + extractAnnotations = false, + typeSubstitution, + classTypeArgsIncludingOuterClasses, + overriddenAttributes = overriddenAttributes + ) + tw.writeCompiler_generated(overloadId, CompilerGeneratedKinds.JVMOVERLOADS_METHOD.kind) + val realFunctionLocId = tw.getLocation(f) + if (extractBody) { + + DeclarationStackAdjuster(f, overriddenAttributes).use { + + // Create a synthetic function body that calls the corresponding $default + // function: + val regularArgs = + paramList.map { it?.let { p -> IrGetValueImpl(-1, -1, p.symbol) } } + + if (f is IrConstructor) { + val blockId = extractBlockBody(overloadId, realFunctionLocId) + val constructorCallId = tw.getFreshIdLabel() + tw.writeStmts_constructorinvocationstmt( + constructorCallId, + blockId, + 0, + overloadId + ) + tw.writeHasLocation(constructorCallId, realFunctionLocId) + tw.writeCallableBinding( + constructorCallId, + getDefaultsMethodLabel(f, parentId) + ) + + extractDefaultsCallArguments( + constructorCallId, + f, + overloadId, + constructorCallId, + regularArgs, + null, + null + ) + } else { + val dispatchReceiver = + f.dispatchReceiverParameter?.let { IrGetValueImpl(-1, -1, it.symbol) } + val extensionReceiver = + f.extensionReceiverParameter?.let { IrGetValueImpl(-1, -1, it.symbol) } + + extractExpressionBody(overloadId, realFunctionLocId).also { returnId -> + extractsDefaultsCall( + f, + realFunctionLocId, + f.returnType, + overloadId, + returnId, + 0, + returnId, + regularArgs, + dispatchReceiver, + extensionReceiver + ) + } + } + } + } + } + + if (!f.hasAnnotation(jvmOverloadsFqName)) { + if ( + f is IrConstructor && + f.valueParameters.isNotEmpty() && + f.valueParameters.all { it.defaultValue != null } && + f.parentClassOrNull?.let { + // Don't create a default constructor for an annotation class, or a class + // that explicitly declares a no-arg constructor. + !it.isAnnotationClass && + it.declarations.none { d -> + d is IrConstructor && d.valueParameters.isEmpty() + } + } == true + ) { + // Per https://kotlinlang.org/docs/classes.html#creating-instances-of-classes, a + // single default overload gets created specifically + // when we have all default parameters, regardless of `@JvmOverloads`. + extractGeneratedOverload(f.valueParameters.map { _ -> null }) + } + return + } + + val paramList: MutableList = f.valueParameters.toMutableList() + for (n in (f.valueParameters.size - 1) downTo 0) { + if (f.valueParameters[n].defaultValue != null) { + paramList[n] = null // Remove this parameter, to be replaced by a default value + extractGeneratedOverload(paramList) + } + } + } + + private fun extractConstructor( + id: Label, + shortName: String, + paramsSignature: String, + parentId: Label, + sourceDeclaration: Label + ) { + val unitType = useType(pluginContext.irBuiltIns.unitType, TypeContext.RETURN) + tw.writeConstrs( + id, + shortName, + "$shortName$paramsSignature", + unitType.javaResult.id, + parentId, + sourceDeclaration + ) + tw.writeConstrsKotlinType(id, unitType.kotlinResult.id) + } + + private fun extractMethod( + id: Label, + locId: Label, + shortName: String, + returnType: IrType, + paramsSignature: String, + parentId: Label, + sourceDeclaration: Label, + origin: IrDeclarationOrigin?, + extractTypeAccess: Boolean + ) { + val returnTypeResults = useType(returnType, TypeContext.RETURN) + tw.writeMethods( + id, + shortName, + "$shortName$paramsSignature", + returnTypeResults.javaResult.id, + parentId, + sourceDeclaration + ) + tw.writeMethodsKotlinType(id, returnTypeResults.kotlinResult.id) + when (origin) { + IrDeclarationOrigin.GENERATED_DATA_CLASS_MEMBER -> + tw.writeCompiler_generated( + id, + CompilerGeneratedKinds.GENERATED_DATA_CLASS_MEMBER.kind + ) + IrDeclarationOrigin.DEFAULT_PROPERTY_ACCESSOR -> + tw.writeCompiler_generated( + id, + CompilerGeneratedKinds.DEFAULT_PROPERTY_ACCESSOR.kind + ) + IrDeclarationOrigin.ENUM_CLASS_SPECIAL_MEMBER -> + tw.writeCompiler_generated( + id, + CompilerGeneratedKinds.ENUM_CLASS_SPECIAL_MEMBER.kind + ) + } + if (extractTypeAccess) { + extractTypeAccessRecursive(returnType, locId, id, -1) + } + } + + private fun signatureOrWarn(t: TypeResult<*>, associatedElement: IrElement?) = + t.signature + ?: "" + .also { + if (associatedElement != null) + logger.warnElement( + "Needed a signature for a type that doesn't have one", + associatedElement + ) + else logger.warn("Needed a signature for a type that doesn't have one") + } + + private fun getNullabilityAnnotationName( + t: IrType, + declOrigin: IrDeclarationOrigin, + existingAnnotations: List, + javaAnnotations: Collection? + ): FqName? { + if (t !is IrSimpleType) return null + + fun hasExistingAnnotation(name: FqName) = + existingAnnotations.any { existing -> existing.type.classFqName == name } + + return if (declOrigin == IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB) { + // Java declaration: restore a NotNull or Nullable annotation if the original Java + // member had one but the Kotlin compiler removed it. + javaAnnotations + ?.mapNotNull { it.classId?.asSingleFqName() } + ?.singleOrNull { + NOT_NULL_ANNOTATIONS.contains(it) || NULLABLE_ANNOTATIONS.contains(it) + } + ?.takeUnless { hasExistingAnnotation(it) } + } else { + // Kotlin declaration: add a NotNull annotation to a non-nullable non-primitive type, + // unless one is already present. + // Usually Kotlin declarations can't have a manual `@NotNull`, but this happens at least + // when delegating members are + // synthesised and inherit the annotation from the delegate (which given it has + // @NotNull, is likely written in Java) + JvmAnnotationNames.JETBRAINS_NOT_NULL_ANNOTATION.takeUnless { + t.isNullable() || + primitiveTypeMapping.getPrimitiveInfo(t) != null || + hasExistingAnnotation(it) + } + } + } + + private fun getNullabilityAnnotation( + t: IrType, + declOrigin: IrDeclarationOrigin, + existingAnnotations: List, + javaAnnotations: Collection? + ) = + getNullabilityAnnotationName(t, declOrigin, existingAnnotations, javaAnnotations)?.let { + getClassByFqName(pluginContext, it)?.let { annotationClass -> + annotationClass.owner.declarations.firstIsInstanceOrNull()?.let { + annotationConstructor -> + IrConstructorCallImpl.fromSymbolOwner( + UNDEFINED_OFFSET, + UNDEFINED_OFFSET, + annotationConstructor.returnType, + annotationConstructor.symbol, + 0 + ) + } + } + } + + private fun forceExtractFunction( + f: IrFunction, + parentId: Label, + extractBody: Boolean, + extractMethodAndParameterTypeAccesses: Boolean, + extractAnnotations: Boolean, + typeSubstitution: TypeSubstitution?, + classTypeArgsIncludingOuterClasses: List?, + extractOrigin: Boolean = true, + overriddenAttributes: OverriddenFunctionAttributes? = null + ): Label { + with("function", f) { + DeclarationStackAdjuster(f, overriddenAttributes).use { + val javaCallable = getJavaCallable(f) + getFunctionTypeParameters(f).mapIndexed { idx, tp -> + extractTypeParameter( + tp, + idx, + (javaCallable as? JavaTypeParameterListOwner) + ?.typeParameters + ?.getOrNull(idx) + ) + } + + val id = + overriddenAttributes?.id + ?: // If this is a class that would ordinarily be replaced by a Java + // equivalent (e.g. kotlin.Map -> java.util.Map), + // don't replace here, really extract the Kotlin version: + useFunction( + f, + parentId, + classTypeArgsIncludingOuterClasses, + noReplace = true + ) + + val sourceDeclaration = + overriddenAttributes?.sourceDeclarationId + ?: if (typeSubstitution != null && overriddenAttributes?.id == null) { + val sourceFunId = useFunction(f) + if (sourceFunId == null) { + logger.errorElement("Cannot get source ID for function", f) + id // TODO: This is wrong; we ought to just fail in this case + } else { + sourceFunId + } + } else { + id + } + + val extReceiver = f.extensionReceiverParameter + // The following parameter order is correct, because member $default methods (where + // the order would be [dispatchParam], [extensionParam], normalParams) are not + // extracted here + val fParameters = + listOfNotNull(extReceiver) + + (overriddenAttributes?.valueParameters ?: f.valueParameters) + val paramTypes = + fParameters.mapIndexed { i, vp -> + extractValueParameter( + vp, + id, + i, + typeSubstitution, + sourceDeclaration, + classTypeArgsIncludingOuterClasses, + extractTypeAccess = extractMethodAndParameterTypeAccesses, + overriddenAttributes?.sourceLoc + ) + } + if (extReceiver != null) { + val extendedType = paramTypes[0] + tw.writeKtExtensionFunctions( + id.cast(), + extendedType.javaResult.id, + extendedType.kotlinResult.id + ) + } + + val paramsSignature = + paramTypes.joinToString(separator = ",", prefix = "(", postfix = ")") { + signatureOrWarn(it.javaResult, f) + } + + val adjustedReturnType = + addJavaLoweringWildcards( + getAdjustedReturnType(f), + false, + (javaCallable as? JavaMethod)?.returnType + ) + val substReturnType = + typeSubstitution?.let { + it(adjustedReturnType, TypeContext.RETURN, pluginContext) + } ?: adjustedReturnType + + val locId = + overriddenAttributes?.sourceLoc + ?: getLocation(f, classTypeArgsIncludingOuterClasses) + + if (f.symbol is IrConstructorSymbol) { + val shortName = + when { + adjustedReturnType.isAnonymous -> "" + typeSubstitution != null -> + useType(substReturnType).javaResult.shortName + else -> + adjustedReturnType.classFqName?.shortName()?.asString() + ?: f.name.asString() + } + extractConstructor( + id.cast(), + shortName, + paramsSignature, + parentId, + sourceDeclaration.cast() + ) + } else { + val shortNames = getFunctionShortName(f) + val methodId = id.cast() + extractMethod( + methodId, + locId, + shortNames.nameInDB, + substReturnType, + paramsSignature, + parentId, + sourceDeclaration.cast(), + if (extractOrigin) f.origin else null, + extractMethodAndParameterTypeAccesses + ) + + if (shortNames.nameInDB != shortNames.kotlinName) { + tw.writeKtFunctionOriginalNames(methodId, shortNames.kotlinName) + } + + if (f.hasInterfaceParent() && f.body != null) { + addModifiers( + methodId, + "default" + ) // The actual output class file may or may not have this modifier, + // depending on the -Xjvm-default setting. + } + } + + tw.writeHasLocation(id, locId) + val body = f.body + if (body != null && extractBody) { + if (typeSubstitution != null) + logger.errorElement( + "Type substitution should only be used to extract a function prototype, not the body", + f + ) + extractBody(body, id) + } + + extractVisibility(f, id, overriddenAttributes?.visibility ?: f.visibility) + + if (f.isInline) { + addModifiers(id, "inline") + } + if (f.shouldExtractAsStatic) { + addModifiers(id, "static") + } + if (f is IrSimpleFunction && f.overriddenSymbols.isNotEmpty()) { + addModifiers(id, "override") + } + if (f.isSuspend) { + addModifiers(id, "suspend") + } + if (f.symbol !is IrConstructorSymbol) { + when (overriddenAttributes?.modality ?: (f as? IrSimpleFunction)?.modality) { + Modality.ABSTRACT -> addModifiers(id, "abstract") + Modality.FINAL -> addModifiers(id, "final") + else -> Unit + } + } + + linesOfCode?.linesOfCodeInDeclaration(f, id) + + if (extractAnnotations) { + val extraAnnotations = + if (f.symbol is IrConstructorSymbol) listOf() + else + listOfNotNull( + getNullabilityAnnotation( + f.returnType, + f.origin, + f.annotations, + getJavaCallable(f)?.annotations + ) + ) + extractAnnotations( + f, + f.annotations + extraAnnotations, + id, + extractMethodAndParameterTypeAccesses + ) + } + + return id + } + } + } + + private fun isStaticFunction(f: IrFunction): Boolean { + return f.dispatchReceiverParameter == null // Has no dispatch receiver, + && + !f + .isLocalFunction() // not a local function. Local functions are extracted as + // instance methods with the local class instantiation as the + // qualifier + && + f.symbol !is IrConstructorSymbol // not a constructor + } + + private fun extractField( + f: IrField, + parentId: Label, + extractAnnotationEnumTypeAccesses: Boolean + ): Label { + with("field", f) { + DeclarationStackAdjuster(f).use { + val fNameSuffix = + getExtensionReceiverType(f)?.let { + it.classFqName?.asString()?.replace(".", "$$") + } ?: "" + val extractType = + if (isAnnotationClassField(f)) kClassToJavaClass(f.type) else f.type + val id = useField(f) + extractAnnotations(f, id, extractAnnotationEnumTypeAccesses) + return extractField( + id, + "${f.name.asString()}$fNameSuffix", + extractType, + parentId, + tw.getLocation(f), + f.visibility, + f, + isExternalDeclaration(f), + f.isFinal, + isDirectlyExposedCompanionObjectField(f) + ) + } + } + } + + private fun extractField( + id: Label, + name: String, + type: IrType, + parentId: Label, + locId: Label, + visibility: DescriptorVisibility, + errorElement: IrElement, + isExternalDeclaration: Boolean, + isFinal: Boolean, + isStatic: Boolean + ): Label { + val t = useType(type) + tw.writeFields(id, name, t.javaResult.id, parentId, id) + tw.writeFieldsKotlinType(id, t.kotlinResult.id) + tw.writeHasLocation(id, locId) + + extractVisibility(errorElement, id, visibility) + if (isFinal) { + addModifiers(id, "final") + } + if (isStatic) { + addModifiers(id, "static") + } + + if (!isExternalDeclaration) { + val fieldDeclarationId = tw.getFreshIdLabel() + tw.writeFielddecls(fieldDeclarationId, parentId) + tw.writeFieldDeclaredIn(id, fieldDeclarationId, 0) + tw.writeHasLocation(fieldDeclarationId, locId) + + extractTypeAccessRecursive(type, locId, fieldDeclarationId, 0) + } + + return id + } + + private fun extractProperty( + p: IrProperty, + parentId: Label, + extractBackingField: Boolean, + extractFunctionBodies: Boolean, + extractPrivateMembers: Boolean, + extractAnnotations: Boolean, + typeSubstitution: TypeSubstitution?, + classTypeArgsIncludingOuterClasses: List? + ) { + with("property", p) { + fun needsInterfaceForwarderQ(f: IrFunction?) = + f?.let { needsInterfaceForwarder(f) } ?: false + + if ( + isFake(p) && + !needsInterfaceForwarderQ(p.getter) && + !needsInterfaceForwarderQ(p.setter) + ) + return + + DeclarationStackAdjuster(p).use { + val id = useProperty(p, parentId, classTypeArgsIncludingOuterClasses) + val locId = getLocation(p, classTypeArgsIncludingOuterClasses) + tw.writeKtProperties(id, p.name.asString()) + tw.writeHasLocation(id, locId) + + val bf = p.backingField + val getter = p.getter + val setter = p.setter + + if (getter == null) { + if (!isExternalDeclaration(p)) { + logger.warnElement("IrProperty without a getter", p) + } + } else if (shouldExtractDecl(getter, extractPrivateMembers)) { + val getterId = + extractFunction( + getter, + parentId, + extractBody = extractFunctionBodies, + extractMethodAndParameterTypeAccesses = extractFunctionBodies, + extractAnnotations = extractAnnotations, + typeSubstitution, + classTypeArgsIncludingOuterClasses + ) + ?.cast() + if (getterId != null) { + tw.writeKtPropertyGetters(id, getterId) + if (getter.origin == IrDeclarationOrigin.DELEGATED_PROPERTY_ACCESSOR) { + tw.writeCompiler_generated( + getterId, + CompilerGeneratedKinds.DELEGATED_PROPERTY_GETTER.kind + ) + } + } + } + + if (setter == null) { + if (p.isVar && !isExternalDeclaration(p)) { + logger.warnElement("isVar property without a setter", p) + } + } else if (shouldExtractDecl(setter, extractPrivateMembers)) { + if (!p.isVar) { + logger.warnElement("!isVar property with a setter", p) + } + val setterId = + extractFunction( + setter, + parentId, + extractBody = extractFunctionBodies, + extractMethodAndParameterTypeAccesses = extractFunctionBodies, + extractAnnotations = extractAnnotations, + typeSubstitution, + classTypeArgsIncludingOuterClasses + ) + ?.cast() + if (setterId != null) { + tw.writeKtPropertySetters(id, setterId) + if (setter.origin == IrDeclarationOrigin.DELEGATED_PROPERTY_ACCESSOR) { + tw.writeCompiler_generated( + setterId, + CompilerGeneratedKinds.DELEGATED_PROPERTY_SETTER.kind + ) + } + } + } + + if (bf != null && extractBackingField) { + val fieldParentId = useDeclarationParentOf(bf, false) + if (fieldParentId != null) { + val fieldId = extractField(bf, fieldParentId.cast(), extractFunctionBodies) + tw.writeKtPropertyBackingFields(id, fieldId) + if (p.isDelegated) { + tw.writeKtPropertyDelegates(id, fieldId) + } + } + } + + extractVisibility(p, id, p.visibility) + + // TODO: extract annotations + + if (p.isLateinit) { + addModifiers(id, "lateinit") + } + } + } + } + + private fun getEnumEntryType(ee: IrEnumEntry): TypeResults? { + val parent = ee.parent + if (parent !is IrClass) { + logger.errorElement("Enum entry with unexpected parent: " + parent.javaClass, ee) + return null + } else if (parent.typeParameters.isNotEmpty()) { + logger.errorElement("Enum entry parent class has type parameters: " + parent.name, ee) + return null + } else { + return useSimpleTypeClass(parent, emptyList(), false) + } + } + + private fun extractEnumEntry( + ee: IrEnumEntry, + parentId: Label, + extractPrivateMembers: Boolean, + extractFunctionBodies: Boolean + ) { + with("enum entry", ee) { + DeclarationStackAdjuster(ee).use { + val id = useEnumEntry(ee) + val type = getEnumEntryType(ee) ?: return + tw.writeFields(id, ee.name.asString(), type.javaResult.id, parentId, id) + tw.writeFieldsKotlinType(id, type.kotlinResult.id) + val locId = tw.getLocation(ee) + tw.writeHasLocation(id, locId) + tw.writeIsEnumConst(id) + + if (extractFunctionBodies) { + val fieldDeclarationId = tw.getFreshIdLabel() + tw.writeFielddecls(fieldDeclarationId, parentId) + tw.writeFieldDeclaredIn(id, fieldDeclarationId, 0) + tw.writeHasLocation(fieldDeclarationId, locId) + + extractTypeAccess(type, locId, fieldDeclarationId, 0) + } + + ee.correspondingClass?.let { + extractDeclaration( + it, + extractPrivateMembers, + extractFunctionBodies, + extractAnnotations = true + ) + } + + extractAnnotations(ee, id, extractFunctionBodies) + } + } + } + + private fun extractTypeAlias(ta: IrTypeAlias) { + with("type alias", ta) { + if (ta.typeParameters.isNotEmpty()) { + // TODO: Extract this information + return + } + val id = useTypeAlias(ta) + val locId = tw.getLocation(ta) + // TODO: We don't really want to generate any Java types here; we only want the KT type: + val type = useType(ta.expandedType) + tw.writeKt_type_alias(id, ta.name.asString(), type.kotlinResult.id) + tw.writeHasLocation(id, locId) + + // TODO: extract annotations + } + } + + private fun extractBody(b: IrBody, callable: Label) { + with("body", b) { + when (b) { + is IrBlockBody -> extractBlockBody(b, callable) + is IrSyntheticBody -> extractSyntheticBody(b, callable) + is IrExpressionBody -> extractExpressionBody(b, callable) + else -> { + logger.errorElement("Unrecognised IrBody: " + b.javaClass, b) + } + } + } + } + + private fun extractBlockBody(callable: Label, locId: Label) = + tw.getFreshIdLabel().also { + tw.writeStmts_block(it, callable, 0, callable) + tw.writeHasLocation(it, locId) + } + + private fun extractBlockBody(b: IrBlockBody, callable: Label) { + with("block body", b) { + extractBlockBody(callable, tw.getLocation(b)).also { + for ((sIdx, stmt) in b.statements.withIndex()) { + extractStatement(stmt, callable, it, sIdx) + } + } + } + } + + private fun extractSyntheticBody(b: IrSyntheticBody, callable: Label) { + with("synthetic body", b) { + val kind = b.kind + when { + kind == IrSyntheticBodyKind.ENUM_VALUES -> tw.writeKtSyntheticBody(callable, 1) + kind == IrSyntheticBodyKind.ENUM_VALUEOF -> tw.writeKtSyntheticBody(callable, 2) + kind == kind_ENUM_ENTRIES -> tw.writeKtSyntheticBody(callable, 3) + else -> { + logger.errorElement("Unhandled synthetic body kind " + kind, b) + } + } + } + } + + private fun extractExpressionBody(b: IrExpressionBody, callable: Label) { + with("expression body", b) { + val locId = tw.getLocation(b) + extractExpressionBody(callable, locId).also { returnId -> + extractExpressionExpr(b.expression, callable, returnId, 0, returnId) + } + } + } + + fun extractExpressionBody( + callable: Label, + locId: Label + ): Label { + val blockId = extractBlockBody(callable, locId) + return tw.getFreshIdLabel().also { returnId -> + tw.writeStmts_returnstmt(returnId, blockId, 0, callable) + tw.writeHasLocation(returnId, locId) + } + } + + private fun getVariableLocationProvider(v: IrVariable): IrElement { + val init = v.initializer + if (v.startOffset < 0 && init != null) { + // IR_TEMPORARY_VARIABLEs have no proper location + return init + } + + return v + } + + private fun extractVariable( + v: IrVariable, + callable: Label, + parent: Label, + idx: Int + ) { + with("variable", v) { + val stmtId = tw.getFreshIdLabel() + val locId = tw.getLocation(getVariableLocationProvider(v)) + tw.writeStmts_localvariabledeclstmt(stmtId, parent, idx, callable) + tw.writeHasLocation(stmtId, locId) + extractVariableExpr(v, callable, stmtId, 1, stmtId) + } + } + + private fun extractVariableExpr( + v: IrVariable, + callable: Label, + parent: Label, + idx: Int, + enclosingStmt: Label, + extractInitializer: Boolean = true + ) { + with("variable expr", v) { + val varId = useVariable(v) + val exprId = tw.getFreshIdLabel() + val locId = tw.getLocation(getVariableLocationProvider(v)) + val type = useType(v.type) + tw.writeLocalvars(varId, v.name.asString(), type.javaResult.id, exprId) + tw.writeLocalvarsKotlinType(varId, type.kotlinResult.id) + tw.writeHasLocation(varId, locId) + tw.writeExprs_localvariabledeclexpr(exprId, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(exprId, type.kotlinResult.id) + extractExprContext(exprId, locId, callable, enclosingStmt) + val i = v.initializer + if (i != null && extractInitializer) { + extractExpressionExpr(i, callable, exprId, 0, enclosingStmt) + } + if (!v.isVar) { + addModifiers(varId, "final") + } + if (v.isLateinit) { + addModifiers(varId, "lateinit") + } + } + } + + private fun extractIfStmt( + locId: Label, + parent: Label, + idx: Int, + callable: Label + ) = + tw.getFreshIdLabel().also { + tw.writeStmts_ifstmt(it, parent, idx, callable) + tw.writeHasLocation(it, locId) + } + + private fun extractStatement( + s: IrStatement, + callable: Label, + parent: Label, + idx: Int + ) { + with("statement", s) { + when (s) { + is IrExpression -> { + extractExpressionStmt(s, callable, parent, idx) + } + is IrVariable -> { + extractVariable(s, callable, parent, idx) + } + is IrClass -> { + extractLocalTypeDeclStmt(s, callable, parent, idx) + } + is IrFunction -> { + if (s.isLocalFunction()) { + val compilerGeneratedKindOverride = + if (s.origin == IrDeclarationOrigin.ADAPTER_FOR_CALLABLE_REFERENCE) { + CompilerGeneratedKinds.DECLARING_CLASSES_OF_ADAPTER_FUNCTIONS + } else { + null + } + val classId = + extractGeneratedClass( + s, + listOf(pluginContext.irBuiltIns.anyType), + compilerGeneratedKindOverride = compilerGeneratedKindOverride + ) + extractLocalTypeDeclStmt(classId, s, callable, parent, idx) + val ids = getLocallyVisibleFunctionLabels(s) + tw.writeKtLocalFunction(ids.function) + } else { + logger.errorElement("Expected to find local function", s) + } + } + is IrLocalDelegatedProperty -> { + val blockId = tw.getFreshIdLabel() + val locId = tw.getLocation(s) + tw.writeStmts_block(blockId, parent, idx, callable) + tw.writeHasLocation(blockId, locId) + extractVariable(s.delegate, callable, blockId, 0) + + val propId = tw.getFreshIdLabel() + tw.writeKtProperties(propId, s.name.asString()) + tw.writeHasLocation(propId, locId) + tw.writeKtPropertyDelegates(propId, useVariable(s.delegate)) + + // Getter: + extractStatement(s.getter, callable, blockId, 1) + val getterLabel = getLocallyVisibleFunctionLabels(s.getter).function + tw.writeKtPropertyGetters(propId, getterLabel) + + val setter = s.setter + if (setter != null) { + extractStatement(setter, callable, blockId, 2) + val setterLabel = getLocallyVisibleFunctionLabels(setter).function + tw.writeKtPropertySetters(propId, setterLabel) + } + } + else -> { + logger.errorElement("Unrecognised IrStatement: " + s.javaClass, s) + } + } + } + } + + /** + * Returns true iff `c` is a call to the function `fName` in the `kotlin.internal.ir` package. + * This is used to find calls to builtin functions, which need to be handled specially as they + * do not have corresponding source definitions. + */ + private fun isBuiltinCallInternal(c: IrCall, fName: String) = + isBuiltinCall(c, fName, "kotlin.internal.ir") + /** + * Returns true iff `c` is a call to the function `fName` in the `kotlin` package. This is used + * to find calls to builtin functions, which need to be handled specially as they do not have + * corresponding source definitions. + */ + private fun isBuiltinCallKotlin(c: IrCall, fName: String) = isBuiltinCall(c, fName, "kotlin") + + /** + * Returns true iff `c` is a call to the function `fName` in package `pName`. This is used to + * find calls to builtin functions, which need to be handled specially as they do not have + * corresponding source definitions. + */ + private fun isBuiltinCall(c: IrCall, fName: String, pName: String): Boolean { + val verbose = false + fun verboseln(s: String) { + if (verbose) println(s) + } + verboseln("Attempting builtin match for $fName") + val target = c.symbol.owner + if (target.name.asString() != fName) { + verboseln("No match as function name is ${target.name.asString()} not $fName") + return false + } + + val targetPkg = target.parent + if (targetPkg !is IrPackageFragment) { + verboseln("No match as didn't find target package") + return false + } + val targetName = targetPkg.packageFqName.asString() + if (targetName != pName) { + verboseln("No match as package name is $targetName") + return false + } + verboseln("Match") + return true + } + + private fun unaryOp( + id: Label, + c: IrCall, + callable: Label, + enclosingStmt: Label + ) { + val locId = tw.getLocation(c) + extractExprContext(id, locId, callable, enclosingStmt) + + val dr = c.dispatchReceiver + if (dr != null) { + logger.errorElement("Unexpected dispatch receiver found", c) + } + + if (c.valueArgumentsCount < 1) { + logger.errorElement("No arguments found", c) + return + } + + extractArgument(id, c, callable, enclosingStmt, 0, "Operand null") + + if (c.valueArgumentsCount > 1) { + logger.errorElement("Extra arguments found", c) + } + } + + private fun binOp( + id: Label, + c: IrCall, + callable: Label, + enclosingStmt: Label + ) { + val locId = tw.getLocation(c) + extractExprContext(id, locId, callable, enclosingStmt) + + val dr = c.dispatchReceiver + if (dr != null) { + logger.errorElement("Unexpected dispatch receiver found", c) + } + + if (c.valueArgumentsCount < 1) { + logger.errorElement("No arguments found", c) + return + } + + extractArgument(id, c, callable, enclosingStmt, 0, "LHS null") + + if (c.valueArgumentsCount < 2) { + logger.errorElement("No RHS found", c) + return + } + + extractArgument(id, c, callable, enclosingStmt, 1, "RHS null") + + if (c.valueArgumentsCount > 2) { + logger.errorElement("Extra arguments found", c) + } + } + + private fun extractArgument( + id: Label, + c: IrCall, + callable: Label, + enclosingStmt: Label, + idx: Int, + msg: String + ) { + val op = c.getValueArgument(idx) + if (op == null) { + logger.errorElement(msg, c) + } else { + extractExpressionExpr(op, callable, id, idx, enclosingStmt) + } + } + + private fun getDeclaringTypeArguments( + callTarget: IrFunction, + receiverType: IrSimpleType + ): List { + val declaringType = callTarget.parentAsClass + val receiverClass = receiverType.classifier.owner as? IrClass ?: return listOf() + val ancestorTypes = ArrayList() + + // KFunctionX doesn't implement FunctionX on versions before 1.7.0: + if ( + (callTarget.name.asString() == "invoke") && + (receiverClass.fqNameWhenAvailable + ?.asString() + ?.startsWith("kotlin.reflect.KFunction") == true) && + (callTarget.parentClassOrNull + ?.fqNameWhenAvailable + ?.asString() + ?.startsWith("kotlin.Function") == true) + ) { + return receiverType.arguments + } + + // Populate ancestorTypes with the path from receiverType's class to its ancestor, + // callTarget's declaring type. + fun walkFrom(c: IrClass): Boolean { + if (declaringType == c) return true + else { + c.superTypes.forEach { + val ancestorClass = + (it as? IrSimpleType)?.classifier?.owner as? IrClass ?: return false + ancestorTypes.add(it) + if (walkFrom(ancestorClass)) return true else ancestorTypes.pop() + } + return false + } + } + + // If a path was found, repeatedly substitute types to get the corresponding specialisation + // of that ancestor. + if (!walkFrom(receiverClass)) { + logger.errorElement( + "Failed to find a class declaring ${callTarget.name} starting at ${receiverClass.name}", + callTarget + ) + return listOf() + } else { + var subbedType: IrSimpleType = receiverType + ancestorTypes.forEach { + val thisClass = subbedType.classifier.owner + if (thisClass !is IrClass) { + logger.errorElement( + "Found ancestor with unexpected type ${thisClass.javaClass}", + callTarget + ) + return listOf() + } + val itSubbed = + it.substituteTypeArguments(thisClass.typeParameters, subbedType.arguments) + if (itSubbed !is IrSimpleType) { + logger.errorElement( + "Substituted type has unexpected type ${itSubbed.javaClass}", + callTarget + ) + return listOf() + } + subbedType = itSubbed + } + return subbedType.arguments + } + } + + private fun extractNewExprForLocalFunction( + ids: LocallyVisibleFunctionLabels, + parent: Label, + locId: Label, + enclosingCallable: Label, + enclosingStmt: Label + ) { + + val idNewexpr = + extractNewExpr( + ids.constructor, + ids.type, + locId, + parent, + -1, + enclosingCallable, + enclosingStmt + ) + extractTypeAccessRecursive( + pluginContext.irBuiltIns.anyType, + locId, + idNewexpr, + -3, + enclosingCallable, + enclosingStmt + ) + } + + private fun extractMethodAccessWithoutArgs( + returnType: IrType, + locId: Label, + enclosingCallable: Label, + callsiteParent: Label, + childIdx: Int, + enclosingStmt: Label, + methodLabel: Label? + ) = + tw.getFreshIdLabel().also { id -> + val type = useType(returnType) + + tw.writeExprs_methodaccess(id, type.javaResult.id, callsiteParent, childIdx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, enclosingCallable, enclosingStmt) + + // The caller should have warned about this before, so we don't repeat the warning here. + if (methodLabel != null) tw.writeCallableBinding(id, methodLabel) + } + + private val defaultConstructorMarkerClass by lazy { + referenceExternalClass("kotlin.jvm.internal.DefaultConstructorMarker") + } + + private val defaultConstructorMarkerType by lazy { defaultConstructorMarkerClass?.typeWith() } + + private fun getDefaultsMethodLastArgType(f: IrFunction) = + (if (f is IrConstructor) defaultConstructorMarkerType else null) + ?: pluginContext.irBuiltIns.anyType + + private fun getDefaultsMethodArgTypes(f: IrFunction) = + // The $default method has type ([dispatchReceiver], [extensionReceiver], paramTypes..., + // int, Object) + // All parameter types are erased. The trailing int is a mask indicating which parameter + // values are real + // and which should be replaced by defaults. The final Object parameter is apparently always + // null. + (listOfNotNull(if (f.shouldExtractAsStatic) null else f.dispatchReceiverParameter?.type) + + listOfNotNull(f.extensionReceiverParameter?.type) + + f.valueParameters.map { it.type } + + listOf(pluginContext.irBuiltIns.intType, getDefaultsMethodLastArgType(f))) + .map { erase(it) } + + private fun getDefaultsMethodName(f: IrFunction) = + if (f is IrConstructor) { + f.returnType.let { + when { + it.isAnonymous -> "" + else -> it.classFqName?.shortName()?.asString() ?: f.name.asString() + } + } + } else { + getFunctionShortName(f).nameInDB + "\$default" + } + + private fun getDefaultsMethodLabel(f: IrFunction): Label? { + val classTypeArgsIncludingOuterClasses = null + val parentId = useDeclarationParentOf(f, false, classTypeArgsIncludingOuterClasses, true) + if (parentId == null) { + logger.errorElement("Couldn't get parent ID for defaults method", f) + return null + } + return getDefaultsMethodLabel(f, parentId) + } + + private fun getDefaultsMethodLabel( + f: IrFunction, + parentId: Label + ): Label { + val defaultsMethodName = if (f is IrConstructor) "" else getDefaultsMethodName(f) + val argTypes = getDefaultsMethodArgTypes(f) + + val defaultMethodLabelStr = + getFunctionLabel( + f.parent, + parentId, + defaultsMethodName, + argTypes, + erase(f.returnType), + extensionParamType = null, // if there's any, that's included already in argTypes + functionTypeParameters = listOf(), + classTypeArgsIncludingOuterClasses = null, + overridesCollectionsMethod = false, + javaSignature = null, + addParameterWildcardsByDefault = false + ) + + return tw.getLabelFor(defaultMethodLabelStr) + } + + private fun extractsDefaultsCall( + syntacticCallTarget: IrFunction, + locId: Label, + resultType: IrType, + enclosingCallable: Label, + callsiteParent: Label, + childIdx: Int, + enclosingStmt: Label, + valueArguments: List, + dispatchReceiver: IrExpression?, + extensionReceiver: IrExpression? + ) { + val callTarget = syntacticCallTarget.target.realOverrideTarget + if (isExternalDeclaration(callTarget)) { + // Ensure the real target gets extracted, as we might not every directly touch it thanks + // to this call being redirected to a $default method. + useFunction(callTarget) + } + + // Default parameter values are inherited by overrides; in this case the call should + // dispatch against the $default method belonging to the class + // that specified the default values, which will in turn dynamically dispatch back to the + // relevant override. + val overriddenCallTarget = + (callTarget as? IrSimpleFunction)?.allOverriddenIncludingSelf()?.firstOrNull { + it.overriddenSymbols.isEmpty() && + it.valueParameters.any { p -> p.defaultValue != null } + } ?: callTarget + if (isExternalDeclaration(overriddenCallTarget)) { + // Likewise, ensure the overridden target gets extracted. + useFunction(overriddenCallTarget) + } + + val defaultMethodLabel = getDefaultsMethodLabel(overriddenCallTarget) + val id = + extractMethodAccessWithoutArgs( + resultType, + locId, + enclosingCallable, + callsiteParent, + childIdx, + enclosingStmt, + defaultMethodLabel + ) + + if (overriddenCallTarget.isLocalFunction()) { + extractTypeAccess( + getLocallyVisibleFunctionLabels(overriddenCallTarget).type, + locId, + id, + -1, + enclosingCallable, + enclosingStmt + ) + } else { + extractStaticTypeAccessQualifierUnchecked( + overriddenCallTarget, + id, + locId, + enclosingCallable, + enclosingStmt + ) + } + + extractDefaultsCallArguments( + id, + overriddenCallTarget, + enclosingCallable, + enclosingStmt, + valueArguments, + dispatchReceiver, + extensionReceiver + ) + } + + private fun extractDefaultsCallArguments( + id: Label, + callTarget: IrFunction, + enclosingCallable: Label, + enclosingStmt: Label, + valueArguments: List, + dispatchReceiver: IrExpression?, + extensionReceiver: IrExpression? + ) { + var nextIdx = 0 + if (dispatchReceiver != null && !callTarget.shouldExtractAsStatic) { + extractExpressionExpr(dispatchReceiver, enclosingCallable, id, nextIdx++, enclosingStmt) + } + + if (extensionReceiver != null) { + extractExpressionExpr( + extensionReceiver, + enclosingCallable, + id, + nextIdx++, + enclosingStmt + ) + } + + val valueArgsWithDummies = + valueArguments.zip(callTarget.valueParameters).map { (expr, param) -> + expr ?: IrConstImpl.defaultValueForType(0, 0, param.type) + } + + var realParamsMask = 0 + valueArguments.forEachIndexed { index, arg -> + if (arg != null) realParamsMask = realParamsMask or (1 shl index) + } + + val extraArgs = + listOf( + IrConstImpl.int(0, 0, pluginContext.irBuiltIns.intType, realParamsMask), + IrConstImpl.defaultValueForType(0, 0, getDefaultsMethodLastArgType(callTarget)) + ) + + extractCallValueArguments( + id, + valueArgsWithDummies + extraArgs, + enclosingStmt, + enclosingCallable, + nextIdx, + extractVarargAsArray = true + ) + } + + private fun getFunctionInvokeMethod(typeArgs: List): IrFunction? { + // For `kotlin.FunctionX` and `kotlin.reflect.KFunctionX` interfaces, we're making sure that + // we + // extract the call to the `invoke` method that does exist, + // `kotlin.jvm.functions.FunctionX::invoke`. + val functionalInterface = getFunctionalInterfaceTypeWithTypeArgs(typeArgs) + if (functionalInterface == null) { + logger.warn("Cannot find functional interface type for raw method access") + return null + } + val functionalInterfaceClass = functionalInterface.classOrNull + if (functionalInterfaceClass == null) { + logger.warn("Cannot find functional interface class for raw method access") + return null + } + val interfaceType = functionalInterfaceClass.owner + val substituted = getJavaEquivalentClass(interfaceType) ?: interfaceType + val function = findFunction(substituted, OperatorNameConventions.INVOKE.asString()) + if (function == null) { + logger.warn("Cannot find invoke function for raw method access") + return null + } + return function + } + + private fun isFunctionInvoke(callTarget: IrFunction, drType: IrSimpleType) = + (drType.isFunctionOrKFunction() || drType.isSuspendFunctionOrKFunction()) && + callTarget.name.asString() == OperatorNameConventions.INVOKE.asString() + + private fun getCalleeMethodId( + callTarget: IrFunction, + drType: IrType?, + allowInstantiatedGenericMethod: Boolean + ): Label? { + if (callTarget.isLocalFunction()) + return getLocallyVisibleFunctionLabels(callTarget).function + + if ( + allowInstantiatedGenericMethod && + drType is IrSimpleType && + !isUnspecialised(drType, logger) + ) { + val calleeIsInvoke = isFunctionInvoke(callTarget, drType) + + val extractionMethod = + if (calleeIsInvoke) getFunctionInvokeMethod(drType.arguments) else callTarget + + return extractionMethod?.let { + val typeArgs = + if (calleeIsInvoke && drType.arguments.size > BuiltInFunctionArity.BIG_ARITY) { + // Big arity `invoke` methods have a special implementation on JVM, they are + // transformed to a call to + // `kotlin.jvm.functions.FunctionN::invoke(vararg args: Any?)`, so we + // only need to pass the type + // argument for the return type. Additionally, the arguments are extracted + // inside an array literal below. + listOf(drType.arguments.last()) + } else { + getDeclaringTypeArguments(callTarget, drType) + } + useFunction(extractionMethod, typeArgs) + } + } else { + return useFunction(callTarget) + } + } + + private fun getCalleeRealOverrideTarget(f: IrFunction): IrFunction { + val target = f.target.realOverrideTarget + return if (overridesCollectionsMethodWithAlteredParameterTypes(f)) + // Cope with the case where an inherited callee can be rewritten with substituted parameter + // types + // if the child class uses it to implement a collections interface + // (for example, `class A { boolean contains(Object o) { ... } }; class B extends A + // implements Set { ... }` + // leads to generating a function `A.contains(B::T)`, with `initialSignatureFunction` + // pointing to `A.contains(Object)`. + (target as? IrLazyFunction)?.initialSignatureFunction ?: target + else target + } + + private fun callUsesDefaultArguments( + callTarget: IrFunction, + valueArguments: List + ): Boolean { + val varargParam = callTarget.valueParameters.withIndex().find { it.value.isVararg } + // If the vararg param is the only one not specified, and it has no default value, then we + // don't need to call a $default method, + // as omitting it already implies passing an empty vararg array. + val nullAllowedIdx = + if (varargParam != null && varargParam.value.defaultValue == null) varargParam.index + else -1 + return valueArguments.withIndex().any { (index, it) -> + it == null && index != nullAllowedIdx + } + } + + fun extractRawMethodAccess( + syntacticCallTarget: IrFunction, + locElement: IrElement, + resultType: IrType, + enclosingCallable: Label, + callsiteParent: Label, + childIdx: Int, + enclosingStmt: Label, + valueArguments: List, + dispatchReceiver: IrExpression?, + extensionReceiver: IrExpression?, + typeArguments: List = listOf(), + extractClassTypeArguments: Boolean = false, + superQualifierSymbol: IrClassSymbol? = null + ) { + + val locId = tw.getLocation(locElement) + + if (callUsesDefaultArguments(syntacticCallTarget, valueArguments)) { + extractsDefaultsCall( + syntacticCallTarget, + locId, + resultType, + enclosingCallable, + callsiteParent, + childIdx, + enclosingStmt, + valueArguments, + dispatchReceiver, + extensionReceiver + ) + } else { + extractRawMethodAccess( + syntacticCallTarget, + locId, + resultType, + enclosingCallable, + callsiteParent, + childIdx, + enclosingStmt, + valueArguments.size, + { argParent, idxOffset -> + extractCallValueArguments( + argParent, + valueArguments, + enclosingStmt, + enclosingCallable, + idxOffset + ) + }, + dispatchReceiver?.type, + dispatchReceiver?.let { + { callId -> + extractExpressionExpr( + dispatchReceiver, + enclosingCallable, + callId, + -1, + enclosingStmt + ) + } + }, + extensionReceiver?.let { + { argParent -> + extractExpressionExpr( + extensionReceiver, + enclosingCallable, + argParent, + 0, + enclosingStmt + ) + } + }, + typeArguments, + extractClassTypeArguments, + superQualifierSymbol + ) + } + } + + fun extractRawMethodAccess( + syntacticCallTarget: IrFunction, + locId: Label, + returnType: IrType, + enclosingCallable: Label, + callsiteParent: Label, + childIdx: Int, + enclosingStmt: Label, + nValueArguments: Int, + extractValueArguments: (Label, Int) -> Unit, + drType: IrType?, + extractDispatchReceiver: ((Label) -> Unit)?, + extractExtensionReceiver: ((Label) -> Unit)?, + typeArguments: List = listOf(), + extractClassTypeArguments: Boolean = false, + superQualifierSymbol: IrClassSymbol? = null + ) { + + val callTarget = getCalleeRealOverrideTarget(syntacticCallTarget) + val methodId = getCalleeMethodId(callTarget, drType, extractClassTypeArguments) + if (methodId == null) { + logger.warn("No method to bind call to for raw method access") + } + + val id = + extractMethodAccessWithoutArgs( + returnType, + locId, + enclosingCallable, + callsiteParent, + childIdx, + enclosingStmt, + methodId + ) + + // type arguments at index -2, -3, ... + extractTypeArguments(typeArguments, locId, id, enclosingCallable, enclosingStmt, -2, true) + + if (callTarget.isLocalFunction()) { + extractNewExprForLocalFunction( + getLocallyVisibleFunctionLabels(callTarget), + id, + locId, + enclosingCallable, + enclosingStmt + ) + } else if (callTarget.shouldExtractAsStatic) { + extractStaticTypeAccessQualifier( + callTarget, + id, + locId, + enclosingCallable, + enclosingStmt + ) + } else if (superQualifierSymbol != null) { + extractSuperAccess( + superQualifierSymbol.typeWith(), + enclosingCallable, + id, + -1, + enclosingStmt, + locId + ) + } else if (extractDispatchReceiver != null) { + extractDispatchReceiver(id) + } + + val idxOffset = if (extractExtensionReceiver != null) 1 else 0 + + val isBigArityFunctionInvoke = + drType is IrSimpleType && + isFunctionInvoke(callTarget, drType) && + drType.arguments.size > BuiltInFunctionArity.BIG_ARITY + + val argParent = + if (isBigArityFunctionInvoke) { + extractArrayCreationWithInitializer( + id, + nValueArguments + idxOffset, + locId, + enclosingCallable, + enclosingStmt + ) + } else { + id + } + + if (extractExtensionReceiver != null) { + extractExtensionReceiver(argParent) + } + + extractValueArguments(argParent, idxOffset) + } + + private fun extractStaticTypeAccessQualifierUnchecked( + target: IrDeclaration, + parentExpr: Label, + locId: Label, + enclosingCallable: Label?, + enclosingStmt: Label? + ) { + val parent = target.parent + if (parent is IrExternalPackageFragment) { + // This is in a file class. + val fqName = getFileClassFqName(target) + if (fqName == null) { + logger.error( + "Can't get FqName for static type access qualifier in external package fragment ${target.javaClass}" + ) + } else { + extractTypeAccess( + useFileClassType(fqName), + locId, + parentExpr, + -1, + enclosingCallable, + enclosingStmt + ) + } + } else if (parent is IrClass) { + extractTypeAccessRecursive( + parent.toRawType(), + locId, + parentExpr, + -1, + enclosingCallable, + enclosingStmt + ) + } else if (parent is IrFile) { + extractTypeAccess( + useFileClassType(parent), + locId, + parentExpr, + -1, + enclosingCallable, + enclosingStmt + ) + } else { + logger.warnElement( + "Unexpected static type access qualifier ${parent.javaClass}", + parent + ) + } + } + + private fun extractStaticTypeAccessQualifier( + target: IrDeclaration, + parentExpr: Label, + locId: Label, + enclosingCallable: Label?, + enclosingStmt: Label? + ) { + if (target.shouldExtractAsStatic) { + extractStaticTypeAccessQualifierUnchecked( + target, + parentExpr, + locId, + enclosingCallable, + enclosingStmt + ) + } + } + + private fun isStaticAnnotatedNonCompanionMember(f: IrSimpleFunction) = + f.parentClassOrNull?.isNonCompanionObject == true && + (f.hasAnnotation(jvmStaticFqName) || + f.correspondingPropertySymbol?.owner?.hasAnnotation(jvmStaticFqName) == true) + + private val IrDeclaration.shouldExtractAsStatic: Boolean + get() = + this is IrSimpleFunction && + (isStaticFunction(this) || isStaticAnnotatedNonCompanionMember(this)) || + this is IrField && this.isStatic || + this is IrEnumEntry + + private fun extractCallValueArguments( + callId: Label, + call: IrFunctionAccessExpression, + enclosingStmt: Label, + enclosingCallable: Label, + idxOffset: Int + ) = + extractCallValueArguments( + callId, + (0 until call.valueArgumentsCount).map { call.getValueArgument(it) }, + enclosingStmt, + enclosingCallable, + idxOffset + ) + + private fun extractCallValueArguments( + callId: Label, + valueArguments: List, + enclosingStmt: Label, + enclosingCallable: Label, + idxOffset: Int, + extractVarargAsArray: Boolean = false + ) { + var i = 0 + valueArguments.forEach { arg -> + if (arg != null) { + if (arg is IrVararg && !extractVarargAsArray) { + arg.elements.forEachIndexed { varargNo, vararg -> + extractVarargElement( + vararg, + enclosingCallable, + callId, + i + idxOffset + varargNo, + enclosingStmt + ) + } + i += arg.elements.size + } else { + extractExpressionExpr( + arg, + enclosingCallable, + callId, + (i++) + idxOffset, + enclosingStmt + ) + } + } + } + } + + private fun findFunction(cls: IrClass, name: String): IrFunction? = + cls.declarations.findSubType { it.name.asString() == name } + + val jvmIntrinsicsClass by lazy { referenceExternalClass("kotlin.jvm.internal.Intrinsics") } + + private fun findJdkIntrinsicOrWarn(name: String, warnAgainstElement: IrElement): IrFunction? { + val result = jvmIntrinsicsClass?.let { findFunction(it, name) } + if (result == null) { + logger.errorElement("Couldn't find JVM intrinsic function $name", warnAgainstElement) + } + return result + } + + private fun findTopLevelFunctionOrWarn( + functionPkg: String, + functionName: String, + type: String, + parameterTypes: Array, + warnAgainstElement: IrElement + ): IrFunction? { + + val fn = + getFunctionsByFqName(pluginContext, functionPkg, functionName) + .firstOrNull { fnSymbol -> + val owner = fnSymbol.owner + (owner.parentClassOrNull?.fqNameWhenAvailable?.asString() == type || + (owner.parent is IrExternalPackageFragment && + getFileClassFqName(owner)?.asString() == type)) && + owner.valueParameters + .map { it.type.classFqName?.asString() } + .toTypedArray() contentEquals parameterTypes + } + ?.owner + + if (fn != null) { + if (fn.parentClassOrNull != null) { + extractExternalClassLater(fn.parentAsClass) + } + } else { + logger.errorElement( + "Couldn't find JVM intrinsic function $functionPkg $functionName in $type", + warnAgainstElement + ) + } + + return fn + } + + private fun findTopLevelPropertyOrWarn( + propertyPkg: String, + propertyName: String, + type: String, + warnAgainstElement: IrElement + ): IrProperty? { + + val prop = + getPropertiesByFqName(pluginContext, propertyPkg, propertyName) + .firstOrNull { it.owner.parentClassOrNull?.fqNameWhenAvailable?.asString() == type } + ?.owner + + if (prop != null) { + if (prop.parentClassOrNull != null) { + extractExternalClassLater(prop.parentAsClass) + } + } else { + logger.errorElement( + "Couldn't find JVM intrinsic property $propertyPkg $propertyName in $type", + warnAgainstElement + ) + } + + return prop + } + + val javaLangString by lazy { referenceExternalClass("java.lang.String") } + + val stringValueOfObjectMethod by lazy { + val result = + javaLangString?.declarations?.findSubType { + it.name.asString() == "valueOf" && + it.valueParameters.size == 1 && + it.valueParameters[0].type == pluginContext.irBuiltIns.anyNType + } + if (result == null) { + logger.error("Couldn't find declaration java.lang.String.valueOf(Object)") + } + result + } + + val objectCloneMethod by lazy { + val result = + javaLangObject?.declarations?.findSubType { it.name.asString() == "clone" } + if (result == null) { + logger.error("Couldn't find declaration java.lang.Object.clone(...)") + } + result + } + + val kotlinNoWhenBranchMatchedExn by lazy { + referenceExternalClass("kotlin.NoWhenBranchMatchedException") + } + + val kotlinNoWhenBranchMatchedConstructor by lazy { + val result = + kotlinNoWhenBranchMatchedExn?.declarations?.findSubType { + it.valueParameters.isEmpty() + } + if (result == null) { + logger.error("Couldn't find no-arg constructor for kotlin.NoWhenBranchMatchedException") + } + result + } + + val javaUtilArrays by lazy { referenceExternalClass("java.util.Arrays") } + + private fun isFunction( + target: IrFunction, + pkgName: String, + classNameLogged: String, + classNamePredicate: (String) -> Boolean, + vararg fNames: String, + isNullable: Boolean? = false + ) = + fNames.any { + isFunction(target, pkgName, classNameLogged, classNamePredicate, it, isNullable) + } + + private fun isFunction( + target: IrFunction, + pkgName: String, + classNameLogged: String, + classNamePredicate: (String) -> Boolean, + fName: String, + isNullable: Boolean? = false + ): Boolean { + val verbose = false + fun verboseln(s: String) { + if (verbose) println(s) + } + verboseln("Attempting match for $pkgName $classNameLogged $fName") + if (target.name.asString() != fName) { + verboseln("No match as function name is ${target.name.asString()} not $fName") + return false + } + val extensionReceiverParameter = target.extensionReceiverParameter + val targetClass = + if (extensionReceiverParameter == null) { + if (isNullable == true) { + verboseln( + "Nullablility of type didn't match (target is not an extension method)" + ) + return false + } + target.parent + } else { + val st = extensionReceiverParameter.type as? IrSimpleType + if (isNullable != null && st?.isNullable() != isNullable) { + verboseln("Nullablility of type didn't match") + return false + } + st?.classifier?.owner + } + if (targetClass !is IrClass) { + verboseln("No match as didn't find target class") + return false + } + if (!classNamePredicate(targetClass.name.asString())) { + verboseln( + "No match as class name is ${targetClass.name.asString()} not $classNameLogged" + ) + return false + } + val targetPkg = targetClass.parent + if (targetPkg !is IrPackageFragment) { + verboseln("No match as didn't find target package") + return false + } + val targetName = targetPkg.packageFqName.asString() + if (targetName != pkgName) { + verboseln("No match as package name is $targetName not $pkgName") + return false + } + verboseln("Match") + return true + } + + private fun isFunction( + target: IrFunction, + pkgName: String, + className: String, + fName: String, + isNullable: Boolean? = false + ) = isFunction(target, pkgName, className, { it == className }, fName, isNullable) + + private fun isNumericFunction(target: IrFunction, fName: String): Boolean { + return isFunction(target, "kotlin", "Int", fName) || + isFunction(target, "kotlin", "Byte", fName) || + isFunction(target, "kotlin", "Short", fName) || + isFunction(target, "kotlin", "Long", fName) || + isFunction(target, "kotlin", "Float", fName) || + isFunction(target, "kotlin", "Double", fName) + } + + private fun isNumericFunction(target: IrFunction, vararg fNames: String) = + fNames.any { isNumericFunction(target, it) } + + private fun isArrayType(typeName: String) = + when (typeName) { + "Array" -> true + "IntArray" -> true + "ByteArray" -> true + "ShortArray" -> true + "LongArray" -> true + "FloatArray" -> true + "DoubleArray" -> true + "CharArray" -> true + "BooleanArray" -> true + else -> false + } + + private fun isGenericArrayType(typeName: String) = + when (typeName) { + "Array" -> true + else -> false + } + + private fun extractCall( + c: IrCall, + callable: Label, + stmtExprParent: StmtExprParent + ) { + with("call", c) { + val owner = getBoundSymbolOwner(c.symbol, c) ?: return + val target = tryReplaceSyntheticFunction(owner) + + // The vast majority of types of call want an expr context, so make one available + // lazily: + val exprParent by lazy { stmtExprParent.expr(c, callable) } + + val parent by lazy { exprParent.parent } + + val idx by lazy { exprParent.idx } + + val enclosingStmt by lazy { exprParent.enclosingStmt } + + fun extractMethodAccess( + syntacticCallTarget: IrFunction, + extractMethodTypeArguments: Boolean = true, + extractClassTypeArguments: Boolean = false + ) { + val typeArgs = + if (extractMethodTypeArguments) + (0 until c.typeArgumentsCount) + .map { c.getTypeArgument(it) } + .requireNoNullsOrNull() + else listOf() + + if (typeArgs == null) { + logger.warn("Missing type argument in extractMethodAccess") + return + } + + extractRawMethodAccess( + syntacticCallTarget, + c, + c.type, + callable, + parent, + idx, + enclosingStmt, + (0 until c.valueArgumentsCount).map { c.getValueArgument(it) }, + c.dispatchReceiver, + c.extensionReceiver, + typeArgs, + extractClassTypeArguments, + c.superQualifierSymbol + ) + } + + fun extractSpecialEnumFunction(fnName: String) { + if (c.typeArgumentsCount != 1) { + logger.errorElement("Expected to find exactly one type argument", c) + return + } + + val enumType = (c.getTypeArgument(0) as? IrSimpleType)?.classifier?.owner + if (enumType == null) { + logger.errorElement("Couldn't find type of enum type", c) + return + } + + if (enumType is IrClass) { + val func = + enumType.declarations.findSubType { + it.name.asString() == fnName + } + if (func == null) { + logger.errorElement("Couldn't find function $fnName on enum type", c) + return + } + + extractMethodAccess(func, false) + } else if (enumType is IrTypeParameter && enumType.isReified) { + // A call to `enumValues()` is being extracted, where `T` is a reified type + // parameter of an `inline` function. + // We can't generate a valid expression here, because we would need to know the + // type of T on the call site. + // TODO: replace error expression with something that better shows this + // expression is unrepresentable. + val id = tw.getFreshIdLabel() + val type = useType(c.type) + + tw.writeExprs_errorexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, tw.getLocation(c), callable, enclosingStmt) + } else { + logger.errorElement("Unexpected enum type rep ${enumType.javaClass}", c) + } + } + + fun binopReceiver( + id: Label, + receiver: IrExpression?, + receiverDescription: String + ) { + extractExprContext(id, tw.getLocation(c), callable, enclosingStmt) + + if (receiver == null) { + logger.errorElement("$receiverDescription not found", c) + } else { + extractExpressionExpr(receiver, callable, id, 0, enclosingStmt) + } + if (c.valueArgumentsCount < 1) { + logger.errorElement("No RHS found", c) + } else { + if (c.valueArgumentsCount > 1) { + logger.errorElement("Extra arguments found", c) + } + val arg = c.getValueArgument(0) + if (arg == null) { + logger.errorElement("RHS null", c) + } else { + extractExpressionExpr(arg, callable, id, 1, enclosingStmt) + } + } + } + + fun unaryopReceiver( + id: Label, + receiver: IrExpression?, + receiverDescription: String + ) { + extractExprContext(id, tw.getLocation(c), callable, enclosingStmt) + + if (receiver == null) { + logger.errorElement("$receiverDescription not found", c) + } else { + extractExpressionExpr(receiver, callable, id, 0, enclosingStmt) + } + if (c.valueArgumentsCount > 0) { + logger.errorElement("Extra arguments found", c) + } + } + + /** + * Populate the lhs of a binary op from this call's dispatch receiver, and the rhs from + * its sole argument. + */ + fun binopDisp(id: Label) { + binopReceiver(id, c.dispatchReceiver, "Dispatch receiver") + } + + fun binopExt(id: Label) { + binopReceiver(id, c.extensionReceiver, "Extension receiver") + } + + fun unaryopDisp(id: Label) { + unaryopReceiver(id, c.dispatchReceiver, "Dispatch receiver") + } + + fun unaryopExt(id: Label) { + unaryopReceiver(id, c.extensionReceiver, "Extension receiver") + } + + val dr = c.dispatchReceiver + when { + isFunction(target, "kotlin", "String", "plus", false) -> { + val id = tw.getFreshIdLabel() + val type = useType(c.type) + tw.writeExprs_addexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + binopDisp(id) + } + isFunction(target, "kotlin", "String", "plus", true) -> { + findJdkIntrinsicOrWarn("stringPlus", c)?.let { stringPlusFn -> + extractRawMethodAccess( + stringPlusFn, + c, + c.type, + callable, + parent, + idx, + enclosingStmt, + listOf(c.extensionReceiver, c.getValueArgument(0)), + null, + null + ) + } + } + isNumericFunction( + target, + "plus", + "minus", + "times", + "div", + "rem", + "and", + "or", + "xor", + "shl", + "shr", + "ushr" + ) -> { + val type = useType(c.type) + val id: Label = + when (val targetName = target.name.asString()) { + "plus" -> { + val id = tw.getFreshIdLabel() + tw.writeExprs_addexpr(id, type.javaResult.id, parent, idx) + id + } + "minus" -> { + val id = tw.getFreshIdLabel() + tw.writeExprs_subexpr(id, type.javaResult.id, parent, idx) + id + } + "times" -> { + val id = tw.getFreshIdLabel() + tw.writeExprs_mulexpr(id, type.javaResult.id, parent, idx) + id + } + "div" -> { + val id = tw.getFreshIdLabel() + tw.writeExprs_divexpr(id, type.javaResult.id, parent, idx) + id + } + "rem" -> { + val id = tw.getFreshIdLabel() + tw.writeExprs_remexpr(id, type.javaResult.id, parent, idx) + id + } + "and" -> { + val id = tw.getFreshIdLabel() + tw.writeExprs_andbitexpr(id, type.javaResult.id, parent, idx) + id + } + "or" -> { + val id = tw.getFreshIdLabel() + tw.writeExprs_orbitexpr(id, type.javaResult.id, parent, idx) + id + } + "xor" -> { + val id = tw.getFreshIdLabel() + tw.writeExprs_xorbitexpr(id, type.javaResult.id, parent, idx) + id + } + "shl" -> { + val id = tw.getFreshIdLabel() + tw.writeExprs_lshiftexpr(id, type.javaResult.id, parent, idx) + id + } + "shr" -> { + val id = tw.getFreshIdLabel() + tw.writeExprs_rshiftexpr(id, type.javaResult.id, parent, idx) + id + } + "ushr" -> { + val id = tw.getFreshIdLabel() + tw.writeExprs_urshiftexpr(id, type.javaResult.id, parent, idx) + id + } + else -> { + logger.errorElement("Unhandled binary target name: $targetName", c) + return + } + } + tw.writeExprsKotlinType(id, type.kotlinResult.id) + if ( + isFunction( + target, + "kotlin", + "Byte or Short", + { it == "Byte" || it == "Short" }, + "and", + "or", + "xor" + ) + ) + binopExt(id) + else binopDisp(id) + } + // != gets desugared into not and ==. Here we resugar it. + c.origin == IrStatementOrigin.EXCLEQ && + isFunction(target, "kotlin", "Boolean", "not") && + c.valueArgumentsCount == 0 && + dr != null && + dr is IrCall && + isBuiltinCallInternal(dr, "EQEQ") -> { + val id = tw.getFreshIdLabel() + val type = useType(c.type) + tw.writeExprs_valueneexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + binOp(id, dr, callable, enclosingStmt) + } + c.origin == IrStatementOrigin.EXCLEQEQ && + isFunction(target, "kotlin", "Boolean", "not") && + c.valueArgumentsCount == 0 && + dr != null && + dr is IrCall && + isBuiltinCallInternal(dr, "EQEQEQ") -> { + val id = tw.getFreshIdLabel() + val type = useType(c.type) + tw.writeExprs_neexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + binOp(id, dr, callable, enclosingStmt) + } + c.origin == IrStatementOrigin.EXCLEQ && + isFunction(target, "kotlin", "Boolean", "not") && + c.valueArgumentsCount == 0 && + dr != null && + dr is IrCall && + isBuiltinCallInternal(dr, "ieee754equals") -> { + val id = tw.getFreshIdLabel() + val type = useType(c.type) + tw.writeExprs_neexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + binOp(id, dr, callable, enclosingStmt) + } + isFunction(target, "kotlin", "Boolean", "not") -> { + val id = tw.getFreshIdLabel() + val type = useType(c.type) + tw.writeExprs_lognotexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + unaryopDisp(id) + } + isNumericFunction(target, "inv", "unaryMinus", "unaryPlus") -> { + val type = useType(c.type) + val id: Label = + when (val targetName = target.name.asString()) { + "inv" -> { + val id = tw.getFreshIdLabel() + tw.writeExprs_bitnotexpr(id, type.javaResult.id, parent, idx) + id + } + "unaryMinus" -> { + val id = tw.getFreshIdLabel() + tw.writeExprs_minusexpr(id, type.javaResult.id, parent, idx) + id + } + "unaryPlus" -> { + val id = tw.getFreshIdLabel() + tw.writeExprs_plusexpr(id, type.javaResult.id, parent, idx) + id + } + else -> { + logger.errorElement("Unhandled unary target name: $targetName", c) + return + } + } + tw.writeExprsKotlinType(id, type.kotlinResult.id) + if ( + isFunction( + target, + "kotlin", + "Byte or Short", + { it == "Byte" || it == "Short" }, + "inv" + ) + ) + unaryopExt(id) + else unaryopDisp(id) + } + // We need to handle all the builtin operators defines in BuiltInOperatorNames in + // compiler/ir/ir.tree/src/org/jetbrains/kotlin/ir/IrBuiltIns.kt + // as they can't be extracted as external dependencies. + isBuiltinCallInternal(c, "less") -> { + if (c.origin != IrStatementOrigin.LT) { + logger.warnElement("Unexpected origin for LT: ${c.origin}", c) + } + val id = tw.getFreshIdLabel() + val type = useType(c.type) + tw.writeExprs_ltexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + binOp(id, c, callable, enclosingStmt) + } + isBuiltinCallInternal(c, "lessOrEqual") -> { + if (c.origin != IrStatementOrigin.LTEQ) { + logger.warnElement("Unexpected origin for LTEQ: ${c.origin}", c) + } + val id = tw.getFreshIdLabel() + val type = useType(c.type) + tw.writeExprs_leexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + binOp(id, c, callable, enclosingStmt) + } + isBuiltinCallInternal(c, "greater") -> { + if (c.origin != IrStatementOrigin.GT) { + logger.warnElement("Unexpected origin for GT: ${c.origin}", c) + } + val id = tw.getFreshIdLabel() + val type = useType(c.type) + tw.writeExprs_gtexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + binOp(id, c, callable, enclosingStmt) + } + isBuiltinCallInternal(c, "greaterOrEqual") -> { + if (c.origin != IrStatementOrigin.GTEQ) { + logger.warnElement("Unexpected origin for GTEQ: ${c.origin}", c) + } + val id = tw.getFreshIdLabel() + val type = useType(c.type) + tw.writeExprs_geexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + binOp(id, c, callable, enclosingStmt) + } + isBuiltinCallInternal(c, "EQEQ") -> { + if (c.origin != IrStatementOrigin.EQEQ) { + logger.warnElement("Unexpected origin for EQEQ: ${c.origin}", c) + } + val id = tw.getFreshIdLabel() + val type = useType(c.type) + tw.writeExprs_valueeqexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + binOp(id, c, callable, enclosingStmt) + } + isBuiltinCallInternal(c, "EQEQEQ") -> { + if (c.origin != IrStatementOrigin.EQEQEQ) { + logger.warnElement("Unexpected origin for EQEQEQ: ${c.origin}", c) + } + val id = tw.getFreshIdLabel() + val type = useType(c.type) + tw.writeExprs_eqexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + binOp(id, c, callable, enclosingStmt) + } + isBuiltinCallInternal(c, "ieee754equals") -> { + if (c.origin != IrStatementOrigin.EQEQ) { + logger.warnElement("Unexpected origin for ieee754equals: ${c.origin}", c) + } + val id = tw.getFreshIdLabel() + val type = useType(c.type) + tw.writeExprs_eqexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + binOp(id, c, callable, enclosingStmt) + } + isBuiltinCallInternal(c, "CHECK_NOT_NULL") -> { + if (c.origin != IrStatementOrigin.EXCLEXCL) { + logger.warnElement("Unexpected origin for CHECK_NOT_NULL: ${c.origin}", c) + } + + val id = tw.getFreshIdLabel() + val type = useType(c.type) + tw.writeExprs_notnullexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + unaryOp(id, c, callable, enclosingStmt) + } + isBuiltinCallInternal(c, "THROW_CCE") -> { + // TODO + logger.errorElement("Unhandled builtin", c) + } + isBuiltinCallInternal(c, "THROW_ISE") -> { + // TODO + logger.errorElement("Unhandled builtin", c) + } + isBuiltinCallInternal(c, "noWhenBranchMatchedException") -> { + kotlinNoWhenBranchMatchedConstructor?.let { + val locId = tw.getLocation(c) + val thrownType = useSimpleTypeClass(it.parentAsClass, listOf(), false) + val stmtParent = stmtExprParent.stmt(c, callable) + val throwId = tw.getFreshIdLabel() + tw.writeStmts_throwstmt( + throwId, + stmtParent.parent, + stmtParent.idx, + callable + ) + tw.writeHasLocation(throwId, locId) + val newExprId = + extractNewExpr( + it, + null, + thrownType, + locId, + throwId, + 0, + callable, + throwId + ) + if (newExprId == null) { + logger.errorElement( + "No ID for newExpr in noWhenBranchMatchedException", + c + ) + } else { + extractTypeAccess(thrownType, locId, newExprId, -3, callable, throwId) + } + } + } + isBuiltinCallInternal(c, "illegalArgumentException") -> { + // TODO + logger.errorElement("Unhandled builtin", c) + } + isBuiltinCallInternal(c, "ANDAND") -> { + // TODO + logger.errorElement("Unhandled builtin", c) + } + isBuiltinCallInternal(c, "OROR") -> { + // TODO + logger.errorElement("Unhandled builtin", c) + } + isFunction(target, "kotlin", "Any", "toString", true) -> { + stringValueOfObjectMethod?.let { + extractRawMethodAccess( + it, + c, + c.type, + callable, + parent, + idx, + enclosingStmt, + listOf(c.extensionReceiver), + null, + null + ) + } + } + isBuiltinCallKotlin(c, "enumValues") -> { + extractSpecialEnumFunction("values") + } + isBuiltinCallKotlin(c, "enumValueOf") -> { + extractSpecialEnumFunction("valueOf") + } + isBuiltinCallKotlin(c, "arrayOfNulls") -> { + val id = tw.getFreshIdLabel() + val type = useType(c.type) + tw.writeExprs_arraycreationexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + val locId = tw.getLocation(c) + extractExprContext(id, locId, callable, enclosingStmt) + + if (c.typeArgumentsCount == 1) { + val typeArgument = c.getTypeArgument(0) + if (typeArgument == null) { + logger.errorElement("Type argument missing in an arrayOfNulls call", c) + } else { + extractTypeAccessRecursive( + typeArgument, + locId, + id, + -1, + callable, + enclosingStmt, + TypeContext.GENERIC_ARGUMENT + ) + } + } else { + logger.errorElement( + "Expected to find exactly one type argument in an arrayOfNulls call", + c + ) + } + + if (c.valueArgumentsCount == 1) { + val dim = c.getValueArgument(0) + if (dim != null) { + extractExpressionExpr(dim, callable, id, 0, enclosingStmt) + } else { + logger.errorElement( + "Expected to find non-null argument in an arrayOfNulls call", + c + ) + } + } else { + logger.errorElement( + "Expected to find only one argument in an arrayOfNulls call", + c + ) + } + } + isBuiltinCallKotlin(c, "arrayOf") || + isBuiltinCallKotlin(c, "doubleArrayOf") || + isBuiltinCallKotlin(c, "floatArrayOf") || + isBuiltinCallKotlin(c, "longArrayOf") || + isBuiltinCallKotlin(c, "intArrayOf") || + isBuiltinCallKotlin(c, "charArrayOf") || + isBuiltinCallKotlin(c, "shortArrayOf") || + isBuiltinCallKotlin(c, "byteArrayOf") || + isBuiltinCallKotlin(c, "booleanArrayOf") -> { + + val isPrimitiveArrayCreation = !isBuiltinCallKotlin(c, "arrayOf") + val elementType = + if (isPrimitiveArrayCreation) { + c.type.getArrayElementType(pluginContext.irBuiltIns) + } else { + // TODO: is there any reason not to always use getArrayElementType? + if (c.typeArgumentsCount == 1) { + c.getTypeArgument(0).also { + if (it == null) { + logger.errorElement( + "Type argument missing in an arrayOf call", + c + ) + } + } + } else { + logger.errorElement( + "Expected to find one type argument in arrayOf call", + c + ) + null + } + } + + val arg = + if (c.valueArgumentsCount == 1) c.getValueArgument(0) + else { + logger.errorElement( + "Expected to find only one (vararg) argument in ${c.symbol.owner.name.asString()} call", + c + ) + null + } + ?.let { + if (it is IrVararg) it + else { + logger.errorElement( + "Expected to find vararg argument in ${c.symbol.owner.name.asString()} call", + c + ) + null + } + } + + extractArrayCreation( + arg, + c.type, + elementType, + isPrimitiveArrayCreation, + c, + parent, + idx, + callable, + enclosingStmt + ) + } + isBuiltinCall(c, "", "kotlin.jvm") -> { + // Special case for KClass<*>.java, which is used in the Parcelize plugin. In + // normal cases, this is already rewritten to the property referenced below: + findTopLevelPropertyOrWarn( + "kotlin.jvm", + "java", + "kotlin.jvm.JvmClassMappingKt", + c + ) + ?.let { javaProp -> + val getter = javaProp.getter + if (getter == null) { + logger.error( + "Couldn't find getter of `kotlin.jvm.JvmClassMappingKt::java`" + ) + return + } + + val ext = c.extensionReceiver + if (ext == null) { + logger.errorElement( + "No extension receiver found for `KClass::java` call", + c + ) + return + } + + val argType = + (ext.type as? IrSimpleType)?.arguments?.firstOrNull()?.typeOrNull + val typeArguments = if (argType == null) listOf() else listOf(argType) + + extractRawMethodAccess( + getter, + c, + c.type, + callable, + parent, + idx, + enclosingStmt, + listOf(), + null, + ext, + typeArguments + ) + } + } + isFunction( + target, + "kotlin", + "(some array type)", + { isArrayType(it) }, + "iterator" + ) -> { + val parentClass = target.parent + if (parentClass !is IrClass) { + logger.errorElement("Iterator parent is not a class", c) + return + } + + var typeFilter = + if (isGenericArrayType(parentClass.name.asString())) { + "kotlin.jvm.internal.ArrayIteratorKt" + } else { + "kotlin.jvm.internal.ArrayIteratorsKt" + } + + findTopLevelFunctionOrWarn( + "kotlin.jvm.internal", + "iterator", + typeFilter, + arrayOf(parentClass.kotlinFqName.asString()), + c + ) + ?.let { iteratorFn -> + val dispatchReceiver = c.dispatchReceiver + if (dispatchReceiver == null) { + logger.errorElement( + "No dispatch receiver found for array iterator call", + c + ) + } else { + val drType = dispatchReceiver.type + if (drType !is IrSimpleType) { + logger.errorElement( + "Dispatch receiver with unexpected type rep found for array iterator call: ${drType.javaClass}", + c + ) + } else { + val typeArgs = + drType.arguments.map { + when (it) { + is IrTypeProjection -> it.type + else -> pluginContext.irBuiltIns.anyNType + } + } + extractRawMethodAccess( + iteratorFn, + c, + c.type, + callable, + parent, + idx, + enclosingStmt, + listOf(c.dispatchReceiver), + null, + null, + typeArgs + ) + } + } + } + } + isFunction(target, "kotlin", "(some array type)", { isArrayType(it) }, "get") && + c.origin == IrStatementOrigin.GET_ARRAY_ELEMENT && + c.dispatchReceiver != null -> { + val id = tw.getFreshIdLabel() + val type = useType(c.type) + tw.writeExprs_arrayaccess(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + binopDisp(id) + } + isFunction(target, "kotlin", "(some array type)", { isArrayType(it) }, "set") && + c.origin == IrStatementOrigin.EQ && + c.dispatchReceiver != null -> { + val array = c.dispatchReceiver + val arrayIdx = c.getValueArgument(0) + val assignedValue = c.getValueArgument(1) + + if (array != null && arrayIdx != null && assignedValue != null) { + + val locId = tw.getLocation(c) + extractAssignExpr(c.type, locId, parent, idx, callable, enclosingStmt) + .also { assignId -> + tw.getFreshIdLabel().also { arrayAccessId -> + val arrayType = useType(array.type) + tw.writeExprs_arrayaccess( + arrayAccessId, + arrayType.javaResult.id, + assignId, + 0 + ) + tw.writeExprsKotlinType( + arrayAccessId, + arrayType.kotlinResult.id + ) + extractExprContext( + arrayAccessId, + locId, + callable, + enclosingStmt + ) + + extractExpressionExpr( + array, + callable, + arrayAccessId, + 0, + enclosingStmt + ) + extractExpressionExpr( + arrayIdx, + callable, + arrayAccessId, + 1, + enclosingStmt + ) + } + extractExpressionExpr( + assignedValue, + callable, + assignId, + 1, + enclosingStmt + ) + } + } else { + logger.errorElement("Unexpected Array.set function signature", c) + } + } + isBuiltinCall(c, "", "kotlin.jvm.internal") -> { + + if (c.valueArgumentsCount != 1) { + logger.errorElement( + "Expected to find one argument for a kotlin.jvm.internal.() call, but found ${c.valueArgumentsCount}", + c + ) + return + } + + if (c.typeArgumentsCount != 2) { + logger.errorElement( + "Expected to find two type arguments for a kotlin.jvm.internal.() call, but found ${c.typeArgumentsCount}", + c + ) + return + } + val valueArg = c.getValueArgument(0) + if (valueArg == null) { + logger.errorElement( + "Cannot find value argument for a kotlin.jvm.internal.() call", + c + ) + return + } + val typeArg = c.getTypeArgument(1) + if (typeArg == null) { + logger.errorElement( + "Cannot find type argument for a kotlin.jvm.internal.() call", + c + ) + return + } + + val id = tw.getFreshIdLabel() + val locId = tw.getLocation(c) + val type = useType(c.type) + tw.writeExprs_unsafecoerceexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, callable, enclosingStmt) + extractTypeAccessRecursive(typeArg, locId, id, 0, callable, enclosingStmt) + extractExpressionExpr(valueArg, callable, id, 1, enclosingStmt) + } + isBuiltinCallInternal(c, "dataClassArrayMemberToString") -> { + val arrayArg = c.getValueArgument(0) + val realArrayClass = arrayArg?.type?.classOrNull + if (realArrayClass == null) { + logger.errorElement( + "Argument to dataClassArrayMemberToString not a class", + c + ) + return + } + val realCallee = + javaUtilArrays?.declarations?.findSubType { decl -> + decl.name.asString() == "toString" && + decl.valueParameters.size == 1 && + decl.valueParameters[0].type.classOrNull?.let { + it == realArrayClass + } == true + } + if (realCallee == null) { + logger.errorElement( + "Couldn't find a java.lang.Arrays.toString method matching class ${realArrayClass.owner.name}", + c + ) + } else { + extractRawMethodAccess( + realCallee, + c, + c.type, + callable, + parent, + idx, + enclosingStmt, + listOf(arrayArg), + null, + null + ) + } + } + isBuiltinCallInternal(c, "dataClassArrayMemberHashCode") -> { + val arrayArg = c.getValueArgument(0) + val realArrayClass = arrayArg?.type?.classOrNull + if (realArrayClass == null) { + logger.errorElement( + "Argument to dataClassArrayMemberHashCode not a class", + c + ) + return + } + val realCallee = + javaUtilArrays?.declarations?.findSubType { decl -> + decl.name.asString() == "hashCode" && + decl.valueParameters.size == 1 && + decl.valueParameters[0].type.classOrNull?.let { + it == realArrayClass + } == true + } + if (realCallee == null) { + logger.errorElement( + "Couldn't find a java.lang.Arrays.hashCode method matching class ${realArrayClass.owner.name}", + c + ) + } else { + extractRawMethodAccess( + realCallee, + c, + c.type, + callable, + parent, + idx, + enclosingStmt, + listOf(arrayArg), + null, + null + ) + } + } + else -> { + extractMethodAccess(target, true, true) + } + } + } + } + + private fun extractArrayCreation( + elementList: IrVararg?, + resultType: IrType, + elementType: IrType?, + allowPrimitiveElementType: Boolean, + locElement: IrElement, + parent: Label, + idx: Int, + enclosingCallable: Label, + enclosingStmt: Label + ) { + // If this is [someType]ArrayOf(*x), x, otherwise null + val clonedArray = + elementList?.let { + if (it.elements.size == 1) { + val onlyElement = it.elements[0] + if (onlyElement is IrSpreadElement) onlyElement.expression else null + } else null + } + + if (clonedArray != null) { + // This is an array clone: extract is as a call to java.lang.Object.clone + objectCloneMethod?.let { + extractRawMethodAccess( + it, + locElement, + resultType, + enclosingCallable, + parent, + idx, + enclosingStmt, + listOf(), + clonedArray, + null + ) + } + } else { + // This is array creation: extract it as a call to new ArrayType[] { ... } + val id = tw.getFreshIdLabel() + val type = useType(resultType) + tw.writeExprs_arraycreationexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + val locId = tw.getLocation(locElement) + extractExprContext(id, locId, enclosingCallable, enclosingStmt) + + if (elementType != null) { + val typeContext = + if (allowPrimitiveElementType) TypeContext.OTHER + else TypeContext.GENERIC_ARGUMENT + extractTypeAccessRecursive( + elementType, + locId, + id, + -1, + enclosingCallable, + enclosingStmt, + typeContext + ) + } + + if (elementList != null) { + val initId = tw.getFreshIdLabel() + tw.writeExprs_arrayinit(initId, type.javaResult.id, id, -2) + tw.writeExprsKotlinType(initId, type.kotlinResult.id) + extractExprContext(initId, locId, enclosingCallable, enclosingStmt) + elementList.elements.forEachIndexed { i, arg -> + extractVarargElement(arg, enclosingCallable, initId, i, enclosingStmt) + } + + extractConstantInteger( + elementList.elements.size, + locId, + id, + 0, + enclosingCallable, + enclosingStmt + ) + } + } + } + + private fun extractNewExpr( + methodId: Label, + constructedType: TypeResults, + locId: Label, + parent: Label, + idx: Int, + callable: Label, + enclosingStmt: Label + ): Label { + val id = tw.getFreshIdLabel() + tw.writeExprs_newexpr(id, constructedType.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, constructedType.kotlinResult.id) + extractExprContext(id, locId, callable, enclosingStmt) + tw.writeCallableBinding(id, methodId) + return id + } + + private fun extractNewExpr( + calledConstructor: IrFunction, + constructorTypeArgs: List?, + constructedType: TypeResults, + locId: Label, + parent: Label, + idx: Int, + callable: Label, + enclosingStmt: Label + ): Label? { + val funId = useFunction(calledConstructor, constructorTypeArgs) + if (funId == null) { + logger.error("Cannot get ID for newExpr function") + return null + } + return extractNewExpr(funId, constructedType, locId, parent, idx, callable, enclosingStmt) + } + + private fun needsObinitFunction(c: IrClass) = + c.primaryConstructor == null && c.constructors.count() > 1 + + private fun getObinitLabel(c: IrClass, parentId: Label): String = + getFunctionLabel( + c, + parentId, + "", + listOf(), + pluginContext.irBuiltIns.unitType, + null, + functionTypeParameters = listOf(), + classTypeArgsIncludingOuterClasses = listOf(), + overridesCollectionsMethod = false, + javaSignature = null, + addParameterWildcardsByDefault = false + ) + + private fun extractConstructorCall( + e: IrFunctionAccessExpression, + parent: Label, + idx: Int, + callable: Label, + enclosingStmt: Label + ) { + val eType = e.type + if (eType !is IrSimpleType) { + logger.errorElement("Constructor call has non-simple type ${eType.javaClass}", e) + return + } + val type = useType(eType) + val isAnonymous = eType.isAnonymous + val locId = tw.getLocation(e) + val valueArgs = (0 until e.valueArgumentsCount).map { e.getValueArgument(it) } + + val id = + if ( + e !is IrEnumConstructorCall && callUsesDefaultArguments(e.symbol.owner, valueArgs) + ) { + val defaultsMethodId = getDefaultsMethodLabel(e.symbol.owner) + if (defaultsMethodId == null) { + logger.errorElement("Cannot get defaults method ID", e) + return + } + extractNewExpr( + defaultsMethodId.cast(), + type, + locId, + parent, + idx, + callable, + enclosingStmt + ) + .also { + extractDefaultsCallArguments( + it, + e.symbol.owner, + callable, + enclosingStmt, + valueArgs, + null, + null + ) + } + } else { + val newExprId = + extractNewExpr( + e.symbol.owner, + eType.arguments, + type, + locId, + parent, + idx, + callable, + enclosingStmt + ) + if (newExprId == null) { + logger.errorElement("Cannot get newExpr ID", e) + return + } + + val realCallTarget = e.symbol.owner.realOverrideTarget + // Generated constructor calls to kotlin.Enum have no arguments in IR, but the + // constructor takes two parameters. + if ( + e is IrEnumConstructorCall && + realCallTarget is IrConstructor && + realCallTarget.parentClassOrNull?.fqNameWhenAvailable?.asString() == + "kotlin.Enum" && + realCallTarget.valueParameters.size == 2 && + realCallTarget.valueParameters[0].type == + pluginContext.irBuiltIns.stringType && + realCallTarget.valueParameters[1].type == pluginContext.irBuiltIns.intType + ) { + + val id0 = + extractNull( + pluginContext.irBuiltIns.stringType, + locId, + newExprId, + 0, + callable, + enclosingStmt + ) + tw.writeCompiler_generated( + id0, + CompilerGeneratedKinds.ENUM_CONSTRUCTOR_ARGUMENT.kind + ) + + val id1 = + extractConstantInteger(0, locId, newExprId, 1, callable, enclosingStmt) + tw.writeCompiler_generated( + id1, + CompilerGeneratedKinds.ENUM_CONSTRUCTOR_ARGUMENT.kind + ) + } else { + extractCallValueArguments(newExprId, e, enclosingStmt, callable, 0) + } + + newExprId + } + + if (isAnonymous) { + tw.writeIsAnonymClass(type.javaResult.id.cast(), id) + } + + val dr = e.dispatchReceiver + if (dr != null) { + extractExpressionExpr(dr, callable, id, -2, enclosingStmt) + } + + val typeAccessType = + if (isAnonymous) { + val c = eType.classifier.owner + if (c !is IrClass) { + logger.warnElement("Anonymous type not a class (${c.javaClass})", e) + } + if ((c as? IrClass)?.superTypes?.size == 1) { + useType(c.superTypes.first()) + } else { + useType(pluginContext.irBuiltIns.anyType) + } + } else { + type + } + + if (e is IrConstructorCall) { + extractConstructorTypeAccess( + eType, + typeAccessType, + e.symbol, + locId, + id, + -3, + callable, + enclosingStmt + ) + } else if (e is IrEnumConstructorCall) { + val enumClass = e.symbol.owner.parent as? IrClass + if (enumClass == null) { + logger.warnElement("Couldn't find declaring class of enum constructor call", e) + return + } + + val args = + (0 until e.typeArgumentsCount).map { e.getTypeArgument(it) }.requireNoNullsOrNull() + if (args == null) { + logger.warnElement("Found null type argument in enum constructor call", e) + return + } + + val enumType = enumClass.typeWith(args) + extractConstructorTypeAccess( + enumType, + useType(enumType), + e.symbol, + locId, + id, + -3, + callable, + enclosingStmt + ) + } else { + logger.errorElement("Unexpected constructor call type: ${e.javaClass}", e) + } + } + + abstract inner class StmtExprParent { + abstract fun stmt(e: IrExpression, callable: Label): StmtParent + + abstract fun expr(e: IrExpression, callable: Label): ExprParent + } + + inner class StmtParent(val parent: Label, val idx: Int) : StmtExprParent() { + override fun stmt(e: IrExpression, callable: Label) = this + + override fun expr(e: IrExpression, callable: Label) = + extractExpressionStmt(tw.getLocation(e), parent, idx, callable).let { id -> + ExprParent(id, 0, id) + } + } + + inner class ExprParent( + val parent: Label, + val idx: Int, + val enclosingStmt: Label + ) : StmtExprParent() { + override fun stmt(e: IrExpression, callable: Label): StmtParent { + val id = tw.getFreshIdLabel() + val type = useType(e.type) + val locId = tw.getLocation(e) + tw.writeExprs_stmtexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, callable, enclosingStmt) + return StmtParent(id, 0) + } + + override fun expr(e: IrExpression, callable: Label): ExprParent { + return this + } + } + + private fun getStatementOriginOperator(origin: IrStatementOrigin?) = + when (origin) { + IrStatementOrigin.PLUSEQ -> "plus" + IrStatementOrigin.MINUSEQ -> "minus" + IrStatementOrigin.MULTEQ -> "times" + IrStatementOrigin.DIVEQ -> "div" + IrStatementOrigin.PERCEQ -> "rem" + else -> null + } + + private fun getUpdateInPlaceRHS( + origin: IrStatementOrigin?, + isExpectedLhs: (IrExpression?) -> Boolean, + updateRhs: IrExpression + ): IrExpression? { + // Check for a desugared in-place update operator, such as "v += e": + return getStatementOriginOperator(origin)?.let { + if (updateRhs is IrCall && isNumericFunction(updateRhs.symbol.owner, it)) { + // Check for an expression like x = get(x).op(e): + val opReceiver = updateRhs.dispatchReceiver + if (isExpectedLhs(opReceiver)) { + updateRhs.getValueArgument(0) + } else null + } else null + } + } + + private fun writeUpdateInPlaceExpr( + origin: IrStatementOrigin + ): (( + tw: TrapWriter, + id: Label, + type: Label, + exprParent: Label, + index: Int + ) -> Unit)? { + when (origin) { + IrStatementOrigin.PLUSEQ -> + return { + tw: TrapWriter, + id: Label, + type: Label, + exprParent: Label, + index: Int -> + tw.writeExprs_assignaddexpr(id.cast(), type, exprParent, index) + } + IrStatementOrigin.MINUSEQ -> + return { + tw: TrapWriter, + id: Label, + type: Label, + exprParent: Label, + index: Int -> + tw.writeExprs_assignsubexpr(id.cast(), type, exprParent, index) + } + IrStatementOrigin.MULTEQ -> + return { + tw: TrapWriter, + id: Label, + type: Label, + exprParent: Label, + index: Int -> + tw.writeExprs_assignmulexpr(id.cast(), type, exprParent, index) + } + IrStatementOrigin.DIVEQ -> + return { + tw: TrapWriter, + id: Label, + type: Label, + exprParent: Label, + index: Int -> + tw.writeExprs_assigndivexpr(id.cast(), type, exprParent, index) + } + IrStatementOrigin.PERCEQ -> + return { + tw: TrapWriter, + id: Label, + type: Label, + exprParent: Label, + index: Int -> + tw.writeExprs_assignremexpr(id.cast(), type, exprParent, index) + } + else -> return null + } + } + + /** + * This method tries to extract a block as an enhanced for loop. It returns true if it succeeds, + * and false otherwise. + */ + private fun tryExtractForLoop( + e: IrContainerExpression, + callable: Label, + parent: StmtExprParent + ): Boolean { + /* + * We're expecting the pattern + * { + * val iterator = [expr].iterator() + * while (iterator.hasNext()) { + * val [loopVar] = iterator.next() + * [block] + * } + * } + */ + + if (e.origin != IrStatementOrigin.FOR_LOOP || e.statements.size != 2) { + return false + } + + val iteratorVariable = e.statements[0] as? IrVariable + val innerWhile = e.statements[1] as? IrWhileLoop + + if ( + iteratorVariable == null || + iteratorVariable.origin != IrDeclarationOrigin.FOR_LOOP_ITERATOR || + innerWhile == null || + innerWhile.origin != IrStatementOrigin.FOR_LOOP_INNER_WHILE + ) { + return false + } + + val initializer = iteratorVariable.initializer as? IrCall + if ( + initializer == null || + initializer.origin != IrStatementOrigin.FOR_LOOP_ITERATOR || + initializer.symbol.owner.name.asString() != "iterator" + ) { + return false + } + + val expr = initializer.dispatchReceiver + val cond = innerWhile.condition as? IrCall + val body = innerWhile.body as? IrBlock + + if ( + expr == null || + cond == null || + cond.origin != IrStatementOrigin.FOR_LOOP_HAS_NEXT || + (cond.dispatchReceiver as? IrGetValue)?.symbol?.owner != iteratorVariable || + body == null || + body.origin != IrStatementOrigin.FOR_LOOP_INNER_WHILE || + body.statements.size < 2 + ) { + return false + } + + val loopVar = body.statements[0] as? IrVariable + val nextCall = loopVar?.initializer as? IrCall + + if ( + loopVar == null || + !(loopVar.origin == IrDeclarationOrigin.FOR_LOOP_VARIABLE || + loopVar.origin == IrDeclarationOrigin.IR_TEMPORARY_VARIABLE) || + nextCall == null || + nextCall.origin != IrStatementOrigin.FOR_LOOP_NEXT || + (nextCall.dispatchReceiver as? IrGetValue)?.symbol?.owner != iteratorVariable + ) { + return false + } + + val id = + extractLoop(innerWhile, null, parent, callable) { p, idx -> + tw.getFreshIdLabel().also { + tw.writeStmts_enhancedforstmt(it, p, idx, callable) + } + } + + extractVariableExpr(loopVar, callable, id, 0, id, extractInitializer = false) + extractExpressionExpr(expr, callable, id, 1, id) + val block = body.statements[1] as? IrBlock + if (body.statements.size == 2 && block != null) { + // Extract the body that was given to us by the compiler + extractExpressionStmt(block, callable, id, 2) + } else { + // Extract a block with all but the first (loop variable declaration) statement + extractBlock(body, body.statements.takeLast(body.statements.size - 1), id, 2, callable) + } + + return true + } + + /** + * This tried to extract a block as an array update. It returns true if it succeeds, and false + * otherwise. + */ + private fun tryExtractArrayUpdate( + e: IrContainerExpression, + callable: Label, + parent: StmtExprParent + ): Boolean { + /* + * We're expecting the pattern + * { + * val array = e1 + * val idx = e2 + * array.set(idx, array.get(idx).op(e3)) + * } + * + * If we find it, we'll extract e1[e2] op= e3 (op is +, -, ...) + */ + if (e.statements.size != 3) return false + (e.statements[0] as? IrVariable)?.let { arrayVarDecl -> + arrayVarDecl.initializer?.let { arrayVarInitializer -> + (e.statements[1] as? IrVariable)?.let { indexVarDecl -> + indexVarDecl.initializer?.let { indexVarInitializer -> + (e.statements[2] as? IrCall)?.let { arraySetCall -> + if ( + isFunction( + arraySetCall.symbol.owner, + "kotlin", + "(some array type)", + { isArrayType(it) }, + "set" + ) + ) { + val updateRhs0 = arraySetCall.getValueArgument(1) + if (updateRhs0 == null) { + logger.errorElement("Update RHS not found", e) + return false + } + getUpdateInPlaceRHS( + e + .origin, // Using e.origin not arraySetCall.origin here + // distinguishes a compiler-generated block + // from a user manually code that looks the + // same. + { oldValue -> + oldValue is IrCall && + isFunction( + oldValue.symbol.owner, + "kotlin", + "(some array type)", + { typeName -> isArrayType(typeName) }, + "get" + ) && + (oldValue.dispatchReceiver as? IrGetValue)?.let { + receiverVal -> + receiverVal.symbol.owner == + arrayVarDecl.symbol.owner + } ?: false + }, + updateRhs0 + ) + ?.let { updateRhs -> + val origin = e.origin + if (origin == null) { + logger.errorElement("No origin found", e) + return false + } + val writeUpdateInPlaceExprFun = + writeUpdateInPlaceExpr(origin) + if (writeUpdateInPlaceExprFun == null) { + logger.errorElement("Unexpected origin", e) + return false + } + + // Create an assignment skeleton _ op= _ + val exprParent = parent.expr(e, callable) + val assignId = tw.getFreshIdLabel() + val type = useType(arrayVarInitializer.type) + val locId = tw.getLocation(e) + tw.writeExprsKotlinType(assignId, type.kotlinResult.id) + extractExprContext( + assignId, + locId, + callable, + exprParent.enclosingStmt + ) + + writeUpdateInPlaceExprFun( + tw, + assignId, + type.javaResult.id, + exprParent.parent, + exprParent.idx + ) + + // Extract e1[e2] + val lhsId = tw.getFreshIdLabel() + val elementType = useType(updateRhs.type) + tw.writeExprs_arrayaccess( + lhsId, + elementType.javaResult.id, + assignId, + 0 + ) + tw.writeExprsKotlinType(lhsId, elementType.kotlinResult.id) + extractExprContext( + lhsId, + locId, + callable, + exprParent.enclosingStmt + ) + extractExpressionExpr( + arrayVarInitializer, + callable, + lhsId, + 0, + exprParent.enclosingStmt + ) + extractExpressionExpr( + indexVarInitializer, + callable, + lhsId, + 1, + exprParent.enclosingStmt + ) + + // Extract e3 + extractExpressionExpr( + updateRhs, + callable, + assignId, + 1, + exprParent.enclosingStmt + ) + + return true + } + } + } + } + } + } + } + + return false + } + + private fun extractExpressionStmt( + locId: Label, + parent: Label, + idx: Int, + callable: Label + ) = + tw.getFreshIdLabel().also { + tw.writeStmts_exprstmt(it, parent, idx, callable) + tw.writeHasLocation(it, locId) + } + + private fun extractExpressionStmt( + e: IrExpression, + callable: Label, + parent: Label, + idx: Int + ) { + extractExpression(e, callable, StmtParent(parent, idx)) + } + + fun extractExpressionExpr( + e: IrExpression, + callable: Label, + parent: Label, + idx: Int, + enclosingStmt: Label + ) { + extractExpression(e, callable, ExprParent(parent, idx, enclosingStmt)) + } + + private fun extractExprContext( + id: Label, + locId: Label, + callable: Label?, + enclosingStmt: Label? + ) { + tw.writeHasLocation(id, locId) + callable?.let { tw.writeCallableEnclosingExpr(id, it) } + enclosingStmt?.let { tw.writeStatementEnclosingExpr(id, it) } + } + + private fun extractEqualsExpression( + locId: Label, + parent: Label, + idx: Int, + callable: Label, + enclosingStmt: Label + ) = + tw.getFreshIdLabel().also { + val type = useType(pluginContext.irBuiltIns.booleanType) + tw.writeExprs_eqexpr(it, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(it, type.kotlinResult.id) + extractExprContext(it, locId, callable, enclosingStmt) + } + + private fun extractAndbitExpression( + type: IrType, + locId: Label, + parent: Label, + idx: Int, + callable: Label, + enclosingStmt: Label + ) = + tw.getFreshIdLabel().also { + val typeResults = useType(type) + tw.writeExprs_andbitexpr(it, typeResults.javaResult.id, parent, idx) + tw.writeExprsKotlinType(it, typeResults.kotlinResult.id) + extractExprContext(it, locId, callable, enclosingStmt) + } + + private fun extractConstantInteger( + v: Number, + locId: Label, + parent: Label, + idx: Int, + callable: Label?, + enclosingStmt: Label?, + overrideId: Label? = null + ) = + exprIdOrFresh(overrideId).also { + val type = useType(pluginContext.irBuiltIns.intType) + tw.writeExprs_integerliteral(it, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(it, type.kotlinResult.id) + tw.writeNamestrings(v.toString(), v.toString(), it) + extractExprContext(it, locId, callable, enclosingStmt) + } + + private fun extractNull( + t: IrType, + locId: Label, + parent: Label, + idx: Int, + callable: Label?, + enclosingStmt: Label?, + overrideId: Label? = null + ) = + exprIdOrFresh(overrideId).also { + val type = useType(t) + tw.writeExprs_nullliteral(it, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(it, type.kotlinResult.id) + extractExprContext(it, locId, callable, enclosingStmt) + } + + private fun extractAssignExpr( + type: IrType, + locId: Label, + parent: Label, + idx: Int, + callable: Label, + enclosingStmt: Label + ) = + tw.getFreshIdLabel().also { + val typeResults = useType(type) + tw.writeExprs_assignexpr(it, typeResults.javaResult.id, parent, idx) + tw.writeExprsKotlinType(it, typeResults.kotlinResult.id) + extractExprContext(it, locId, callable, enclosingStmt) + } + + private fun extractExpression( + e: IrExpression, + callable: Label, + parent: StmtExprParent + ) { + with("expression", e) { + when (e) { + is IrDelegatingConstructorCall -> { + val stmtParent = parent.stmt(e, callable) + + val irCallable = declarationStack.peek().first + + val delegatingClass = e.symbol.owner.parent + val currentClass = irCallable.parent + + if (delegatingClass !is IrClass) { + logger.warnElement( + "Delegating class isn't a class: " + delegatingClass.javaClass, + e + ) + } + if (currentClass !is IrClass) { + logger.warnElement( + "Current class isn't a class: " + currentClass.javaClass, + e + ) + } + + val id: Label + if (delegatingClass != currentClass) { + id = tw.getFreshIdLabel() + tw.writeStmts_superconstructorinvocationstmt( + id, + stmtParent.parent, + stmtParent.idx, + callable + ) + } else { + id = tw.getFreshIdLabel() + tw.writeStmts_constructorinvocationstmt( + id, + stmtParent.parent, + stmtParent.idx, + callable + ) + } + + val locId = tw.getLocation(e) + val methodId = useFunction(e.symbol.owner) + if (methodId == null) { + logger.errorElement("Cannot get ID for delegating constructor", e) + } else { + tw.writeCallableBinding(id.cast(), methodId) + } + + tw.writeHasLocation(id, locId) + extractCallValueArguments(id, e, id, callable, 0) + val dr = e.dispatchReceiver + if (dr != null) { + extractExpressionExpr(dr, callable, id, -1, id) + } + + // todo: type arguments at index -2, -3, ... + } + is IrThrow -> { + val stmtParent = parent.stmt(e, callable) + val id = tw.getFreshIdLabel() + val locId = tw.getLocation(e) + tw.writeStmts_throwstmt(id, stmtParent.parent, stmtParent.idx, callable) + tw.writeHasLocation(id, locId) + extractExpressionExpr(e.value, callable, id, 0, id) + } + is IrBreak -> { + val stmtParent = parent.stmt(e, callable) + val id = tw.getFreshIdLabel() + tw.writeStmts_breakstmt(id, stmtParent.parent, stmtParent.idx, callable) + extractBreakContinue(e, id) + } + is IrContinue -> { + val stmtParent = parent.stmt(e, callable) + val id = tw.getFreshIdLabel() + tw.writeStmts_continuestmt(id, stmtParent.parent, stmtParent.idx, callable) + extractBreakContinue(e, id) + } + is IrReturn -> { + val stmtParent = parent.stmt(e, callable) + val id = tw.getFreshIdLabel() + val locId = tw.getLocation(e) + tw.writeStmts_returnstmt(id, stmtParent.parent, stmtParent.idx, callable) + tw.writeHasLocation(id, locId) + extractExpressionExpr(e.value, callable, id, 0, id) + } + is IrTry -> { + val stmtParent = parent.stmt(e, callable) + val id = tw.getFreshIdLabel() + val locId = tw.getLocation(e) + tw.writeStmts_trystmt(id, stmtParent.parent, stmtParent.idx, callable) + tw.writeHasLocation(id, locId) + extractExpressionStmt(e.tryResult, callable, id, -1) + val finallyStmt = e.finallyExpression + if (finallyStmt != null) { + extractExpressionStmt(finallyStmt, callable, id, -2) + } + for ((catchIdx, catchClause) in e.catches.withIndex()) { + val catchId = tw.getFreshIdLabel() + tw.writeStmts_catchclause(catchId, id, catchIdx, callable) + val catchLocId = tw.getLocation(catchClause) + tw.writeHasLocation(catchId, catchLocId) + extractTypeAccessRecursive( + catchClause.catchParameter.type, + tw.getLocation(catchClause.catchParameter), + catchId, + -1, + callable, + catchId + ) + extractVariableExpr( + catchClause.catchParameter, + callable, + catchId, + 0, + catchId + ) + extractExpressionStmt(catchClause.result, callable, catchId, 1) + } + } + is IrContainerExpression -> { + if ( + !tryExtractArrayUpdate(e, callable, parent) && + !tryExtractForLoop(e, callable, parent) + ) { + + extractBlock(e, e.statements, parent, callable) + } + } + is IrWhileLoop -> { + extractLoopWithCondition(e, parent, callable) + } + is IrDoWhileLoop -> { + extractLoopWithCondition(e, parent, callable) + } + is IrInstanceInitializerCall -> { + val irConstructor = declarationStack.peek().first as? IrConstructor + if (irConstructor == null) { + logger.errorElement("IrInstanceInitializerCall outside constructor", e) + return + } + if (needsObinitFunction(irConstructor.parentAsClass)) { + val exprParent = parent.expr(e, callable) + val id = tw.getFreshIdLabel() + val type = useType(pluginContext.irBuiltIns.unitType) + val locId = tw.getLocation(e) + val parentClass = irConstructor.parentAsClass + val parentId = useDeclarationParentOf(irConstructor, false, null, true) + if (parentId == null) { + logger.errorElement("Cannot get parent ID for obinit", e) + return + } + val methodLabel = getObinitLabel(parentClass, parentId) + val methodId = tw.getLabelFor(methodLabel) + tw.writeExprs_methodaccess( + id, + type.javaResult.id, + exprParent.parent, + exprParent.idx + ) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, callable, exprParent.enclosingStmt) + tw.writeCallableBinding(id, methodId) + } else { + val stmtParent = parent.stmt(e, callable) + extractInstanceInitializerBlock(stmtParent, irConstructor) + } + } + is IrConstructorCall -> { + val exprParent = parent.expr(e, callable) + extractConstructorCall( + e, + exprParent.parent, + exprParent.idx, + callable, + exprParent.enclosingStmt + ) + } + is IrEnumConstructorCall -> { + val exprParent = parent.expr(e, callable) + extractConstructorCall( + e, + exprParent.parent, + exprParent.idx, + callable, + exprParent.enclosingStmt + ) + } + is IrCall -> { + extractCall(e, callable, parent) + } + is IrStringConcatenation -> { + val exprParent = parent.expr(e, callable) + val id = tw.getFreshIdLabel() + val type = useType(e.type) + val locId = tw.getLocation(e) + tw.writeExprs_stringtemplateexpr( + id, + type.javaResult.id, + exprParent.parent, + exprParent.idx + ) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, callable, exprParent.enclosingStmt) + e.arguments.forEachIndexed { i, a -> + extractExpressionExpr(a, callable, id, i, exprParent.enclosingStmt) + } + } + is IrConst<*> -> { + val exprParent = parent.expr(e, callable) + extractConstant( + e, + exprParent.parent, + exprParent.idx, + callable, + exprParent.enclosingStmt + ) + } + is IrGetValue -> { + val exprParent = parent.expr(e, callable) + val owner = e.symbol.owner + if ( + owner is IrValueParameter && + owner.index == -1 && + !owner.isExtensionReceiver() + ) { + extractThisAccess(e, owner.parent, exprParent, callable) + } else { + val isAnnotationClassParameter = + ((owner as? IrValueParameter)?.parent as? IrConstructor) + ?.parentClassOrNull + ?.kind == ClassKind.ANNOTATION_CLASS + val extractType = + if (isAnnotationClassParameter) kClassToJavaClass(e.type) else e.type + extractVariableAccess( + useValueDeclaration(owner), + extractType, + tw.getLocation(e), + exprParent.parent, + exprParent.idx, + callable, + exprParent.enclosingStmt + ) + } + } + is IrGetField -> { + val exprParent = parent.expr(e, callable) + val owner = tryReplaceAndroidSyntheticField(e.symbol.owner) + val locId = tw.getLocation(e) + val fieldType = + if (isAnnotationClassField(owner)) kClassToJavaClass(e.type) else e.type + extractVariableAccess( + useField(owner), + fieldType, + locId, + exprParent.parent, + exprParent.idx, + callable, + exprParent.enclosingStmt + ) + .also { id -> + val receiver = e.receiver + if (receiver != null) { + extractExpressionExpr( + receiver, + callable, + id, + -1, + exprParent.enclosingStmt + ) + } else if (owner.isStatic) { + extractStaticTypeAccessQualifier( + owner, + id, + locId, + callable, + exprParent.enclosingStmt + ) + } + } + } + is IrGetEnumValue -> { + val exprParent = parent.expr(e, callable) + extractEnumValue( + e, + exprParent.parent, + exprParent.idx, + callable, + exprParent.enclosingStmt + ) + } + is IrSetValue, + is IrSetField -> { + val exprParent = parent.expr(e, callable) + val id = tw.getFreshIdLabel() + val type = useType(e.type) + val rhsValue = + when (e) { + is IrSetValue -> e.value + is IrSetField -> e.value + else -> { + logger.errorElement("Unhandled IrSet* element.", e) + return + } + } + // The set operation's location as actually that of its LHS. Hence, the + // assignment spans the + // set op plus its RHS, while the varAccess takes its location from `e`. + val locId = tw.getLocation(e.startOffset, rhsValue.endOffset) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, callable, exprParent.enclosingStmt) + + val lhsId = tw.getFreshIdLabel() + val lhsLocId = tw.getLocation(e) + extractExprContext(lhsId, lhsLocId, callable, exprParent.enclosingStmt) + + when (e) { + is IrSetValue -> { + // Check for a desugared in-place update operator, such as "v += e": + val inPlaceUpdateRhs = + getUpdateInPlaceRHS( + e.origin, + { it is IrGetValue && it.symbol.owner == e.symbol.owner }, + rhsValue + ) + if (inPlaceUpdateRhs != null) { + val origin = e.origin + if (origin == null) { + logger.errorElement("No origin for set-value", e) + return + } else { + val writeUpdateInPlaceExprFun = writeUpdateInPlaceExpr(origin) + if (writeUpdateInPlaceExprFun == null) { + logger.errorElement("Unexpected origin for set-value", e) + return + } + writeUpdateInPlaceExprFun( + tw, + id, + type.javaResult.id, + exprParent.parent, + exprParent.idx + ) + } + } else { + tw.writeExprs_assignexpr( + id, + type.javaResult.id, + exprParent.parent, + exprParent.idx + ) + } + + val lhsType = useType(e.symbol.owner.type) + tw.writeExprs_varaccess(lhsId, lhsType.javaResult.id, id, 0) + tw.writeExprsKotlinType(lhsId, lhsType.kotlinResult.id) + val vId = useValueDeclaration(e.symbol.owner) + if (vId != null) { + tw.writeVariableBinding(lhsId, vId) + } + extractExpressionExpr( + inPlaceUpdateRhs ?: rhsValue, + callable, + id, + 1, + exprParent.enclosingStmt + ) + } + is IrSetField -> { + tw.writeExprs_assignexpr( + id, + type.javaResult.id, + exprParent.parent, + exprParent.idx + ) + val realField = tryReplaceAndroidSyntheticField(e.symbol.owner) + val lhsType = useType(realField.type) + tw.writeExprs_varaccess(lhsId, lhsType.javaResult.id, id, 0) + tw.writeExprsKotlinType(lhsId, lhsType.kotlinResult.id) + val vId = useField(realField) + tw.writeVariableBinding(lhsId, vId) + extractExpressionExpr( + e.value, + callable, + id, + 1, + exprParent.enclosingStmt + ) + + val receiver = e.receiver + if (receiver != null) { + extractExpressionExpr( + receiver, + callable, + lhsId, + -1, + exprParent.enclosingStmt + ) + } else if (realField.isStatic) { + extractStaticTypeAccessQualifier( + realField, + lhsId, + lhsLocId, + callable, + exprParent.enclosingStmt + ) + } + } + else -> { + logger.errorElement("Unhandled IrSet* element.", e) + } + } + } + is IrWhen -> { + val isAndAnd = e.origin == IrStatementOrigin.ANDAND + val isOrOr = e.origin == IrStatementOrigin.OROR + + if ( + (isAndAnd || isOrOr) && + e.branches.size == 2 && + (e.branches[1].condition as? IrConst<*>)?.value == true && + (e.branches[if (e.origin == IrStatementOrigin.ANDAND) 1 else 0].result + as? IrConst<*>) + ?.value == isOrOr + ) { + + // resugar binary logical operators: + + val exprParent = parent.expr(e, callable) + val type = useType(e.type) + + val id = + if (e.origin == IrStatementOrigin.ANDAND) { + val id = tw.getFreshIdLabel() + tw.writeExprs_andlogicalexpr( + id, + type.javaResult.id, + exprParent.parent, + exprParent.idx + ) + id + } else { + val id = tw.getFreshIdLabel() + tw.writeExprs_orlogicalexpr( + id, + type.javaResult.id, + exprParent.parent, + exprParent.idx + ) + id + } + val locId = tw.getLocation(e) + + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, callable, exprParent.enclosingStmt) + + extractExpressionExpr( + e.branches[0].condition, + callable, + id, + 0, + exprParent.enclosingStmt + ) + + var rhsIdx = if (e.origin == IrStatementOrigin.ANDAND) 0 else 1 + extractExpressionExpr( + e.branches[rhsIdx].result, + callable, + id, + 1, + exprParent.enclosingStmt + ) + + return + } + + val exprParent = parent.expr(e, callable) + val id = tw.getFreshIdLabel() + val type = useType(e.type) + val locId = tw.getLocation(e) + tw.writeExprs_whenexpr( + id, + type.javaResult.id, + exprParent.parent, + exprParent.idx + ) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, callable, exprParent.enclosingStmt) + if (e.origin == IrStatementOrigin.IF) { + tw.writeWhen_if(id) + } + e.branches.forEachIndexed { i, b -> + val bId = tw.getFreshIdLabel() + val bLocId = tw.getLocation(b) + tw.writeStmts_whenbranch(bId, id, i, callable) + tw.writeHasLocation(bId, bLocId) + extractExpressionExpr(b.condition, callable, bId, 0, bId) + extractExpressionStmt(b.result, callable, bId, 1) + if (b is IrElseBranch) { + tw.writeWhen_branch_else(bId) + } + } + } + is IrGetClass -> { + val exprParent = parent.expr(e, callable) + val id = tw.getFreshIdLabel() + val locId = tw.getLocation(e) + val type = useType(e.type) + tw.writeExprs_getclassexpr( + id, + type.javaResult.id, + exprParent.parent, + exprParent.idx + ) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, callable, exprParent.enclosingStmt) + extractExpressionExpr(e.argument, callable, id, 0, exprParent.enclosingStmt) + } + is IrTypeOperatorCall -> { + val exprParent = parent.expr(e, callable) + extractTypeOperatorCall( + e, + callable, + exprParent.parent, + exprParent.idx, + exprParent.enclosingStmt + ) + } + is IrVararg -> { + // There are lowered IR cases when the vararg expression is not within a call, + // such as + // val temp0 = [*expr]. + // This AST element can also occur as a collection literal in an annotation + // class, such as + // annotation class Ann(val strings: Array = []) + val exprParent = parent.expr(e, callable) + extractArrayCreation( + e, + e.type, + e.varargElementType, + true, + e, + exprParent.parent, + exprParent.idx, + callable, + exprParent.enclosingStmt + ) + } + is IrGetObjectValue -> { + // For `object MyObject { ... }`, the .class has an + // automatically-generated `public static final MyObject INSTANCE` + // field that we are accessing here. + val exprParent = parent.expr(e, callable) + val c = getBoundSymbolOwner(e.symbol, e) ?: return + + val instance = + if (c.isCompanion) useCompanionObjectClassInstance(c) + else useObjectClassInstance(c) + + if (instance != null) { + val id = tw.getFreshIdLabel() + val type = useType(e.type) + val locId = tw.getLocation(e) + tw.writeExprs_varaccess( + id, + type.javaResult.id, + exprParent.parent, + exprParent.idx + ) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, callable, exprParent.enclosingStmt) + + tw.writeVariableBinding(id, instance.id) + } + } + is IrFunctionReference -> { + extractFunctionReference(e, parent, callable) + } + is IrFunctionExpression -> { + /* + * Extract generated class: + * ``` + * class C : Any, kotlin.FunctionI { + * constructor() { super(); } + * fun invoke(a0:T0, a1:T1, ... aI: TI): R { ... } + * } + * ``` + * or in case of big arity lambdas + * ``` + * class C : Any, kotlin.FunctionN { + * constructor() { super(); } + * fun invoke(a0:T0, a1:T1, ... aI: TI): R { ... } + * fun invoke(vararg args: Any?): R { + * return invoke(args[0] as T0, args[1] as T1, ..., args[I] as TI) + * } + * } + * ``` + **/ + + val ids = getLocallyVisibleFunctionLabels(e.function) + val locId = tw.getLocation(e) + + val ext = e.function.extensionReceiverParameter + val parameters = + if (ext != null) { + listOf(ext) + e.function.valueParameters + } else { + e.function.valueParameters + } + + var types = parameters.map { it.type } + types += e.function.returnType + + val isBigArity = types.size > BuiltInFunctionArity.BIG_ARITY + if (isBigArity) { + implementFunctionNInvoke(e.function, ids, locId, parameters) + } else { + addModifiers(ids.function, "override") + } + + val exprParent = parent.expr(e, callable) + val idLambdaExpr = tw.getFreshIdLabel() + tw.writeExprs_lambdaexpr( + idLambdaExpr, + ids.type.javaResult.id, + exprParent.parent, + exprParent.idx + ) + tw.writeExprsKotlinType(idLambdaExpr, ids.type.kotlinResult.id) + extractExprContext(idLambdaExpr, locId, callable, exprParent.enclosingStmt) + tw.writeCallableBinding(idLambdaExpr, ids.constructor) + + // todo: fix hard coded block body of lambda + tw.writeLambdaKind(idLambdaExpr, 1) + + val fnInterfaceType = getFunctionalInterfaceType(types) + if (fnInterfaceType == null) { + logger.warnElement( + "Cannot find functional interface type for function expression", + e + ) + } else { + val id = + extractGeneratedClass( + e + .function, // We're adding this function as a member, and + // changing its name to `invoke` to implement + // `kotlin.FunctionX<,,,>.invoke(,,)` + listOf(pluginContext.irBuiltIns.anyType, fnInterfaceType) + ) + + extractTypeAccessRecursive( + fnInterfaceType, + locId, + idLambdaExpr, + -3, + callable, + exprParent.enclosingStmt + ) + + tw.writeIsAnonymClass(id, idLambdaExpr) + } + } + is IrClassReference -> { + val exprParent = parent.expr(e, callable) + extractClassReference( + e, + exprParent.parent, + exprParent.idx, + callable, + exprParent.enclosingStmt + ) + } + is IrPropertyReference -> { + extractPropertyReference( + "property reference", + e, + e.getter, + e.setter, + e.field, + parent, + callable + ) + } + is IrLocalDelegatedPropertyReference -> { + extractPropertyReference( + "local delegated property reference", + e, + e.getter, + e.setter, + null, + parent, + callable + ) + } + else -> { + logger.errorElement("Unrecognised IrExpression: " + e.javaClass, e) + } + } + return + } + } + + private fun extractBlock( + e: IrContainerExpression, + statements: List, + parent: StmtExprParent, + callable: Label + ) { + val stmtParent = parent.stmt(e, callable) + extractBlock(e, statements, stmtParent.parent, stmtParent.idx, callable) + } + + private fun extractBlock( + e: IrElement, + statements: List, + parent: Label, + idx: Int, + callable: Label + ) { + val id = tw.getFreshIdLabel() + val locId = tw.getLocation(e) + tw.writeStmts_block(id, parent, idx, callable) + tw.writeHasLocation(id, locId) + statements.forEachIndexed { i, s -> extractStatement(s, callable, id, i) } + } + + private inline fun getBoundSymbolOwner( + symbol: IrBindableSymbol, + e: IrExpression + ): B? { + if (symbol.isBound) { + return symbol.owner + } + + logger.errorElement("Unbound symbol found, skipping extraction of expression", e) + return null + } + + private fun extractSuperAccess( + irType: IrType, + callable: Label, + parent: Label, + idx: Int, + enclosingStmt: Label, + locId: Label + ) = + tw.getFreshIdLabel().also { + val type = useType(irType) + tw.writeExprs_superaccess(it, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(it, type.kotlinResult.id) + extractExprContext(it, locId, callable, enclosingStmt) + extractTypeAccessRecursive(irType, locId, it, 0) + } + + private fun extractThisAccess( + type: TypeResults, + callable: Label, + parent: Label, + idx: Int, + enclosingStmt: Label, + locId: Label + ) = + tw.getFreshIdLabel().also { + tw.writeExprs_thisaccess(it, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(it, type.kotlinResult.id) + extractExprContext(it, locId, callable, enclosingStmt) + } + + private fun extractThisAccess( + irType: IrType, + callable: Label, + parent: Label, + idx: Int, + enclosingStmt: Label, + locId: Label + ) = extractThisAccess(useType(irType), callable, parent, idx, enclosingStmt, locId) + + private fun extractThisAccess( + e: IrGetValue, + thisParamParent: IrDeclarationParent, + exprParent: ExprParent, + callable: Label + ) { + val containingDeclaration = declarationStack.peek().first + val locId = tw.getLocation(e) + + if ( + containingDeclaration.shouldExtractAsStatic && + containingDeclaration.parentClassOrNull?.isNonCompanionObject == true + ) { + // Use of `this` in a non-companion object member that will be lowered to a static + // function -- replace with a reference + // to the corresponding static object instance. + val instanceField = useObjectClassInstance(containingDeclaration.parentAsClass) + extractVariableAccess( + instanceField.id, + e.type, + locId, + exprParent.parent, + exprParent.idx, + callable, + exprParent.enclosingStmt + ) + .also { varAccessId -> + extractStaticTypeAccessQualifier( + containingDeclaration, + varAccessId, + locId, + callable, + exprParent.enclosingStmt + ) + } + } else { + if (thisParamParent is IrFunction) { + val overriddenAttributes = + declarationStack.findOverriddenAttributes(thisParamParent) + val replaceWithParamIdx = + overriddenAttributes?.valueParameters?.indexOf(e.symbol.owner) + if (replaceWithParamIdx != null && replaceWithParamIdx != -1) { + // Use of 'this' in a function where the dispatch receiver is passed like an + // ordinary parameter, + // such as a `$default` static function that substitutes in default arguments as + // needed. + val paramDeclarerId = + overriddenAttributes.id ?: useDeclarationParent(thisParamParent, false) + val replacementParamId = + tw.getLabelFor( + getValueParameterLabel(paramDeclarerId, replaceWithParamIdx) + ) + extractVariableAccess( + replacementParamId, + e.type, + locId, + exprParent.parent, + exprParent.idx, + callable, + exprParent.enclosingStmt + ) + return + } + } + + val id = + extractThisAccess( + e.type, + callable, + exprParent.parent, + exprParent.idx, + exprParent.enclosingStmt, + locId + ) + + fun extractTypeAccess(parent: IrClass) { + extractTypeAccessRecursive( + parent.typeWith(listOf()), + locId, + id, + 0, + callable, + exprParent.enclosingStmt + ) + } + + val owner = e.symbol.owner + when (val ownerParent = owner.parent) { + is IrFunction -> { + if ( + ownerParent.dispatchReceiverParameter == owner && + ownerParent.extensionReceiverParameter != null + ) { + + val ownerParent2 = ownerParent.parent + if (ownerParent2 is IrClass) { + extractTypeAccess(ownerParent2) + } else { + logger.errorElement("Unhandled qualifier for this", e) + } + } + } + is IrClass -> { + if (ownerParent.thisReceiver == owner) { + extractTypeAccess(ownerParent) + } + } + else -> { + logger.errorElement( + "Unexpected owner parent for this access: " + ownerParent.javaClass, + e + ) + } + } + } + } + + private fun extractVariableAccess( + variable: Label?, + type: TypeResults, + locId: Label, + parent: Label, + idx: Int, + callable: Label, + enclosingStmt: Label + ) = + tw.getFreshIdLabel().also { + tw.writeExprs_varaccess(it, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(it, type.kotlinResult.id) + extractExprContext(it, locId, callable, enclosingStmt) + + if (variable != null) { + tw.writeVariableBinding(it, variable) + } + } + + private fun extractVariableAccess( + variable: Label?, + irType: IrType, + locId: Label, + parent: Label, + idx: Int, + callable: Label, + enclosingStmt: Label + ) = + extractVariableAccess( + variable, + useType(irType), + locId, + parent, + idx, + callable, + enclosingStmt + ) + + private fun extractLoop( + loop: IrLoop, + bodyIdx: Int?, + stmtExprParent: StmtExprParent, + callable: Label, + getId: (Label, Int) -> Label + ): Label { + val stmtParent = stmtExprParent.stmt(loop, callable) + val locId = tw.getLocation(loop) + + val idx: Int + val parent: Label + + val label = loop.label + if (label != null) { + val labeledStmt = tw.getFreshIdLabel() + tw.writeStmts_labeledstmt(labeledStmt, stmtParent.parent, stmtParent.idx, callable) + tw.writeHasLocation(labeledStmt, locId) + + tw.writeNamestrings(label, "", labeledStmt) + idx = 0 + parent = labeledStmt + } else { + idx = stmtParent.idx + parent = stmtParent.parent + } + + val id = getId(parent, idx) + tw.writeHasLocation(id, locId) + + val body = loop.body + if (body != null && bodyIdx != null) { + extractExpressionStmt(body, callable, id, bodyIdx) + } + + return id + } + + private fun extractLoopWithCondition( + loop: IrLoop, + stmtExprParent: StmtExprParent, + callable: Label + ) { + val id = + extractLoop(loop, 1, stmtExprParent, callable) { parent, idx -> + if (loop is IrWhileLoop) { + tw.getFreshIdLabel().also { + tw.writeStmts_whilestmt(it, parent, idx, callable) + } + } else { + tw.getFreshIdLabel().also { + tw.writeStmts_dostmt(it, parent, idx, callable) + } + } + } + extractExpressionExpr(loop.condition, callable, id, 0, id) + } + + private fun exprIdOrFresh(id: Label?) = + id?.cast() ?: tw.getFreshIdLabel() + + private fun extractClassReference( + e: IrClassReference, + parent: Label, + idx: Int, + enclosingCallable: Label?, + enclosingStmt: Label?, + overrideId: Label? = null, + typeAccessOverrideId: Label? = null, + useJavaLangClassType: Boolean = false + ) = + exprIdOrFresh(overrideId).also { id -> + val locId = tw.getLocation(e) + val jlcType = + if (useJavaLangClassType) this.javaLangClass?.let { it.typeWith() } else null + val type = useType(jlcType ?: e.type) + tw.writeExprs_typeliteral(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, enclosingCallable, enclosingStmt) + + extractTypeAccessRecursive( + e.classType, + locId, + id, + 0, + enclosingCallable, + enclosingStmt, + overrideId = typeAccessOverrideId + ) + } + + private fun extractEnumValue( + e: IrGetEnumValue, + parent: Label, + idx: Int, + enclosingCallable: Label?, + enclosingStmt: Label?, + extractTypeAccess: Boolean = true, + overrideId: Label? = null + ) = + exprIdOrFresh(overrideId).also { id -> + val type = useType(e.type) + val locId = tw.getLocation(e) + tw.writeExprs_varaccess(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, enclosingCallable, enclosingStmt) + + getBoundSymbolOwner(e.symbol, e)?.let { owner -> + val vId = useEnumEntry(owner) + tw.writeVariableBinding(id, vId) + + if (extractTypeAccess) + extractStaticTypeAccessQualifier( + owner, + id, + locId, + enclosingCallable, + enclosingStmt + ) + } + } + + private fun escapeCharForQuotedLiteral(c: Char) = + when (c) { + '\r' -> "\\r" + '\n' -> "\\n" + '\t' -> "\\t" + '\\' -> "\\\\" + '"' -> "\\\"" + else -> c.toString() + } + + // Render a string literal as it might occur in Kotlin source. Note this is a reasonable guess; + // the real source + // could use other escape sequences to describe the same String. Importantly, this is the same + // guess the Java + // extractor makes regarding string literals occurring within annotations, which we need to + // coincide with to ensure + // database consistency. + private fun toQuotedLiteral(s: String) = + s.toCharArray().joinToString(separator = "", prefix = "\"", postfix = "\"") { c -> + escapeCharForQuotedLiteral(c) + } + + private fun extractConstant( + e: IrConst<*>, + parent: Label, + idx: Int, + enclosingCallable: Label?, + enclosingStmt: Label?, + overrideId: Label? = null + ): Label? { + + val v = e.value + return when { + v is Number && (v is Int || v is Short || v is Byte) -> { + extractConstantInteger( + v, + tw.getLocation(e), + parent, + idx, + enclosingCallable, + enclosingStmt, + overrideId = overrideId + ) + } + v is Long -> { + exprIdOrFresh(overrideId).also { id -> + val type = useType(e.type) + val locId = tw.getLocation(e) + tw.writeExprs_longliteral(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, enclosingCallable, enclosingStmt) + tw.writeNamestrings(v.toString(), v.toString(), id) + } + } + v is Float -> { + exprIdOrFresh(overrideId).also { id -> + val type = useType(e.type) + val locId = tw.getLocation(e) + tw.writeExprs_floatingpointliteral(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, enclosingCallable, enclosingStmt) + tw.writeNamestrings(v.toString(), v.toString(), id) + } + } + v is Double -> { + exprIdOrFresh(overrideId).also { id -> + val type = useType(e.type) + val locId = tw.getLocation(e) + tw.writeExprs_doubleliteral(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, enclosingCallable, enclosingStmt) + tw.writeNamestrings(v.toString(), v.toString(), id) + } + } + v is Boolean -> { + exprIdOrFresh(overrideId).also { id -> + val type = useType(e.type) + val locId = tw.getLocation(e) + tw.writeExprs_booleanliteral(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, enclosingCallable, enclosingStmt) + tw.writeNamestrings(v.toString(), v.toString(), id) + } + } + v is Char -> { + exprIdOrFresh(overrideId).also { id -> + val type = useType(e.type) + val locId = tw.getLocation(e) + tw.writeExprs_characterliteral(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, enclosingCallable, enclosingStmt) + tw.writeNamestrings(v.toString(), v.toString(), id) + } + } + v is String -> { + exprIdOrFresh(overrideId).also { id -> + val type = useType(e.type) + val locId = tw.getLocation(e) + tw.writeExprs_stringliteral(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, enclosingCallable, enclosingStmt) + tw.writeNamestrings(toQuotedLiteral(v.toString()), v.toString(), id) + } + } + v == null -> { + extractNull( + e.type, + tw.getLocation(e), + parent, + idx, + enclosingCallable, + enclosingStmt, + overrideId = overrideId + ) + } + else -> { + null.also { logger.errorElement("Unrecognised IrConst: " + v.javaClass, e) } + } + } + } + + private fun IrValueParameter.isExtensionReceiver(): Boolean { + val parentFun = parent as? IrFunction ?: return false + return parentFun.extensionReceiverParameter == this + } + + private open inner class GeneratedClassHelper( + protected val locId: Label, + protected val ids: GeneratedClassLabels + ) { + protected val classId = ids.type.javaResult.id.cast() + + /** + * Extract a parameter to field assignment, such as `this.field = paramName` below: + * ``` + * constructor(paramName: type) { + * this.field = paramName + * } + * ``` + */ + fun extractParameterToFieldAssignmentInConstructor( + paramName: String, + paramType: IrType, + fieldId: Label, + paramIdx: Int, + stmtIdx: Int + ) { + val paramId = tw.getFreshIdLabel() + extractValueParameter( + paramId, + paramType, + paramName, + locId, + ids.constructor, + paramIdx, + paramId, + syntheticParameterNames = false, + isVararg = false, + isNoinline = false, + isCrossinline = false + ) + + extractExpressionStmt(locId, ids.constructorBlock, stmtIdx, ids.constructor).also { + assignmentStmtId -> + extractAssignExpr( + paramType, + locId, + assignmentStmtId, + 0, + ids.constructor, + assignmentStmtId + ) + .also { assignmentId -> + extractVariableAccess( + fieldId, + paramType, + locId, + assignmentId, + 0, + ids.constructor, + assignmentStmtId + ) + .also { lhsId -> + extractThisAccess( + ids.type, + ids.constructor, + lhsId, + -1, + assignmentStmtId, + locId + ) + } + extractVariableAccess( + paramId, + paramType, + locId, + assignmentId, + 1, + ids.constructor, + assignmentStmtId + ) + } + } + } + } + + data class ReceiverInfo( + val receiver: IrExpression, + val type: IrType, + val field: Label, + val indexOffset: Int + ) + + private fun makeReceiverInfo(receiver: IrExpression?, indexOffset: Int): ReceiverInfo? { + if (receiver == null) { + return null + } + val type = receiver.type + val field: Label = tw.getFreshIdLabel() + return ReceiverInfo(receiver, type, field, indexOffset) + } + + /** + * This is used when extracting callable references, i.e. `::someCallable` or + * `::someReceiver::someCallable`. + */ + private open inner class CallableReferenceHelper( + protected val callableReferenceExpr: IrCallableReference, + locId: Label, + ids: GeneratedClassLabels + ) : GeneratedClassHelper(locId, ids) { + + // Only one of the receivers can be non-null, but we defensively handle the case when both + // are null anyway + private val dispatchReceiverInfo = + makeReceiverInfo(callableReferenceExpr.dispatchReceiver, 0) + private val extensionReceiverInfo = + makeReceiverInfo( + callableReferenceExpr.extensionReceiver, + if (dispatchReceiverInfo == null) 0 else 1 + ) + + fun extractReceiverField() { + val firstAssignmentStmtIdx = 1 + + if (dispatchReceiverInfo != null) { + extractField( + dispatchReceiverInfo.field, + "", + dispatchReceiverInfo.type, + classId, + locId, + DescriptorVisibilities.PRIVATE, + callableReferenceExpr, + isExternalDeclaration = false, + isFinal = true, + isStatic = false + ) + extractParameterToFieldAssignmentInConstructor( + "", + dispatchReceiverInfo.type, + dispatchReceiverInfo.field, + 0 + dispatchReceiverInfo.indexOffset, + firstAssignmentStmtIdx + dispatchReceiverInfo.indexOffset + ) + } + + if (extensionReceiverInfo != null) { + extractField( + extensionReceiverInfo.field, + "", + extensionReceiverInfo.type, + classId, + locId, + DescriptorVisibilities.PRIVATE, + callableReferenceExpr, + isExternalDeclaration = false, + isFinal = true, + isStatic = false + ) + extractParameterToFieldAssignmentInConstructor( + "", + extensionReceiverInfo.type, + extensionReceiverInfo.field, + 0 + extensionReceiverInfo.indexOffset, + firstAssignmentStmtIdx + extensionReceiverInfo.indexOffset + ) + } + } + + protected fun writeVariableAccessInFunctionBody( + pType: TypeResults, + idx: Int, + variable: Label, + parent: Label, + callable: Label, + stmt: Label + ): Label { + val pId = tw.getFreshIdLabel() + tw.writeExprs_varaccess(pId, pType.javaResult.id, parent, idx) + tw.writeExprsKotlinType(pId, pType.kotlinResult.id) + tw.writeVariableBinding(pId, variable) + extractExprContext(pId, locId, callable, stmt) + return pId + } + + private fun writeFieldAccessInFunctionBody( + pType: IrType, + idx: Int, + variable: Label, + parent: Label, + callable: Label, + stmt: Label + ) { + val accessId = + writeVariableAccessInFunctionBody( + useType(pType), + idx, + variable, + parent, + callable, + stmt + ) + writeThisAccess(accessId, callable, stmt) + } + + protected fun writeThisAccess( + parent: Label, + callable: Label, + stmt: Label + ) { + extractThisAccess(ids.type, callable, parent, -1, stmt, locId) + } + + fun extractFieldWriteOfReflectionTarget( + labels: FunctionLabels, // labels of the containing function + target: IrFieldSymbol, // the target field being accessed) + ) { + val fieldType = useType(target.owner.type) + + extractExpressionStmt(locId, labels.blockId, 0, labels.methodId).also { exprStmtId -> + extractAssignExpr( + target.owner.type, + locId, + exprStmtId, + 0, + labels.methodId, + exprStmtId + ) + .also { assignExprId -> + extractFieldAccess(fieldType, assignExprId, exprStmtId, labels, target) + val p = labels.parameters.first() + writeVariableAccessInFunctionBody( + p.second, + 1, + p.first, + assignExprId, + labels.methodId, + exprStmtId + ) + } + } + } + + fun extractFieldReturnOfReflectionTarget( + labels: FunctionLabels, // labels of the containing function + target: IrFieldSymbol, // the target field being accessed + ) { + val retId = tw.getFreshIdLabel() + tw.writeStmts_returnstmt(retId, labels.blockId, 0, labels.methodId) + tw.writeHasLocation(retId, locId) + + val fieldType = useType(target.owner.type) + extractFieldAccess(fieldType, retId, retId, labels, target) + } + + private fun extractFieldAccess( + fieldType: TypeResults, + parent: Label, + stmt: Label, + labels: FunctionLabels, + target: IrFieldSymbol + ) { + val accessId = tw.getFreshIdLabel() + tw.writeExprs_varaccess(accessId, fieldType.javaResult.id, parent, 0) + tw.writeExprsKotlinType(accessId, fieldType.kotlinResult.id) + + extractExprContext(accessId, locId, labels.methodId, stmt) + + val fieldId = useField(target.owner) + tw.writeVariableBinding(accessId, fieldId) + + if (dispatchReceiverInfo != null) { + writeFieldAccessInFunctionBody( + dispatchReceiverInfo.type, + -1, + dispatchReceiverInfo.field, + accessId, + labels.methodId, + stmt + ) + } + } + + /** + * Extracts a call to `target` inside the function identified by `labels`. Special + * parameters (`dispatch` and `extension`) are also handled. + * + * Examples are: + * ``` + * this..fn(this., param1, param2, param3, ...) + * param1.fn(this., param2, ...) + * param1.fn(param2, param3, ...) + * fn(this., param1, param2, ...) + * fn(param1, param2, ...) + * new MyType(param1, param2, ...) + * ``` + * + * The parameters with default argument values cover special cases: + * - dispatchReceiverIdx is usually -1, except if a constructor is referenced + * - big arity function references need to call `invoke` with arguments received in an + * object array: `fn(param1[0] as T0, param1[1] as T1, ...)` + */ + fun extractCallToReflectionTarget( + labels: FunctionLabels, // labels of the containing function + target: IrFunctionSymbol, // the target function/constructor being called + returnType: + IrType, // the return type of the called function. Note that + // `target.owner.returnType` and `returnType` doesn't match for generic + // functions + expressionTypeArgs: List, // type arguments of the extracted expression + classTypeArgsIncludingOuterClasses: + List< + IrTypeArgument + >?, // type arguments of the class containing the callable reference + dispatchReceiverIdx: Int = + -1, // dispatch receiver index: -1 in case of functions, -2 for constructors + bigArityParameterTypes: List? = + null // parameter types used for the cast expressions in a big arity `invoke` + // invocation. null if not a big arity invocation. + ) { + // Return statement of generated function: + val retId = tw.getFreshIdLabel() + tw.writeStmts_returnstmt(retId, labels.blockId, 0, labels.methodId) + tw.writeHasLocation(retId, locId) + + // Call to target function: + val callType = useType(returnType) + + val callId: Label = + if (target is IrConstructorSymbol) { + val callId = tw.getFreshIdLabel() + tw.writeExprs_newexpr(callId, callType.javaResult.id, retId, 0) + tw.writeExprsKotlinType(callId, callType.kotlinResult.id) + + extractConstructorTypeAccess( + returnType, + callType, + target, + locId, + callId, + -3, + labels.methodId, + retId + ) + callId + } else { + val callId = tw.getFreshIdLabel() + tw.writeExprs_methodaccess(callId, callType.javaResult.id, retId, 0) + tw.writeExprsKotlinType(callId, callType.kotlinResult.id) + extractTypeArguments( + expressionTypeArgs, + locId, + callId, + labels.methodId, + retId, + -2, + true + ) + callId + } + + extractExprContext(callId, locId, labels.methodId, retId) + + val callableId = + useFunction( + target.owner.realOverrideTarget, + classTypeArgsIncludingOuterClasses + ) + if (callableId == null) { + logger.error("Cannot get ID for reflection target") + } else { + tw.writeCallableBinding(callId.cast(), callableId) + } + + val useFirstArgAsDispatch: Boolean + if (dispatchReceiverInfo != null) { + writeFieldAccessInFunctionBody( + dispatchReceiverInfo.type, + dispatchReceiverIdx, + dispatchReceiverInfo.field, + callId, + labels.methodId, + retId + ) + + useFirstArgAsDispatch = false + } else { + if (target.owner.isLocalFunction()) { + val ids = getLocallyVisibleFunctionLabels(target.owner) + extractNewExprForLocalFunction(ids, callId, locId, labels.methodId, retId) + useFirstArgAsDispatch = false + } else { + useFirstArgAsDispatch = target.owner.dispatchReceiverParameter != null + + if (isStaticFunction(target.owner)) { + extractStaticTypeAccessQualifier( + target.owner, + callId, + locId, + labels.methodId, + retId + ) + } + } + } + + val extensionIdxOffset: Int + if (extensionReceiverInfo != null) { + writeFieldAccessInFunctionBody( + extensionReceiverInfo.type, + 0, + extensionReceiverInfo.field, + callId, + labels.methodId, + retId + ) + extensionIdxOffset = 1 + } else { + extensionIdxOffset = 0 + } + + if (bigArityParameterTypes != null) { + // In case we're extracting a big arity function reference: + addArgumentsToInvocationInInvokeNBody( + bigArityParameterTypes, + labels, + retId, + callId, + locId, + extensionIdxOffset, + useFirstArgAsDispatch, + dispatchReceiverIdx + ) + } else { + val dispatchIdxOffset = if (useFirstArgAsDispatch) 1 else 0 + for ((pIdx, p) in labels.parameters.withIndex()) { + val childIdx = + if (pIdx == 0 && useFirstArgAsDispatch) { + dispatchReceiverIdx + } else { + pIdx + extensionIdxOffset - dispatchIdxOffset + } + writeVariableAccessInFunctionBody( + p.second, + childIdx, + p.first, + callId, + labels.methodId, + retId + ) + } + } + } + + fun extractConstructorArguments( + callable: Label, + idCtorRef: Label, + enclosingStmt: Label + ) { + if (dispatchReceiverInfo != null) { + extractExpressionExpr( + dispatchReceiverInfo.receiver, + callable, + idCtorRef, + 0 + dispatchReceiverInfo.indexOffset, + enclosingStmt + ) + } + + if (extensionReceiverInfo != null) { + extractExpressionExpr( + extensionReceiverInfo.receiver, + callable, + idCtorRef, + 0 + extensionReceiverInfo.indexOffset, + enclosingStmt + ) + } + } + } + + private inner class PropertyReferenceHelper( + callableReferenceExpr: IrCallableReference, + locId: Label, + ids: GeneratedClassLabels + ) : CallableReferenceHelper(callableReferenceExpr, locId, ids) { + + fun extractPropertyReferenceInvoke( + getId: Label, + getterParameterTypes: List, + getterReturnType: IrType + ) { + // Extracting this method is not (strictly) needed for interface member implementation. + // `[Mutable]PropertyReferenceX` already implements it, but its signature doesn't match + // the + // generic one, because it's a raw method implementation. Also, by adding the `invoke` + // explicitly, + // we have better data flow analysis support. + val invokeLabels = + addFunctionManual( + tw.getFreshIdLabel(), + OperatorNameConventions.INVOKE.asString(), + getterParameterTypes, + getterReturnType, + classId, + locId + ) + + // return this.get(a0, a1, ...) + val retId = tw.getFreshIdLabel() + tw.writeStmts_returnstmt(retId, invokeLabels.blockId, 0, invokeLabels.methodId) + tw.writeHasLocation(retId, locId) + + // Call to target function: + val callType = useType(getterReturnType) + val callId = tw.getFreshIdLabel() + tw.writeExprs_methodaccess(callId, callType.javaResult.id, retId, 0) + tw.writeExprsKotlinType(callId, callType.kotlinResult.id) + extractExprContext(callId, locId, invokeLabels.methodId, retId) + + tw.writeCallableBinding(callId, getId) + + this.writeThisAccess(callId, invokeLabels.methodId, retId) + for ((pIdx, p) in invokeLabels.parameters.withIndex()) { + this.writeVariableAccessInFunctionBody( + p.second, + pIdx, + p.first, + callId, + invokeLabels.methodId, + retId + ) + } + } + } + + private val propertyRefType by lazy { + referenceExternalClass("kotlin.jvm.internal.PropertyReference")?.typeWith() + } + + private fun extractPropertyReference( + exprKind: String, + propertyReferenceExpr: IrCallableReference, + getter: IrSimpleFunctionSymbol?, + setter: IrSimpleFunctionSymbol?, + backingField: IrFieldSymbol?, + parent: StmtExprParent, + callable: Label + ) { + with(exprKind, propertyReferenceExpr) { + /* + * Extract generated class: + * ``` + * class C : kotlin.jvm.internal.PropertyReference, kotlin.reflect.KMutableProperty0 { + * private dispatchReceiver: TD + * constructor(dispatchReceiver: TD) { + * super() + * this.dispatchReceiver = dispatchReceiver + * } + * + * override fun get(): R { return this.dispatchReceiver.FN1() } + * + * override fun set(a0: R): Unit { return this.dispatchReceiver.FN2(a0) } + * + * override fun invoke(): R { return this.get() } + * } + * ``` + * + * Variations: + * - KProperty vs KMutableProperty + * - KProperty0<> vs KProperty1<,> + * - no receiver vs dispatchReceiver vs extensionReceiver + **/ + + val kPropertyType = propertyReferenceExpr.type + if (kPropertyType !is IrSimpleType) { + logger.errorElement( + "Unexpected: property reference with non simple type. ${kPropertyType.classFqName?.asString()}", + propertyReferenceExpr + ) + return + } + val kPropertyClass = kPropertyType.classOrNull + if (kPropertyClass == null) { + logger.errorElement( + "Cannot find class for kPropertyType. ${kPropertyType.classFqName?.asString()}", + propertyReferenceExpr + ) + return + } + val parameterTypes: List? = + kPropertyType.arguments + .map { + if (it is IrType) { + it + } else { + logger.errorElement( + "Unexpected: Non-IrType (${it.javaClass}) property reference parameter.", + propertyReferenceExpr + ) + null + } + } + .requireNoNullsOrNull() + if (parameterTypes == null) { + logger.errorElement( + "Unexpected: One or more non-IrType property reference parameters.", + propertyReferenceExpr + ) + return + } + + val locId = tw.getLocation(propertyReferenceExpr) + + val javaResult = TypeResult(tw.getFreshIdLabel(), "", "") + val kotlinResult = TypeResult(tw.getFreshIdLabel(), "", "") + tw.writeKt_notnull_types(kotlinResult.id, javaResult.id) + val ids = + GeneratedClassLabels( + TypeResults(javaResult, kotlinResult), + constructor = tw.getFreshIdLabel(), + constructorBlock = tw.getFreshIdLabel() + ) + + val declarationParent = + peekDeclStackAsDeclarationParent(propertyReferenceExpr) ?: return + // The base class could be `Any`. `PropertyReference` is used to keep symmetry with + // function references. + val baseClass = propertyRefType ?: pluginContext.irBuiltIns.anyType + + val classId = + extractGeneratedClass( + ids, + listOf(baseClass, kPropertyType), + locId, + propertyReferenceExpr, + declarationParent + ) + + val helper = PropertyReferenceHelper(propertyReferenceExpr, locId, ids) + + helper.extractReceiverField() + + val classTypeArguments = + (propertyReferenceExpr.dispatchReceiver?.type as? IrSimpleType)?.arguments + ?: if ( + (getter?.owner?.dispatchReceiverParameter + ?: setter?.owner?.dispatchReceiverParameter) != null + ) { + (kPropertyType.arguments.first() as? IrSimpleType)?.arguments + } else { + null + } + + val expressionTypeArguments = + (0 until propertyReferenceExpr.typeArgumentsCount).mapNotNull { + propertyReferenceExpr.getTypeArgument(it) + } + + val idPropertyRef = tw.getFreshIdLabel() + + val getterParameterTypes = parameterTypes.dropLast(1) + val getterReturnType = parameterTypes.last() + + if (getter != null) { + val getterCallableId = + useFunction(getter.owner.realOverrideTarget, classTypeArguments) + if (getterCallableId == null) { + logger.errorElement("Cannot get ID for getter", propertyReferenceExpr) + } else { + val getLabels = + addFunctionManual( + tw.getFreshIdLabel(), + OperatorNameConventions.GET.asString(), + getterParameterTypes, + getterReturnType, + classId, + locId + ) + + helper.extractCallToReflectionTarget( + getLabels, + getter, + getterReturnType, + expressionTypeArguments, + classTypeArguments + ) + + tw.writePropertyRefGetBinding(idPropertyRef, getterCallableId) + + helper.extractPropertyReferenceInvoke( + getLabels.methodId, + getterParameterTypes, + getterReturnType + ) + } + } else { + // Property without a getter. + if (backingField == null) { + logger.errorElement( + "Expected to find getter or backing field for property reference.", + propertyReferenceExpr + ) + return + } + + val getLabels = + addFunctionManual( + tw.getFreshIdLabel(), + OperatorNameConventions.GET.asString(), + getterParameterTypes, + getterReturnType, + classId, + locId + ) + val fieldId = useField(backingField.owner) + + helper.extractFieldReturnOfReflectionTarget(getLabels, backingField) + + tw.writePropertyRefFieldBinding(idPropertyRef, fieldId) + + helper.extractPropertyReferenceInvoke( + getLabels.methodId, + getterParameterTypes, + getterReturnType + ) + } + + if (setter != null) { + val setterCallableId = + useFunction(setter.owner.realOverrideTarget, classTypeArguments) + if (setterCallableId == null) { + logger.errorElement("Cannot get ID for setter", propertyReferenceExpr) + } else { + val setLabels = + addFunctionManual( + tw.getFreshIdLabel(), + OperatorNameConventions.SET.asString(), + parameterTypes, + pluginContext.irBuiltIns.unitType, + classId, + locId + ) + + helper.extractCallToReflectionTarget( + setLabels, + setter, + pluginContext.irBuiltIns.unitType, + expressionTypeArguments, + classTypeArguments + ) + + tw.writePropertyRefSetBinding(idPropertyRef, setterCallableId) + } + } else { + if (backingField != null && !backingField.owner.isFinal) { + val setLabels = + addFunctionManual( + tw.getFreshIdLabel(), + OperatorNameConventions.SET.asString(), + parameterTypes, + pluginContext.irBuiltIns.unitType, + classId, + locId + ) + val fieldId = useField(backingField.owner) + + helper.extractFieldWriteOfReflectionTarget(setLabels, backingField) + + tw.writePropertyRefFieldBinding(idPropertyRef, fieldId) + } + } + + // Add constructor (property ref) call: + val exprParent = parent.expr(propertyReferenceExpr, callable) + tw.writeExprs_propertyref( + idPropertyRef, + ids.type.javaResult.id, + exprParent.parent, + exprParent.idx + ) + tw.writeExprsKotlinType(idPropertyRef, ids.type.kotlinResult.id) + extractExprContext(idPropertyRef, locId, callable, exprParent.enclosingStmt) + tw.writeCallableBinding(idPropertyRef, ids.constructor) + + extractTypeAccessRecursive( + kPropertyType, + locId, + idPropertyRef, + -3, + callable, + exprParent.enclosingStmt + ) + + helper.extractConstructorArguments(callable, idPropertyRef, exprParent.enclosingStmt) + + tw.writeIsAnonymClass(classId, idPropertyRef) + } + } + + private val functionRefType by lazy { + referenceExternalClass("kotlin.jvm.internal.FunctionReference")?.typeWith() + } + + private fun extractFunctionReference( + functionReferenceExpr: IrFunctionReference, + parent: StmtExprParent, + callable: Label + ) { + with("function reference", functionReferenceExpr) { + val target = + if (functionReferenceExpr.origin == IrStatementOrigin.ADAPTED_FUNCTION_REFERENCE) + // For an adaptation (e.g. to adjust the number or type of arguments or results), + // the symbol field points at the adapter while `.reflectionTarget` points at the + // source-level target. + functionReferenceExpr.symbol + else + // TODO: Consider whether we could always target the symbol + functionReferenceExpr.reflectionTarget + ?: run { + logger.warnElement( + "Expected to find reflection target for function reference. Using underlying symbol instead.", + functionReferenceExpr + ) + functionReferenceExpr.symbol + } + + /* + * Extract generated class: + * ``` + * class C : kotlin.jvm.internal.FunctionReference, kotlin.FunctionI { + * private dispatchReceiver: TD + * private extensionReceiver: TE + * constructor(dispatchReceiver: TD, extensionReceiver: TE) { + * super() + * this.dispatchReceiver = dispatchReceiver + * this.extensionReceiver = extensionReceiver + * } + * fun invoke(a0:T0, a1:T1, ... aI: TI): R { return this.dispatchReceiver.FN(a0,a1,...,aI) } OR + * fun invoke( a1:T1, ... aI: TI): R { return this.dispatchReceiver.FN(this.dispatchReceiver,a1,...,aI) } OR + * fun invoke(a0:T0, a1:T1, ... aI: TI): R { return Ctor(a0,a1,...,aI) } + * } + * ``` + * or in case of big arity lambdas ???? + * ``` + * class C : kotlin.jvm.internal.FunctionReference, kotlin.FunctionN { + * private receiver: TD + * constructor(receiver: TD) { super(); this.receiver = receiver; } + * fun invoke(vararg args: Any?): R { + * return this.receiver.FN(args[0] as T0, args[1] as T1, ..., args[I] as TI) + * } + * } + * ``` + **/ + + if ( + functionReferenceExpr.dispatchReceiver != null && + functionReferenceExpr.extensionReceiver != null + ) { + logger.errorElement( + "Unexpected: dispatchReceiver and extensionReceiver are both non-null", + functionReferenceExpr + ) + return + } + + if ( + target.owner.dispatchReceiverParameter != null && + target.owner.extensionReceiverParameter != null + ) { + logger.errorElement( + "Unexpected: dispatch and extension parameters are both non-null", + functionReferenceExpr + ) + return + } + + val type = functionReferenceExpr.type + if (type !is IrSimpleType) { + logger.errorElement( + "Unexpected: function reference with non simple type. ${type.classFqName?.asString()}", + functionReferenceExpr + ) + return + } + + val parameterTypes: List? = + type.arguments + .map { + if (it is IrType) { + it + } else { + logger.errorElement( + "Unexpected: Non-IrType (${it.javaClass}) function reference parameter.", + functionReferenceExpr + ) + null + } + } + .requireNoNullsOrNull() + if (parameterTypes == null) { + logger.errorElement( + "Unexpected: One or more non-IrType function reference parameters.", + functionReferenceExpr + ) + return + } + + val dispatchReceiverIdx: Int + val expressionTypeArguments: List + val classTypeArguments: List? + + if (target is IrConstructorSymbol) { + // In case a constructor is referenced, the return type of the `KFunctionX<,,,>` is + // the type if the constructed type. + classTypeArguments = (type.arguments.last() as? IrSimpleType)?.arguments + expressionTypeArguments = listOf(parameterTypes.last()) + dispatchReceiverIdx = -2 + } else { + classTypeArguments = + (functionReferenceExpr.dispatchReceiver?.type as? IrSimpleType)?.arguments + ?: if (target.owner.dispatchReceiverParameter != null) { + (type.arguments.first() as? IrSimpleType)?.arguments + } else { + null + } + expressionTypeArguments = + (0 until functionReferenceExpr.typeArgumentsCount).mapNotNull { + functionReferenceExpr.getTypeArgument(it) + } + dispatchReceiverIdx = -1 + } + + val locId = tw.getLocation(functionReferenceExpr) + + val javaResult = TypeResult(tw.getFreshIdLabel(), "", "") + val kotlinResult = TypeResult(tw.getFreshIdLabel(), "", "") + tw.writeKt_notnull_types(kotlinResult.id, javaResult.id) + val ids = + LocallyVisibleFunctionLabels( + TypeResults(javaResult, kotlinResult), + constructor = tw.getFreshIdLabel(), + function = tw.getFreshIdLabel(), + constructorBlock = tw.getFreshIdLabel() + ) + + // Add constructor (member ref) call: + val exprParent = parent.expr(functionReferenceExpr, callable) + val idMemberRef = tw.getFreshIdLabel() + tw.writeExprs_memberref( + idMemberRef, + ids.type.javaResult.id, + exprParent.parent, + exprParent.idx + ) + tw.writeExprsKotlinType(idMemberRef, ids.type.kotlinResult.id) + extractExprContext(idMemberRef, locId, callable, exprParent.enclosingStmt) + tw.writeCallableBinding(idMemberRef, ids.constructor) + + val targetCallableId = + useFunction(target.owner.realOverrideTarget, classTypeArguments) + if (targetCallableId == null) { + logger.errorElement( + "Cannot get ID for function reference callable", + functionReferenceExpr + ) + } else { + tw.writeMemberRefBinding(idMemberRef, targetCallableId) + } + + val helper = CallableReferenceHelper(functionReferenceExpr, locId, ids) + + val fnInterfaceType = getFunctionalInterfaceTypeWithTypeArgs(type.arguments) + if (fnInterfaceType == null) { + logger.warnElement( + "Cannot find functional interface type for function reference", + functionReferenceExpr + ) + } else { + val declarationParent = + peekDeclStackAsDeclarationParent(functionReferenceExpr) ?: return + // `FunctionReference` base class is required, because that's implementing + // `KFunction`. + val baseClass = functionRefType ?: pluginContext.irBuiltIns.anyType + + val classId = + extractGeneratedClass( + ids, + listOf(baseClass, fnInterfaceType), + locId, + functionReferenceExpr, + declarationParent, + null, + { it.valueParameters.size == 1 } + ) { + // The argument to FunctionReference's constructor is the function arity. + extractConstantInteger( + type.arguments.size - 1, + locId, + it, + 0, + ids.constructor, + it + ) + } + + helper.extractReceiverField() + + val isBigArity = type.arguments.size > BuiltInFunctionArity.BIG_ARITY + val funLabels = + if (isBigArity) { + addFunctionNInvoke(ids.function, parameterTypes.last(), classId, locId) + } else { + addFunctionInvoke( + ids.function, + parameterTypes.dropLast(1), + parameterTypes.last(), + classId, + locId + ) + } + + helper.extractCallToReflectionTarget( + funLabels, + target, + parameterTypes.last(), + expressionTypeArguments, + classTypeArguments, + dispatchReceiverIdx, + if (isBigArity) parameterTypes.dropLast(1) else null + ) + + val typeAccessArguments = + if (isBigArity) listOf(parameterTypes.last()) else parameterTypes + if (target is IrConstructorSymbol) { + val returnType = typeAccessArguments.last() + + val typeAccessId = + extractTypeAccess( + useType(fnInterfaceType, TypeContext.OTHER), + locId, + idMemberRef, + -3, + callable, + exprParent.enclosingStmt + ) + typeAccessArguments.dropLast(1).forEachIndexed { argIdx, arg -> + extractTypeAccessRecursive( + arg, + locId, + typeAccessId, + argIdx, + callable, + exprParent.enclosingStmt, + TypeContext.GENERIC_ARGUMENT + ) + } + + extractConstructorTypeAccess( + returnType, + useType(returnType), + target, + locId, + typeAccessId, + typeAccessArguments.count() - 1, + callable, + exprParent.enclosingStmt + ) + } else { + extractTypeAccessRecursive( + fnInterfaceType, + locId, + idMemberRef, + -3, + callable, + exprParent.enclosingStmt + ) + } + + helper.extractConstructorArguments(callable, idMemberRef, exprParent.enclosingStmt) + + tw.writeIsAnonymClass(classId, idMemberRef) + } + } + } + + private fun getFunctionalInterfaceType(functionNTypeArguments: List) = + getFunctionalInterfaceTypeWithTypeArgs( + functionNTypeArguments.map { makeTypeProjection(it, Variance.INVARIANT) } + ) + + private fun getFunctionalInterfaceTypeWithTypeArgs( + functionNTypeArguments: List + ) = + if (functionNTypeArguments.size > BuiltInFunctionArity.BIG_ARITY) + referenceExternalClass("kotlin.jvm.functions.FunctionN") + ?.symbol + ?.typeWithArguments(listOf(functionNTypeArguments.last())) + else + functionN(pluginContext)(functionNTypeArguments.size - 1) + .symbol + .typeWithArguments(functionNTypeArguments) + + private data class FunctionLabels( + val methodId: Label, + val blockId: Label, + val parameters: List, TypeResults>> + ) + + /** + * Adds a function `invoke(a: Any[])` with the specified return type to the class identified by + * `parentId`. + */ + private fun addFunctionNInvoke( + methodId: Label, + returnType: IrType, + parentId: Label, + locId: Label + ): FunctionLabels { + return addFunctionInvoke( + methodId, + listOf(pluginContext.irBuiltIns.arrayClass.typeWith(pluginContext.irBuiltIns.anyNType)), + returnType, + parentId, + locId + ) + } + + /** + * Adds a function named `invoke` with the specified parameter types and return type to the + * class identified by `parentId`. + */ + private fun addFunctionInvoke( + methodId: Label, + parameterTypes: List, + returnType: IrType, + parentId: Label, + locId: Label + ): FunctionLabels { + return addFunctionManual( + methodId, + OperatorNameConventions.INVOKE.asString(), + parameterTypes, + returnType, + parentId, + locId + ) + } + + /** + * Extracts a function with the given name, parameter types, return type, containing type, and + * location. + */ + private fun addFunctionManual( + methodId: Label, + name: String, + parameterTypes: List, + returnType: IrType, + parentId: Label, + locId: Label + ): FunctionLabels { + + val parameters = + parameterTypes.mapIndexed { idx, p -> + val paramId = tw.getFreshIdLabel() + val paramType = + extractValueParameter( + paramId, + p, + "a$idx", + locId, + methodId, + idx, + paramId, + syntheticParameterNames = false, + isVararg = false, + isNoinline = false, + isCrossinline = false + ) + + Pair(paramId, paramType) + } + + val paramsSignature = + parameters.joinToString(separator = ",", prefix = "(", postfix = ")") { + signatureOrWarn(it.second.javaResult, declarationStack.tryPeek()?.first) + } + + val rt = useType(returnType, TypeContext.RETURN) + tw.writeMethods( + methodId, + name, + "$name$paramsSignature", + rt.javaResult.id, + parentId, + methodId + ) + tw.writeMethodsKotlinType(methodId, rt.kotlinResult.id) + tw.writeHasLocation(methodId, locId) + + addModifiers(methodId, "public") + addModifiers(methodId, "override") + + return FunctionLabels(methodId, extractBlockBody(methodId, locId), parameters) + } + + /* + * This function generates an implementation for `fun kotlin.FunctionN.invoke(vararg args: Any?): R` + * + * The following body is added: + * ``` + * fun invoke(vararg a0: Any?): R { + * return invoke(a0[0] as T0, a0[1] as T1, ..., a0[I] as TI) + * } + * ``` + * */ + private fun implementFunctionNInvoke( + lambda: IrFunction, + ids: LocallyVisibleFunctionLabels, + locId: Label, + parameters: List + ) { + val funLabels = + addFunctionNInvoke( + tw.getFreshIdLabel(), + lambda.returnType, + ids.type.javaResult.id.cast(), + locId + ) + + // Return + val retId = tw.getFreshIdLabel() + tw.writeStmts_returnstmt(retId, funLabels.blockId, 0, funLabels.methodId) + tw.writeHasLocation(retId, locId) + + // Call to original `invoke`: + val callId = tw.getFreshIdLabel() + val callType = useType(lambda.returnType) + tw.writeExprs_methodaccess(callId, callType.javaResult.id, retId, 0) + tw.writeExprsKotlinType(callId, callType.kotlinResult.id) + extractExprContext(callId, locId, funLabels.methodId, retId) + val calledMethodId = useFunction(lambda) + if (calledMethodId == null) { + logger.errorElement("Cannot get ID for called lambda", lambda) + } else { + tw.writeCallableBinding(callId, calledMethodId) + } + + // this access + extractThisAccess(ids.type, funLabels.methodId, callId, -1, retId, locId) + + addArgumentsToInvocationInInvokeNBody( + parameters.map { it.type }, + funLabels, + retId, + callId, + locId + ) + } + + /** + * Adds the arguments to the method call inside `invoke(a0: Any[])`. Each argument is an array + * access with a cast: + * ``` + * fun invoke(a0: Any[]) : T { + * return fn(a0[0] as T0, a0[1] as T1, ...) + * } + * ``` + */ + private fun addArgumentsToInvocationInInvokeNBody( + parameterTypes: List, // list of parameter types + funLabels: FunctionLabels, // already generated labels for the function definition + enclosingStmtId: Label, // label for the enclosing statement (return) + exprParentId: Label, // label for the expression parent (call) + locId: Label, // label for the location of all generated items + firstArgumentOffset: Int = + 0, // 0 or 1, the index used for the first argument. 1 in case an extension parameter is + // already accessed at index 0 + useFirstArgAsDispatch: Boolean = + false, // true if the first argument should be used as the dispatch receiver + dispatchReceiverIdx: Int = + -1 // index of the dispatch receiver. -1 in case of functions, -2 in case of + // constructors + ) { + val argsParamType = + pluginContext.irBuiltIns.arrayClass.typeWith(pluginContext.irBuiltIns.anyNType) + val argsType = useType(argsParamType) + val anyNType = useType(pluginContext.irBuiltIns.anyNType) + + val dispatchIdxOffset = if (useFirstArgAsDispatch) 1 else 0 + + for ((pIdx, pType) in parameterTypes.withIndex()) { + // `a0[i] as Ti` is generated below for each parameter + + val childIdx = + if (pIdx == 0 && useFirstArgAsDispatch) { + dispatchReceiverIdx + } else { + pIdx + firstArgumentOffset - dispatchIdxOffset + } + + // cast: `(Ti)a0[i]` + val castId = tw.getFreshIdLabel() + val type = useType(pType) + tw.writeExprs_castexpr(castId, type.javaResult.id, exprParentId, childIdx) + tw.writeExprsKotlinType(castId, type.kotlinResult.id) + extractExprContext(castId, locId, funLabels.methodId, enclosingStmtId) + + // type access `Ti` + extractTypeAccessRecursive(pType, locId, castId, 0, funLabels.methodId, enclosingStmtId) + + // element access: `a0[i]` + val arrayAccessId = tw.getFreshIdLabel() + tw.writeExprs_arrayaccess(arrayAccessId, anyNType.javaResult.id, castId, 1) + tw.writeExprsKotlinType(arrayAccessId, anyNType.kotlinResult.id) + extractExprContext(arrayAccessId, locId, funLabels.methodId, enclosingStmtId) + + // parameter access: `a0` + val argsAccessId = tw.getFreshIdLabel() + tw.writeExprs_varaccess(argsAccessId, argsType.javaResult.id, arrayAccessId, 0) + tw.writeExprsKotlinType(argsAccessId, argsType.kotlinResult.id) + extractExprContext(argsAccessId, locId, funLabels.methodId, enclosingStmtId) + tw.writeVariableBinding(argsAccessId, funLabels.parameters.first().first) + + // index access: `i` + extractConstantInteger( + pIdx, + locId, + arrayAccessId, + 1, + funLabels.methodId, + enclosingStmtId + ) + } + } + + private fun extractVarargElement( + e: IrVarargElement, + callable: Label, + parent: Label, + idx: Int, + enclosingStmt: Label + ) { + with("vararg element", e) { + val argExpr = + when (e) { + is IrExpression -> e + is IrSpreadElement -> e.expression + else -> { + logger.errorElement("Unrecognised IrVarargElement: " + e.javaClass, e) + null + } + } + argExpr?.let { extractExpressionExpr(it, callable, parent, idx, enclosingStmt) } + } + } + + /** + * Extracts a type access expression and its generic arguments for a constructor call. It only + * extracts type arguments relating to the constructed type, not the constructor itself, which + * makes a difference in case of nested generics. + */ + private fun extractConstructorTypeAccess( + irType: IrType, + type: TypeResults, + target: IrFunctionSymbol, + locId: Label, + parent: Label, + idx: Int, + enclosingCallable: Label, + enclosingStmt: Label + ) { + val typeAccessId = + extractTypeAccess(type, locId, parent, idx, enclosingCallable, enclosingStmt) + if (irType is IrSimpleType) { + extractTypeArguments( + irType.arguments + .take(target.owner.parentAsClass.typeParameters.size) + .filterIsInstance(), + locId, + typeAccessId, + enclosingCallable, + enclosingStmt + ) + } + } + + /** + * Extracts a single wildcard type access expression with no enclosing callable and statement. + */ + private fun extractWildcardTypeAccess( + type: TypeResultsWithoutSignatures, + location: Label, + parent: Label, + idx: Int + ): Label { + val id = tw.getFreshIdLabel() + tw.writeExprs_wildcardtypeaccess(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + tw.writeHasLocation(id, location) + return id + } + + /** Extracts a single type access expression with no enclosing callable and statement. */ + private fun extractTypeAccess( + type: TypeResults, + location: Label, + parent: Label, + idx: Int, + overrideId: Label? = null + ): Label { + // TODO: elementForLocation allows us to give some sort of + // location, but a proper location for the type access will + // require upstream changes + val id = exprIdOrFresh(overrideId) + tw.writeExprs_unannotatedtypeaccess(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + tw.writeHasLocation(id, location) + return id + } + + /** Extracts a single type access expression with enclosing callable and statement. */ + private fun extractTypeAccess( + type: TypeResults, + location: Label, + parent: Label, + idx: Int, + enclosingCallable: Label?, + enclosingStmt: Label?, + overrideId: Label? = null + ): Label { + val id = extractTypeAccess(type, location, parent, idx, overrideId = overrideId) + if (enclosingCallable != null) { + tw.writeCallableEnclosingExpr(id, enclosingCallable) + } + if (enclosingStmt != null) { + tw.writeStatementEnclosingExpr(id, enclosingStmt) + } + return id + } + + /** + * Extracts a type argument type access, introducing a wildcard type access if appropriate, or + * directly calling `extractTypeAccessRecursive` if the argument is invariant. No enclosing + * callable and statement is extracted, this is useful for type access extraction in field + * declarations. + */ + private fun extractWildcardTypeAccessRecursive( + t: IrTypeArgument, + location: Label, + parent: Label, + idx: Int + ) { + val typeLabels by lazy { + TypeResultsWithoutSignatures( + getTypeArgumentLabel(t), + TypeResultWithoutSignature(fakeKotlinType(), Unit, "TODO") + ) + } + when (t) { + is IrStarProjection -> extractWildcardTypeAccess(typeLabels, location, parent, idx) + is IrTypeProjection -> + when (t.variance) { + Variance.INVARIANT -> + extractTypeAccessRecursive( + t.type, + location, + parent, + idx, + TypeContext.GENERIC_ARGUMENT + ) + else -> { + val wildcardLabel = + extractWildcardTypeAccess(typeLabels, location, parent, idx) + // Mimic a Java extractor oddity, that it uses the child index to indicate + // what kind of wildcard this is + val boundChildIdx = if (t.variance == Variance.OUT_VARIANCE) 0 else 1 + extractTypeAccessRecursive( + t.type, + location, + wildcardLabel, + boundChildIdx, + TypeContext.GENERIC_ARGUMENT + ) + } + } + } + } + + /** + * Extracts a type access expression and its child type access expressions in case of a generic + * type. Nested generics are also handled. No enclosing callable and statement is extracted, + * this is useful for type access extraction in field declarations. + */ + private fun extractTypeAccessRecursive( + t: IrType, + location: Label, + parent: Label, + idx: Int, + typeContext: TypeContext = TypeContext.OTHER + ): Label { + val typeAccessId = extractTypeAccess(useType(t, typeContext), location, parent, idx) + if (t is IrSimpleType) { + // From 1.9, the list might change when we call erase, + // so we make a copy that it is safe to iterate over. + val argumentsCopy = t.arguments.toList() + argumentsCopy.forEachIndexed { argIdx, arg -> + extractWildcardTypeAccessRecursive(arg, location, typeAccessId, argIdx) + } + } + return typeAccessId + } + + /** + * Extracts a type access expression and its child type access expressions in case of a generic + * type. Nested generics are also handled. + */ + private fun extractTypeAccessRecursive( + t: IrType, + location: Label, + parent: Label, + idx: Int, + enclosingCallable: Label?, + enclosingStmt: Label?, + typeContext: TypeContext = TypeContext.OTHER, + overrideId: Label? = null + ): Label { + // TODO: `useType` substitutes types to their java equivalent, and sometimes that also means + // changing the number of type arguments. The below logic doesn't take this into account. + // For example `KFunction2` becomes `KFunction` with three child + // type access expressions: `Int`, `Double`, `String`. + val typeAccessId = + extractTypeAccess( + useType(t, typeContext), + location, + parent, + idx, + enclosingCallable, + enclosingStmt, + overrideId = overrideId + ) + if (t is IrSimpleType) { + if (t.arguments.isNotEmpty() && overrideId != null) { + logger.error( + "Unexpected parameterized type with an overridden expression ID; children will be assigned fresh IDs" + ) + } + extractTypeArguments( + t.arguments.filterIsInstance(), + location, + typeAccessId, + enclosingCallable, + enclosingStmt + ) + } + return typeAccessId + } + + /** + * Extracts a list of types as type access expressions. Nested generics are also handled. Used + * for extracting nested type access expressions, and type arguments of constructor or function + * calls. + */ + private fun extractTypeArguments( + typeArgs: List, + location: Label, + parentExpr: Label, + enclosingCallable: Label?, + enclosingStmt: Label?, + startIndex: Int = 0, + reverse: Boolean = false + ) { + typeArgs.forEachIndexed { argIdx, arg -> + val mul = if (reverse) -1 else 1 + extractTypeAccessRecursive( + arg, + location, + parentExpr, + argIdx * mul + startIndex, + enclosingCallable, + enclosingStmt, + TypeContext.GENERIC_ARGUMENT + ) + } + } + + /** + * Extracts type arguments of a member access expression as type access expressions. Nested + * generics are also handled. Used for extracting nested type access expressions, and type + * arguments of constructor or function calls. + */ + private fun extractTypeArguments( + c: IrMemberAccessExpression, + parentExpr: Label, + enclosingCallable: Label, + enclosingStmt: Label, + startIndex: Int = 0, + reverse: Boolean = false + ) { + val typeArguments = + (0 until c.typeArgumentsCount).map { c.getTypeArgument(it) }.requireNoNullsOrNull() + if (typeArguments == null) { + logger.errorElement("Found a null type argument for a member access expression", c) + } else { + extractTypeArguments( + typeArguments, + tw.getLocation(c), + parentExpr, + enclosingCallable, + enclosingStmt, + startIndex, + reverse + ) + } + } + + private fun extractArrayCreationWithInitializer( + parent: Label, + arraySize: Int, + locId: Label, + enclosingCallable: Label, + enclosingStmt: Label + ): Label { + + val arrayCreationId = tw.getFreshIdLabel() + val arrayType = + pluginContext.irBuiltIns.arrayClass.typeWith(pluginContext.irBuiltIns.anyNType) + val at = useType(arrayType) + tw.writeExprs_arraycreationexpr(arrayCreationId, at.javaResult.id, parent, 0) + tw.writeExprsKotlinType(arrayCreationId, at.kotlinResult.id) + extractExprContext(arrayCreationId, locId, enclosingCallable, enclosingStmt) + + extractTypeAccessRecursive( + pluginContext.irBuiltIns.anyNType, + locId, + arrayCreationId, + -1, + enclosingCallable, + enclosingStmt + ) + + val initId = tw.getFreshIdLabel() + tw.writeExprs_arrayinit(initId, at.javaResult.id, arrayCreationId, -2) + tw.writeExprsKotlinType(initId, at.kotlinResult.id) + extractExprContext(initId, locId, enclosingCallable, enclosingStmt) + + extractConstantInteger( + arraySize, + locId, + arrayCreationId, + 0, + enclosingCallable, + enclosingStmt + ) + + return initId + } + + private fun extractTypeOperatorCall( + e: IrTypeOperatorCall, + callable: Label, + parent: Label, + idx: Int, + enclosingStmt: Label + ) { + with("type operator call", e) { + when (e.operator) { + IrTypeOperator.CAST -> { + val id = tw.getFreshIdLabel() + val locId = tw.getLocation(e) + val type = useType(e.type) + tw.writeExprs_castexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, callable, enclosingStmt) + extractTypeAccessRecursive(e.typeOperand, locId, id, 0, callable, enclosingStmt) + extractExpressionExpr(e.argument, callable, id, 1, enclosingStmt) + } + IrTypeOperator.IMPLICIT_CAST -> { + val id = tw.getFreshIdLabel() + val locId = tw.getLocation(e) + val type = useType(e.type) + tw.writeExprs_implicitcastexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, callable, enclosingStmt) + extractTypeAccessRecursive(e.typeOperand, locId, id, 0, callable, enclosingStmt) + extractExpressionExpr(e.argument, callable, id, 1, enclosingStmt) + } + IrTypeOperator.IMPLICIT_NOTNULL -> { + val id = tw.getFreshIdLabel() + val locId = tw.getLocation(e) + val type = useType(e.type) + tw.writeExprs_implicitnotnullexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, callable, enclosingStmt) + extractTypeAccessRecursive(e.typeOperand, locId, id, 0, callable, enclosingStmt) + extractExpressionExpr(e.argument, callable, id, 1, enclosingStmt) + } + IrTypeOperator.IMPLICIT_COERCION_TO_UNIT -> { + val id = tw.getFreshIdLabel() + val locId = tw.getLocation(e) + val type = useType(e.type) + tw.writeExprs_implicitcoerciontounitexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, callable, enclosingStmt) + extractTypeAccessRecursive(e.typeOperand, locId, id, 0, callable, enclosingStmt) + extractExpressionExpr(e.argument, callable, id, 1, enclosingStmt) + } + IrTypeOperator.SAFE_CAST -> { + val id = tw.getFreshIdLabel() + val locId = tw.getLocation(e) + val type = useType(e.type) + tw.writeExprs_safecastexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, callable, enclosingStmt) + extractTypeAccessRecursive(e.typeOperand, locId, id, 0, callable, enclosingStmt) + extractExpressionExpr(e.argument, callable, id, 1, enclosingStmt) + } + IrTypeOperator.INSTANCEOF -> { + val id = tw.getFreshIdLabel() + val locId = tw.getLocation(e) + val type = useType(e.type) + tw.writeExprs_instanceofexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, callable, enclosingStmt) + extractExpressionExpr(e.argument, callable, id, 0, enclosingStmt) + extractTypeAccessRecursive(e.typeOperand, locId, id, 1, callable, enclosingStmt) + } + IrTypeOperator.NOT_INSTANCEOF -> { + val id = tw.getFreshIdLabel() + val locId = tw.getLocation(e) + val type = useType(e.type) + tw.writeExprs_notinstanceofexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, callable, enclosingStmt) + extractExpressionExpr(e.argument, callable, id, 0, enclosingStmt) + extractTypeAccessRecursive(e.typeOperand, locId, id, 1, callable, enclosingStmt) + } + IrTypeOperator.SAM_CONVERSION -> { + + /* + The following Kotlin code + + ``` + fun interface IntPredicate { + fun accept(i: Int): Boolean + } + + val x = IntPredicate { it % 2 == 0 } + ``` + + is extracted as + + ``` + interface IntPredicate { + Boolean accept(Integer i); + } + class extends Object implements IntPredicate { + Function1 ; + public (Function1 ) { this. = ; } + public override Boolean accept(Integer i) { return .invoke(i); } + } + + IntPredicate x = (IntPredicate)new (...); + ``` + */ + + val st = e.argument.type as? IrSimpleType + if (st == null) { + logger.errorElement("Expected to find a simple type in SAM conversion.", e) + return + } + + fun IrSimpleType.isKProperty() = + classFqName?.asString()?.startsWith("kotlin.reflect.KProperty") == true + + if ( + !st.isFunctionOrKFunction() && + !st.isSuspendFunctionOrKFunction() && + !st.isKProperty() + ) { + logger.errorElement( + "Expected to find expression with function type in SAM conversion.", + e + ) + return + } + + // Either Function1, ... Function22 or FunctionN type, but not Function23 or + // above. + val functionType = getFunctionalInterfaceTypeWithTypeArgs(st.arguments) + if (functionType == null) { + logger.errorElement("Cannot find functional interface.", e) + return + } + + val invokeMethod = + functionType.classOrNull?.owner?.declarations?.findSubType { + it.name.asString() == OperatorNameConventions.INVOKE.asString() + } + if (invokeMethod == null) { + logger.errorElement( + "Couldn't find `invoke` method on functional interface.", + e + ) + return + } + + val typeOwner = e.typeOperand.classifierOrFail.owner + if (typeOwner !is IrClass) { + logger.errorElement( + "Expected to find SAM conversion to IrClass. Found '${typeOwner.javaClass}' instead. Can't implement SAM interface.", + e + ) + return + } + val samMember = + typeOwner.declarations.findSubType { + it is IrOverridableMember && it.modality == Modality.ABSTRACT + } + if (samMember == null) { + logger.errorElement( + "Couldn't find SAM member in type '${typeOwner.kotlinFqName.asString()}'. Can't implement SAM interface.", + e + ) + return + } + + val javaResult = TypeResult(tw.getFreshIdLabel(), "", "") + val kotlinResult = TypeResult(tw.getFreshIdLabel(), "", "") + tw.writeKt_notnull_types(kotlinResult.id, javaResult.id) + val ids = + LocallyVisibleFunctionLabels( + TypeResults(javaResult, kotlinResult), + constructor = tw.getFreshIdLabel(), + constructorBlock = tw.getFreshIdLabel(), + function = tw.getFreshIdLabel() + ) + + val locId = tw.getLocation(e) + val helper = GeneratedClassHelper(locId, ids) + + val declarationParent = peekDeclStackAsDeclarationParent(e) ?: return + val classId = + extractGeneratedClass( + ids, + listOf(pluginContext.irBuiltIns.anyType, e.typeOperand), + locId, + e, + declarationParent + ) + + // add field + val fieldId = tw.getFreshIdLabel() + extractField( + fieldId, + "", + functionType, + classId, + locId, + DescriptorVisibilities.PRIVATE, + e, + isExternalDeclaration = false, + isFinal = true, + isStatic = false + ) + + // adjust constructor + helper.extractParameterToFieldAssignmentInConstructor( + "", + functionType, + fieldId, + 0, + 1 + ) + + // add implementation function + val classTypeArgs = (e.type as? IrSimpleType)?.arguments + val typeSub = + classTypeArgs?.let { makeGenericSubstitutionFunction(typeOwner, it) } + + fun trySub(t: IrType, context: TypeContext) = + if (typeSub == null) t else typeSub(t, context, pluginContext) + + // Force extraction of this function even if this is a fake override -- + // This happens in the case where a functional interface inherits its only + // abstract member, + // which usually we wouldn't extract, but in this case we're effectively using + // it as a template + // for the real function we're extracting that will implement this interface, + // and it serves fine + // for that purpose. By contrast if we looked through the fake to the underlying + // abstract method + // we would need to compose generic type substitutions -- for example, if we're + // implementing + // T UnaryOperator.apply(T t) here, we would need to compose substitutions so + // we can implement + // the real underlying R Function.apply(T t). + forceExtractFunction( + samMember, + classId, + extractBody = false, + extractMethodAndParameterTypeAccesses = true, + extractAnnotations = false, + typeSub, + classTypeArgs, + overriddenAttributes = + OverriddenFunctionAttributes( + id = ids.function, + sourceLoc = tw.getLocation(e), + modality = Modality.FINAL + ) + ) + + addModifiers(ids.function, "override") + if (st.isSuspendFunctionOrKFunction()) { + addModifiers(ids.function, "suspend") + } + + // body + val blockId = extractBlockBody(ids.function, locId) + + // return stmt + val returnId = tw.getFreshIdLabel() + tw.writeStmts_returnstmt(returnId, blockId, 0, ids.function) + tw.writeHasLocation(returnId, locId) + + // .invoke(vp0, cp1, vp2, vp3, ...) or + // .invoke(new Object[x]{vp0, vp1, vp2, ...}) + + // Call to original `invoke`: + val callId = tw.getFreshIdLabel() + val callType = useType(trySub(samMember.returnType, TypeContext.RETURN)) + tw.writeExprs_methodaccess(callId, callType.javaResult.id, returnId, 0) + tw.writeExprsKotlinType(callId, callType.kotlinResult.id) + extractExprContext(callId, locId, ids.function, returnId) + val calledMethodId = useFunction(invokeMethod, functionType.arguments) + if (calledMethodId == null) { + logger.errorElement("Cannot get ID for called method", invokeMethod) + } else { + tw.writeCallableBinding(callId, calledMethodId) + } + + // access + val lhsId = tw.getFreshIdLabel() + val lhsType = useType(functionType) + tw.writeExprs_varaccess(lhsId, lhsType.javaResult.id, callId, -1) + tw.writeExprsKotlinType(lhsId, lhsType.kotlinResult.id) + extractExprContext(lhsId, locId, ids.function, returnId) + tw.writeVariableBinding(lhsId, fieldId) + + val parameters = mutableListOf() + val extParam = samMember.extensionReceiverParameter + if (extParam != null) { + parameters.add(extParam) + } + parameters.addAll(samMember.valueParameters) + + fun extractArgument( + p: IrValueParameter, + idx: Int, + parent: Label + ) { + val argsAccessId = tw.getFreshIdLabel() + val paramType = useType(trySub(p.type, TypeContext.OTHER)) + tw.writeExprs_varaccess(argsAccessId, paramType.javaResult.id, parent, idx) + tw.writeExprsKotlinType(argsAccessId, paramType.kotlinResult.id) + extractExprContext(argsAccessId, locId, ids.function, returnId) + tw.writeVariableBinding(argsAccessId, useValueParameter(p, ids.function)) + } + + val isBigArity = st.arguments.size > BuiltInFunctionArity.BIG_ARITY + val argParent = + if (isBigArity) { + // .invoke(new Object[x]{vp0, vp1, vp2, ...}) + extractArrayCreationWithInitializer( + callId, + parameters.size, + locId, + ids.function, + returnId + ) + } else { + // .invoke(vp0, cp1, vp2, vp3, ...) or + callId + } + + for ((parameterIdx, vp) in parameters.withIndex()) { + extractArgument(vp, parameterIdx, argParent) + } + + val id = tw.getFreshIdLabel() + val type = useType(e.typeOperand) + tw.writeExprs_castexpr(id, type.javaResult.id, parent, idx) + tw.writeExprsKotlinType(id, type.kotlinResult.id) + extractExprContext(id, locId, callable, enclosingStmt) + extractTypeAccessRecursive(e.typeOperand, locId, id, 0, callable, enclosingStmt) + + val idNewexpr = + extractNewExpr( + ids.constructor, + ids.type, + locId, + id, + 1, + callable, + enclosingStmt + ) + + tw.writeIsAnonymClass( + ids.type.javaResult.id.cast(), + idNewexpr + ) + + extractTypeAccessRecursive( + e.typeOperand, + locId, + idNewexpr, + -3, + callable, + enclosingStmt + ) + + extractExpressionExpr(e.argument, callable, idNewexpr, 0, enclosingStmt) + } + else -> { + logger.errorElement( + "Unrecognised IrTypeOperatorCall for ${e.operator}: " + e.render(), + e + ) + } + } + } + } + + private fun extractBreakContinue(e: IrBreakContinue, id: Label) { + with("break/continue", e) { + val locId = tw.getLocation(e) + tw.writeHasLocation(id, locId) + val label = e.label + if (label != null) { + tw.writeNamestrings(label, "", id) + } + } + } + + private val IrType.isAnonymous: Boolean + get() = ((this as? IrSimpleType)?.classifier?.owner as? IrClass)?.isAnonymousObject ?: false + + private fun addVisibilityModifierToLocalOrAnonymousClass(id: Label) { + addModifiers(id, "private") + } + + /** Extracts the class around a local function, a lambda, or a function reference. */ + private fun extractGeneratedClass( + ids: GeneratedClassLabels, + superTypes: List, + locId: Label, + elementToReportOn: IrElement, + declarationParent: IrDeclarationParent, + compilerGeneratedKindOverride: CompilerGeneratedKinds? = null, + superConstructorSelector: (IrFunction) -> Boolean = { it.valueParameters.isEmpty() }, + extractSuperconstructorArgs: (Label) -> Unit = {}, + ): Label { + // Write class + val id = ids.type.javaResult.id.cast() + val pkgId = extractPackage("") + tw.writeClasses_or_interfaces(id, "", pkgId, id) + tw.writeCompiler_generated( + id, + (compilerGeneratedKindOverride ?: CompilerGeneratedKinds.CALLABLE_CLASS).kind + ) + tw.writeHasLocation(id, locId) + + // Extract constructor + val unitType = useType(pluginContext.irBuiltIns.unitType) + tw.writeConstrs(ids.constructor, "", "", unitType.javaResult.id, id, ids.constructor) + tw.writeConstrsKotlinType(ids.constructor, unitType.kotlinResult.id) + tw.writeHasLocation(ids.constructor, locId) + addModifiers(ids.constructor, "public") + + // Constructor body + val constructorBlockId = ids.constructorBlock + tw.writeStmts_block(constructorBlockId, ids.constructor, 0, ids.constructor) + tw.writeHasLocation(constructorBlockId, locId) + + // Super call + val baseClass = superTypes.first().classOrNull + if (baseClass == null) { + logger.warnElement("Cannot find base class", elementToReportOn) + } else { + val baseConstructor = + baseClass.owner.declarations.findSubType { + it.symbol is IrConstructorSymbol && superConstructorSelector(it) + } + if (baseConstructor == null) { + logger.warnElement("Cannot find base constructor", elementToReportOn) + } else { + val baseConstructorId = useFunction(baseConstructor) + if (baseConstructorId == null) { + logger.errorElement("Cannot find base constructor ID", elementToReportOn) + } else { + val superCallId = tw.getFreshIdLabel() + tw.writeStmts_superconstructorinvocationstmt( + superCallId, + constructorBlockId, + 0, + ids.constructor + ) + + tw.writeHasLocation(superCallId, locId) + tw.writeCallableBinding(superCallId.cast(), baseConstructorId) + extractSuperconstructorArgs(superCallId) + } + } + } + + addModifiers(id, "final") + addVisibilityModifierToLocalOrAnonymousClass(id) + extractClassSupertypes( + superTypes, + listOf(), + id, + isInterface = false, + inReceiverContext = true + ) + + extractEnclosingClass(declarationParent, id, null, locId, listOf()) + + return id + } + + /** + * Extracts the class around a local function or a lambda. The superclass must have a no-arg + * constructor. + */ + private fun extractGeneratedClass( + localFunction: IrFunction, + superTypes: List, + compilerGeneratedKindOverride: CompilerGeneratedKinds? = null + ): Label { + with("generated class", localFunction) { + val ids = getLocallyVisibleFunctionLabels(localFunction) + + val id = + extractGeneratedClass( + ids, + superTypes, + tw.getLocation(localFunction), + localFunction, + localFunction.parent, + compilerGeneratedKindOverride = compilerGeneratedKindOverride + ) + + // Extract local function as a member + extractFunction( + localFunction, + id, + extractBody = true, + extractMethodAndParameterTypeAccesses = true, + extractAnnotations = false, + null, + listOf() + ) + + return id + } + } + + private inner class DeclarationStackAdjuster( + val declaration: IrDeclaration, + val overriddenAttributes: OverriddenFunctionAttributes? = null + ) : Closeable { + init { + declarationStack.push(declaration, overriddenAttributes) + } + + override fun close() { + declarationStack.pop() + } + } + + class DeclarationStack { + private val stack: Stack> = Stack() + + fun push(item: IrDeclaration, overriddenAttributes: OverriddenFunctionAttributes?) = + stack.push(Pair(item, overriddenAttributes)) + + fun pop() = stack.pop() + + fun isEmpty() = stack.isEmpty() + + fun peek() = stack.peek() + + fun tryPeek() = if (stack.isEmpty()) null else stack.peek() + + fun findOverriddenAttributes(f: IrFunction) = stack.lastOrNull { it.first == f }?.second + } + + data class OverriddenFunctionAttributes( + val id: Label? = null, + val sourceDeclarationId: Label? = null, + val sourceLoc: Label? = null, + val valueParameters: List? = null, + val typeParameters: List? = null, + val isStatic: Boolean? = null, + val visibility: DescriptorVisibility? = null, + val modality: Modality? = null, + ) + + private fun peekDeclStackAsDeclarationParent( + elementToReportOn: IrElement + ): IrDeclarationParent? { + val trapWriter = tw + if (declarationStack.isEmpty() && trapWriter is SourceFileTrapWriter) { + // If the current declaration is used as a parent, we might end up with an empty stack. + // In this case, the source file is the parent. + return trapWriter.irFile + } + + val dp = declarationStack.peek().first as? IrDeclarationParent + if (dp == null) + logger.errorElement("Couldn't find current declaration parent", elementToReportOn) + return dp + } + + private enum class CompilerGeneratedKinds(val kind: Int) { + DECLARING_CLASSES_OF_ADAPTER_FUNCTIONS(1), + GENERATED_DATA_CLASS_MEMBER(2), + DEFAULT_PROPERTY_ACCESSOR(3), + CLASS_INITIALISATION_METHOD(4), + ENUM_CLASS_SPECIAL_MEMBER(5), + DELEGATED_PROPERTY_GETTER(6), + DELEGATED_PROPERTY_SETTER(7), + JVMSTATIC_PROXY_METHOD(8), + JVMOVERLOADS_METHOD(9), + DEFAULT_ARGUMENTS_METHOD(10), + INTERFACE_FORWARDER(11), + ENUM_CONSTRUCTOR_ARGUMENT(12), + CALLABLE_CLASS(13), + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/KotlinUsesExtractor.kt b/java/kotlin-extractor2/src/main/kotlin/KotlinUsesExtractor.kt new file mode 100644 index 00000000000..3be94d65690 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/KotlinUsesExtractor.kt @@ -0,0 +1,2251 @@ +package com.github.codeql + +import com.github.codeql.utils.* +import com.github.codeql.utils.versions.* +import com.semmle.extractor.java.OdasaOutput +import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext +import org.jetbrains.kotlin.backend.common.ir.* +import org.jetbrains.kotlin.backend.jvm.ir.propertyIfAccessor +import org.jetbrains.kotlin.codegen.JvmCodegenUtil +import org.jetbrains.kotlin.descriptors.* +import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.ir.expressions.* +import org.jetbrains.kotlin.ir.symbols.* +import org.jetbrains.kotlin.ir.types.* +import org.jetbrains.kotlin.ir.types.impl.* +import org.jetbrains.kotlin.ir.util.* +import org.jetbrains.kotlin.load.java.BuiltinMethodsWithSpecialGenericSignature +import org.jetbrains.kotlin.load.java.JvmAbi +import org.jetbrains.kotlin.load.java.sources.JavaSourceElement +import org.jetbrains.kotlin.load.java.structure.* +import org.jetbrains.kotlin.load.java.typeEnhancement.hasEnhancedNullability +import org.jetbrains.kotlin.load.kotlin.getJvmModuleNameForDeserializedDescriptor +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.name.NameUtils +import org.jetbrains.kotlin.name.SpecialNames +import org.jetbrains.kotlin.types.Variance +import org.jetbrains.kotlin.util.OperatorNameConventions + +open class KotlinUsesExtractor( + open val logger: Logger, + open val tw: TrapWriter, + val dependencyCollector: OdasaOutput.TrapFileManager?, + val externalClassExtractor: ExternalDeclExtractor, + val primitiveTypeMapping: PrimitiveTypeMapping, + val pluginContext: IrPluginContext, + val globalExtensionState: KotlinExtractorGlobalState +) { + fun referenceExternalClass(name: String) = + getClassByFqName(pluginContext, FqName(name))?.owner.also { + if (it == null) logger.warn("Unable to resolve external class $name") + else extractExternalClassLater(it) + } + + val javaLangObject by lazy { referenceExternalClass("java.lang.Object") } + + val javaLangObjectType by lazy { javaLangObject?.typeWith() } + + private fun usePackage(pkg: String): Label { + return extractPackage(pkg) + } + + fun extractPackage(pkg: String): Label { + val pkgLabel = "@\"package;$pkg\"" + val id: Label = tw.getLabelFor(pkgLabel, { tw.writePackages(it, pkg) }) + return id + } + + fun useFileClassType(f: IrFile) = + TypeResults(TypeResult(extractFileClass(f), "", ""), TypeResult(fakeKotlinType(), "", "")) + + fun useFileClassType(fqName: FqName) = + TypeResults( + TypeResult(extractFileClass(fqName), "", ""), + TypeResult(fakeKotlinType(), "", "") + ) + + private fun useFileClassType(pkg: String, jvmName: String) = + TypeResults( + TypeResult(extractFileClass(pkg, jvmName), "", ""), + TypeResult(fakeKotlinType(), "", "") + ) + + fun extractFileClass(f: IrFile): Label { + val pkg = f.packageFqName.asString() + val jvmName = getFileClassName(f) + val id = extractFileClass(pkg, jvmName) + if (tw.lm.fileClassLocationsExtracted.add(f)) { + val fileId = tw.mkFileId(f.path, false) + val locId = tw.getWholeFileLocation(fileId) + tw.writeHasLocation(id, locId) + } + return id + } + + private fun extractFileClass(fqName: FqName): Label { + val pkg = if (fqName.isRoot()) "" else fqName.parent().asString() + val jvmName = fqName.shortName().asString() + return extractFileClass(pkg, jvmName) + } + + private fun extractFileClass(pkg: String, jvmName: String): Label { + val qualClassName = if (pkg.isEmpty()) jvmName else "$pkg.$jvmName" + val label = "@\"class;$qualClassName\"" + val id: Label = + tw.getLabelFor(label) { + val pkgId = extractPackage(pkg) + tw.writeClasses_or_interfaces(it, jvmName, pkgId, it) + tw.writeFile_class(it) + + addModifiers(it, "public", "final") + } + return id + } + + data class UseClassInstanceResult( + val typeResult: TypeResult, + val javaClass: IrClass + ) + + fun useType(t: IrType, context: TypeContext = TypeContext.OTHER): TypeResults { + when (t) { + is IrSimpleType -> return useSimpleType(t, context) + else -> { + logger.error("Unrecognised IrType: " + t.javaClass) + return extractErrorType() + } + } + } + + private fun extractJavaErrorType(): TypeResult { + val typeId = tw.getLabelFor("@\"errorType\"") { tw.writeError_type(it) } + return TypeResult(typeId, "", "") + } + + private fun extractErrorType(): TypeResults { + val javaResult = extractJavaErrorType() + val kotlinTypeId = + tw.getLabelFor("@\"errorKotlinType\"") { + tw.writeKt_nullable_types(it, javaResult.id) + } + return TypeResults( + javaResult, + TypeResult(kotlinTypeId, "", "") + ) + } + + fun getJavaEquivalentClass(c: IrClass) = + getJavaEquivalentClassId(c)?.let { getClassByClassId(pluginContext, it) }?.owner + + /** + * Gets a KotlinFileExtractor based on this one, except it attributes locations to the file that + * declares the given class. + */ + private fun withFileOfClass(cls: IrClass): KotlinFileExtractor { + val clsFile = cls.fileOrNull + + if (this is KotlinFileExtractor && this.filePath == clsFile?.path) { + return this + } + + val newDeclarationStack = + if (this is KotlinFileExtractor) this.declarationStack + else KotlinFileExtractor.DeclarationStack() + + if (clsFile == null || isExternalDeclaration(cls)) { + val filePath = getIrClassBinaryPath(cls) + val newTrapWriter = tw.makeFileTrapWriter(filePath, true) + val newLoggerTrapWriter = logger.dtw.makeFileTrapWriter(filePath, false) + val newLogger = FileLogger(logger.loggerBase, newLoggerTrapWriter) + return KotlinFileExtractor( + newLogger, + newTrapWriter, + null, + filePath, + dependencyCollector, + externalClassExtractor, + primitiveTypeMapping, + pluginContext, + newDeclarationStack, + globalExtensionState + ) + } + + val newTrapWriter = tw.makeSourceFileTrapWriter(clsFile, true) + val newLoggerTrapWriter = logger.dtw.makeSourceFileTrapWriter(clsFile, false) + val newLogger = FileLogger(logger.loggerBase, newLoggerTrapWriter) + return KotlinFileExtractor( + newLogger, + newTrapWriter, + null, + clsFile.path, + dependencyCollector, + externalClassExtractor, + primitiveTypeMapping, + pluginContext, + newDeclarationStack, + globalExtensionState + ) + } + + // The Kotlin compiler internal representation of Outer.Inner.InnerInner is + // InnerInner. This function returns just `R`. + fun removeOuterClassTypeArgs( + c: IrClass, + argsIncludingOuterClasses: List? + ): List? { + return argsIncludingOuterClasses?.let { + if (it.size > c.typeParameters.size) it.take(c.typeParameters.size) else null + } ?: argsIncludingOuterClasses + } + + private fun isStaticClass(c: IrClass) = + c.visibility != DescriptorVisibilities.LOCAL && !c.isInner + + // Gets nested inner classes starting at `c` and proceeding outwards to the innermost enclosing + // static class. + // For example, for (java syntax) `class A { static class B { class C { class D { } } } }`, + // `nonStaticParentsWithSelf(D)` = `[D, C, B]`. + private fun parentsWithTypeParametersInScope(c: IrClass): List { + val parentsList = c.parentsWithSelf.toList() + val firstOuterClassIdx = parentsList.indexOfFirst { it is IrClass && isStaticClass(it) } + return if (firstOuterClassIdx == -1) parentsList + else parentsList.subList(0, firstOuterClassIdx + 1) + } + + // Gets the type parameter symbols that are in scope for class `c` in Kotlin order (i.e. for + // `class NotInScope { static class OutermostInScope { class QueryClass { } } }`, + // `getTypeParametersInScope(QueryClass)` = `[C, D, A, B]`. + private fun getTypeParametersInScope(c: IrClass) = + parentsWithTypeParametersInScope(c).mapNotNull({ getTypeParameters(it) }).flatten() + + // Returns a map from `c`'s type variables in scope to type arguments + // `argsIncludingOuterClasses`. + // Hack for the time being: the substituted types are always nullable, to prevent downstream + // code + // from replacing a generic parameter by a primitive. As and when we extract Kotlin types we + // will + // need to track this information in more detail. + private fun makeTypeGenericSubstitutionMap( + c: IrClass, + argsIncludingOuterClasses: List + ) = + getTypeParametersInScope(c) + .map({ it.symbol }) + .zip(argsIncludingOuterClasses.map { it.withQuestionMark(true) }) + .toMap() + + fun makeGenericSubstitutionFunction( + c: IrClass, + argsIncludingOuterClasses: List + ) = + makeTypeGenericSubstitutionMap(c, argsIncludingOuterClasses).let { + { x: IrType, useContext: TypeContext, pluginContext: IrPluginContext -> + x.substituteTypeAndArguments(it, useContext, pluginContext) + } + } + + // The Kotlin compiler internal representation of Outer.Inner.InnerInner.someFunction.LocalClass is LocalClass. This + // function returns [A, B, C, D, E, F, G, H, I, J]. + private fun orderTypeArgsLeftToRight( + c: IrClass, + argsIncludingOuterClasses: List? + ): List? { + if (argsIncludingOuterClasses.isNullOrEmpty()) return argsIncludingOuterClasses + val ret = ArrayList() + // Iterate over nested inner classes starting at `c`'s surrounding top-level or static + // nested class and ending at `c`, from the outermost inwards: + val truncatedParents = parentsWithTypeParametersInScope(c) + for (parent in truncatedParents.reversed()) { + val parentTypeParameters = getTypeParameters(parent) + val firstArgIdx = + argsIncludingOuterClasses.size - (ret.size + parentTypeParameters.size) + ret.addAll( + argsIncludingOuterClasses.subList( + firstArgIdx, + firstArgIdx + parentTypeParameters.size + ) + ) + } + return ret + } + + // `typeArgs` can be null to describe a raw generic type. + // For non-generic types it will be zero-length list. + fun useClassInstance( + c: IrClass, + typeArgs: List?, + inReceiverContext: Boolean = false + ): UseClassInstanceResult { + val substituteClass = getJavaEquivalentClass(c) + + val extractClass = substituteClass ?: c + + // `KFunction1` is substituted by `KFunction`. The last type argument is the + // return type. + // Similarly Function23 and above get replaced by kotlin.jvm.functions.FunctionN with only + // one type arg, the result type. + // References to SomeGeneric where SomeGeneric is declared SomeGeneric are extracted + // as if they were references to the unbound type SomeGeneric. + val extractedTypeArgs = + when { + extractClass.symbol.isKFunction() && typeArgs != null && typeArgs.isNotEmpty() -> + listOf(typeArgs.last()) + extractClass.fqNameWhenAvailable == FqName("kotlin.jvm.functions.FunctionN") && + typeArgs != null && + typeArgs.isNotEmpty() -> listOf(typeArgs.last()) + typeArgs != null && isUnspecialised(c, typeArgs, logger) -> listOf() + else -> typeArgs + } + + val classTypeResult = addClassLabel(extractClass, extractedTypeArgs, inReceiverContext) + + // Extract both the Kotlin and equivalent Java classes, so that we have database entries + // for both even if all internal references to the Kotlin type are substituted. + if (c != extractClass) { + extractClassLaterIfExternal(c) + } + + return UseClassInstanceResult(classTypeResult, extractClass) + } + + private fun extractClassLaterIfExternal(c: IrClass) { + if (isExternalDeclaration(c)) { + extractExternalClassLater(c) + } + } + + private fun extractExternalEnclosingClassLater(d: IrDeclaration) { + when (val parent = d.parent) { + is IrClass -> extractExternalClassLater(parent) + is IrFunction -> extractExternalEnclosingClassLater(parent) + is IrFile -> logger.error("extractExternalEnclosingClassLater but no enclosing class.") + is IrExternalPackageFragment -> { + // The parent is a (multi)file class. We don't need + // extract it separately. + } + else -> + logger.error( + "Unrecognised extractExternalEnclosingClassLater ${parent.javaClass} for ${d.javaClass}" + ) + } + } + + private fun propertySignature(p: IrProperty) = + ((p.getter ?: p.setter)?.extensionReceiverParameter?.let { + useType(erase(it.type)).javaResult.signature + } ?: "") + + fun getTrapFileSignature(d: IrDeclaration) = + when (d) { + is IrFunction -> + // Note we erase the parameter types before calling useType even though the + // signature should be the same + // in order to prevent an infinite loop through useTypeParameter -> + // useDeclarationParent -> useFunction + // -> extractFunctionLaterIfExternalFileMember, which would result for `fun f(t: + // T) { ... }` for example. + (listOfNotNull(d.extensionReceiverParameter) + d.valueParameters) + .map { useType(erase(it.type)).javaResult.signature } + .joinToString(separator = ",", prefix = "(", postfix = ")") + is IrProperty -> propertySignature(d) + externalClassExtractor.propertySignature + is IrField -> + (d.correspondingPropertySymbol?.let { propertySignature(it.owner) } ?: "") + + externalClassExtractor.fieldSignature + else -> + "unknown signature" + .also { logger.warn("Trap file signature requested for unexpected element $d") } + } + + private fun extractParentExternalClassLater(d: IrDeclaration) { + val p = d.parent + when (p) { + is IrClass -> extractExternalClassLater(p) + is IrExternalPackageFragment -> { + // The parent is a (multi)file class. We don't need to + // extract it separately. + } + else -> { + logger.warn("Unexpected parent type ${p.javaClass} for external file class member") + } + } + } + + private fun extractPropertyLaterIfExternalFileMember(p: IrProperty) { + if (isExternalFileClassMember(p)) { + extractParentExternalClassLater(p) + val signature = getTrapFileSignature(p) + dependencyCollector?.addDependency(p, signature) + externalClassExtractor.extractLater(p, signature) + } + } + + private fun extractFieldLaterIfExternalFileMember(f: IrField) { + if (isExternalFileClassMember(f)) { + extractParentExternalClassLater(f) + val signature = getTrapFileSignature(f) + dependencyCollector?.addDependency(f, signature) + externalClassExtractor.extractLater(f, signature) + } + } + + private fun extractFunctionLaterIfExternalFileMember(f: IrFunction) { + if (isExternalFileClassMember(f)) { + extractParentExternalClassLater(f) + (f as? IrSimpleFunction)?.correspondingPropertySymbol?.let { + extractPropertyLaterIfExternalFileMember(it.owner) + // No need to extract the function specifically, as the property's + // getters and setters are extracted alongside it + return + } + val signature = getTrapFileSignature(f) + dependencyCollector?.addDependency(f, signature) + externalClassExtractor.extractLater(f, signature) + } + } + + fun extractExternalClassLater(c: IrClass) { + dependencyCollector?.addDependency(c) + externalClassExtractor.extractLater(c) + } + + private fun tryReplaceAndroidSyntheticClass(c: IrClass): IrClass { + // The Android Kotlin Extensions Gradle plugin introduces synthetic functions, fields and + // classes. The most + // obvious signature is that they lack any supertype information even though they are not + // root classes. + // If possible, replace them by a real version of the same class. + if ( + c.superTypes.isNotEmpty() || + c.origin != IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB || + c.hasEqualFqName(FqName("java.lang.Object")) + ) + return c + return globalExtensionState.syntheticToRealClassMap.getOrPut(c) { + val qualifiedName = c.fqNameWhenAvailable + if (qualifiedName == null) { + logger.warn( + "Failed to replace synthetic class ${c.name} because it has no fully qualified name" + ) + return@getOrPut null + } + + val result = getClassByFqName(pluginContext, qualifiedName)?.owner + if (result != null) { + logger.info("Replaced synthetic class ${c.name} with its real equivalent") + return@getOrPut result + } + + // The above doesn't work for (some) generated nested classes, such as R$id, which + // should be R.id + val fqn = qualifiedName.asString() + if (fqn.indexOf('$') >= 0) { + val nested = getClassByFqName(pluginContext, fqn.replace('$', '.'))?.owner + if (nested != null) { + logger.info( + "Replaced synthetic nested class ${c.name} with its real equivalent" + ) + return@getOrPut nested + } + } + + logger.warn("Failed to replace synthetic class ${c.name}") + return@getOrPut null + } ?: c + } + + private fun tryReplaceFunctionInSyntheticClass( + f: IrFunction, + getClassReplacement: (IrClass) -> IrClass + ): IrFunction { + val parentClass = f.parent as? IrClass ?: return f + val replacementClass = getClassReplacement(parentClass) + if (replacementClass === parentClass) return f + return globalExtensionState.syntheticToRealFunctionMap.getOrPut(f) { + val result = + replacementClass.declarations.findSubType { replacementDecl -> + replacementDecl.name == f.name && + replacementDecl.valueParameters.size == f.valueParameters.size && + replacementDecl.valueParameters.zip(f.valueParameters).all { + erase(it.first.type) == erase(it.second.type) + } + } + if (result == null) { + logger.warn("Failed to replace synthetic class function ${f.name}") + } else { + logger.info("Replaced synthetic class function ${f.name} with its real equivalent") + } + result + } ?: f + } + + fun tryReplaceSyntheticFunction(f: IrFunction): IrFunction { + val androidReplacement = + tryReplaceFunctionInSyntheticClass(f) { tryReplaceAndroidSyntheticClass(it) } + return tryReplaceFunctionInSyntheticClass(androidReplacement) { + tryReplaceParcelizeRawType(it)?.first ?: it + } + } + + fun tryReplaceAndroidSyntheticField(f: IrField): IrField { + val parentClass = f.parent as? IrClass ?: return f + val replacementClass = tryReplaceAndroidSyntheticClass(parentClass) + if (replacementClass === parentClass) return f + return globalExtensionState.syntheticToRealFieldMap.getOrPut(f) { + val result = + replacementClass.declarations.findSubType { replacementDecl -> + replacementDecl.name == f.name + } + ?: replacementClass.declarations + .findSubType { it.backingField?.name == f.name } + ?.backingField + if (result == null) { + logger.warn("Failed to replace synthetic class field ${f.name}") + } else { + logger.info("Replaced synthetic class field ${f.name} with its real equivalent") + } + result + } ?: f + } + + private fun tryReplaceType( + cBeforeReplacement: IrClass, + argsIncludingOuterClassesBeforeReplacement: List? + ): Pair?> { + val c = tryReplaceAndroidSyntheticClass(cBeforeReplacement) + val p = tryReplaceParcelizeRawType(c) + return Pair(p?.first ?: c, p?.second ?: argsIncludingOuterClassesBeforeReplacement) + } + + // `typeArgs` can be null to describe a raw generic type. + // For non-generic types it will be zero-length list. + private fun addClassLabel( + cBeforeReplacement: IrClass, + argsIncludingOuterClassesBeforeReplacement: List?, + inReceiverContext: Boolean = false + ): TypeResult { + val replaced = + tryReplaceType(cBeforeReplacement, argsIncludingOuterClassesBeforeReplacement) + val replacedClass = replaced.first + val replacedArgsIncludingOuterClasses = replaced.second + + val classLabelResult = getClassLabel(replacedClass, replacedArgsIncludingOuterClasses) + + var instanceSeenBefore = true + + val classLabel: Label = + tw.getLabelFor(classLabelResult.classLabel) { + instanceSeenBefore = false + + extractClassLaterIfExternal(replacedClass) + } + + if ( + replacedArgsIncludingOuterClasses == null || + replacedArgsIncludingOuterClasses.isNotEmpty() + ) { + // If this is a generic type instantiation or a raw type then it has no + // source entity, so we need to extract it here + val shouldExtractClassDetails = + inReceiverContext && + tw.lm.genericSpecialisationsExtracted.add(classLabelResult.classLabel) + if (!instanceSeenBefore || shouldExtractClassDetails) { + this.withFileOfClass(replacedClass) + .extractClassInstance( + classLabel, + replacedClass, + replacedArgsIncludingOuterClasses, + !instanceSeenBefore, + shouldExtractClassDetails + ) + } + } + + val fqName = replacedClass.fqNameWhenAvailable + val signature = + if (replacedClass.isAnonymousObject) { + null + } else if (fqName == null) { + logger.error("Unable to find signature/fqName for ${replacedClass.name}") + null + } else { + fqName.asString() + } + return TypeResult(classLabel, signature, classLabelResult.shortName) + } + + private fun tryReplaceParcelizeRawType(c: IrClass): Pair?>? { + if ( + c.superTypes.isNotEmpty() || + c.origin != IrDeclarationOrigin.DEFINED || + c.hasEqualFqName(FqName("java.lang.Object")) + ) { + return null + } + + val fqName = c.fqNameWhenAvailable + if (fqName == null) { + return null + } + + fun tryGetPair(arity: Int): Pair?>? { + val replaced = getClassByFqName(pluginContext, fqName)?.owner ?: return null + return Pair( + replaced, + List(arity) { + makeTypeProjection(pluginContext.irBuiltIns.anyNType, Variance.INVARIANT) + } + ) + } + + // The list of types handled here match + // https://github.com/JetBrains/kotlin/blob/d7c7d1efd2c0983c13b175e9e4b1cda979521159/plugins/parcelize/parcelize-compiler/src/org/jetbrains/kotlin/parcelize/ir/AndroidSymbols.kt + // Specifically, types are added for generic types created in AndroidSymbols.kt. + // This replacement is from a raw type to its matching parameterized type with `Object` type + // arguments. + return when (fqName.asString()) { + "java.util.ArrayList" -> tryGetPair(1) + "java.util.LinkedHashMap" -> tryGetPair(2) + "java.util.LinkedHashSet" -> tryGetPair(1) + "java.util.List" -> tryGetPair(1) + "java.util.TreeMap" -> tryGetPair(2) + "java.util.TreeSet" -> tryGetPair(1) + "java.lang.Class" -> tryGetPair(1) + else -> null + } + } + + private fun useAnonymousClass(c: IrClass) = + tw.lm.anonymousTypeMapping.getOrPut(c) { + TypeResults( + TypeResult(tw.getFreshIdLabel(), "", ""), + TypeResult(fakeKotlinType(), "TODO", "TODO") + ) + } + + fun fakeKotlinType(): Label { + val fakeKotlinPackageId: Label = + tw.getLabelFor("@\"FakeKotlinPackage\"", { tw.writePackages(it, "fake.kotlin") }) + val fakeKotlinClassId: Label = + tw.getLabelFor( + "@\"FakeKotlinClass\"", + { tw.writeClasses_or_interfaces(it, "FakeKotlinClass", fakeKotlinPackageId, it) } + ) + val fakeKotlinTypeId: Label = + tw.getLabelFor( + "@\"FakeKotlinType\"", + { tw.writeKt_nullable_types(it, fakeKotlinClassId) } + ) + return fakeKotlinTypeId + } + + // `args` can be null to describe a raw generic type. + // For non-generic types it will be zero-length list. + fun useSimpleTypeClass( + c: IrClass, + args: List?, + hasQuestionMark: Boolean + ): TypeResults { + val classInstanceResult = useClassInstance(c, args) + val javaClassId = classInstanceResult.typeResult.id + val kotlinQualClassName = getUnquotedClassLabel(c, args).classLabel + val javaResult = classInstanceResult.typeResult + val kotlinResult = + if (true) TypeResult(fakeKotlinType(), "TODO", "TODO") + else if (hasQuestionMark) { + val kotlinSignature = "$kotlinQualClassName?" // TODO: Is this right? + val kotlinLabel = "@\"kt_type;nullable;$kotlinQualClassName\"" + val kotlinId: Label = + tw.getLabelFor(kotlinLabel, { tw.writeKt_nullable_types(it, javaClassId) }) + TypeResult(kotlinId, kotlinSignature, "TODO") + } else { + val kotlinSignature = kotlinQualClassName // TODO: Is this right? + val kotlinLabel = "@\"kt_type;notnull;$kotlinQualClassName\"" + val kotlinId: Label = + tw.getLabelFor(kotlinLabel, { tw.writeKt_notnull_types(it, javaClassId) }) + TypeResult(kotlinId, kotlinSignature, "TODO") + } + return TypeResults(javaResult, kotlinResult) + } + + // Given either a primitive array or a boxed array, returns primitive arrays unchanged, + // but returns boxed arrays with a nullable, invariant component type, with any nested arrays + // similarly transformed. For example, Array> would become Array?> + // Array<*> will become Array. + private fun getInvariantNullableArrayType(arrayType: IrSimpleType): IrSimpleType = + if (arrayType.isPrimitiveArray()) arrayType + else { + val componentType = arrayType.getArrayElementType(pluginContext.irBuiltIns) + val componentTypeBroadened = + when (componentType) { + is IrSimpleType -> + if (isArray(componentType)) getInvariantNullableArrayType(componentType) + else componentType + else -> componentType + } + val unchanged = + componentType == componentTypeBroadened && + (arrayType.arguments[0] as? IrTypeProjection)?.variance == Variance.INVARIANT && + componentType.isNullable() + if (unchanged) arrayType + else + IrSimpleTypeImpl( + arrayType.classifier, + true, + listOf(makeTypeProjection(componentTypeBroadened, Variance.INVARIANT)), + listOf() + ) + } + + /* + Kotlin arrays can be broken down as: + + isArray(t) + |- t.isBoxedArray + | |- t.isArray() e.g. Array, Array + | |- t.isNullableArray() e.g. Array?, Array? + |- t.isPrimitiveArray() e.g. BooleanArray + + For the corresponding Java types: + Boxed arrays are represented as e.g. java.lang.Boolean[]. + Primitive arrays are represented as e.g. boolean[]. + */ + + private fun isArray(t: IrType) = t.isBoxedArray || t.isPrimitiveArray() + + data class ArrayInfo( + val elementTypeResults: TypeResults, + val componentTypeResults: TypeResults, + val dimensions: Int + ) + + /** + * `t` is somewhere in a stack of array types, or possibly the element type of the innermost + * array. For example, in `Array>`, we will be called with `t` being + * `Array>`, then `Array`, then `Int`. `isPrimitiveArray` is true if we are + * immediately nested inside a primitive array. + */ + private fun useArrayType(t: IrType, isPrimitiveArray: Boolean): ArrayInfo { + + if (!isArray(t)) { + val nullableT = if (t.isPrimitiveType() && !isPrimitiveArray) t.makeNullable() else t + val typeResults = useType(nullableT) + return ArrayInfo(typeResults, typeResults, 0) + } + + if (t !is IrSimpleType) { + logger.error("Unexpected non-simple array type: ${t.javaClass}") + return ArrayInfo(extractErrorType(), extractErrorType(), 0) + } + + val arrayClass = t.classifier.owner + if (arrayClass !is IrClass) { + logger.error("Unexpected owner type for array type: ${arrayClass.javaClass}") + return ArrayInfo(extractErrorType(), extractErrorType(), 0) + } + + // Because Java's arrays are covariant, Kotlin will render + // Array as Object[], Array> as Object[][] etc. + val elementType = + if ( + (t.arguments.singleOrNull() as? IrTypeProjection)?.variance == Variance.IN_VARIANCE + ) { + pluginContext.irBuiltIns.anyType + } else { + t.getArrayElementType(pluginContext.irBuiltIns) + } + + val recInfo = useArrayType(elementType, t.isPrimitiveArray()) + + val javaShortName = recInfo.componentTypeResults.javaResult.shortName + "[]" + val kotlinShortName = recInfo.componentTypeResults.kotlinResult.shortName + "[]" + val elementTypeLabel = recInfo.elementTypeResults.javaResult.id + val componentTypeLabel = recInfo.componentTypeResults.javaResult.id + val dimensions = recInfo.dimensions + 1 + + val id = + tw.getLabelFor("@\"array;$dimensions;{$elementTypeLabel}\"") { + tw.writeArrays(it, javaShortName, elementTypeLabel, dimensions, componentTypeLabel) + + extractClassSupertypes( + arrayClass, + it, + ExtractSupertypesMode.Specialised(t.arguments) + ) + + // array.length + val length = tw.getLabelFor("@\"field;{$it};length\"") + val intTypeIds = useType(pluginContext.irBuiltIns.intType) + tw.writeFields(length, "length", intTypeIds.javaResult.id, it, length) + tw.writeFieldsKotlinType(length, intTypeIds.kotlinResult.id) + addModifiers(length, "public", "final") + + // Note we will only emit one `clone()` method per Java array type, so we choose + // `Array` as its Kotlin + // return type, where C is the component type with any nested arrays themselves + // invariant and nullable. + val kotlinCloneReturnType = getInvariantNullableArrayType(t).makeNullable() + val kotlinCloneReturnTypeLabel = useType(kotlinCloneReturnType).kotlinResult.id + + val clone = tw.getLabelFor("@\"callable;{$it}.clone(){$it}\"") + tw.writeMethods(clone, "clone", "clone()", it, it, clone) + tw.writeMethodsKotlinType(clone, kotlinCloneReturnTypeLabel) + addModifiers(clone, "public") + } + + val javaResult = + TypeResult(id, recInfo.componentTypeResults.javaResult.signature + "[]", javaShortName) + val kotlinResult = + TypeResult( + fakeKotlinType(), + recInfo.componentTypeResults.kotlinResult.signature + "[]", + kotlinShortName + ) + val typeResults = TypeResults(javaResult, kotlinResult) + + return ArrayInfo(recInfo.elementTypeResults, typeResults, dimensions) + } + + enum class TypeContext { + RETURN, + GENERIC_ARGUMENT, + OTHER + } + + private fun useSimpleType(s: IrSimpleType, context: TypeContext): TypeResults { + if (s.abbreviation != null) { + // TODO: Extract this information + } + // We use this when we don't actually have an IrClass for a class + // we want to refer to + // TODO: Eliminate the need for this if possible + fun makeClass(pkgName: String, className: String): Label { + val pkgId = extractPackage(pkgName) + val label = "@\"class;$pkgName.$className\"" + val classId: Label = + tw.getLabelFor(label, { tw.writeClasses_or_interfaces(it, className, pkgId, it) }) + return classId + } + fun primitiveType( + kotlinClass: IrClass, + primitiveName: String?, + otherIsPrimitive: Boolean, + javaClass: IrClass, + kotlinPackageName: String, + kotlinClassName: String + ): TypeResults { + // Note the use of `hasEnhancedNullability` here covers cases like `@NotNull Integer`, + // which must be extracted as `Integer` not `int`. + val javaResult = + if ( + (context == TypeContext.RETURN || + (context == TypeContext.OTHER && otherIsPrimitive)) && + !s.isNullable() && + getKotlinType(s)?.hasEnhancedNullability() != true && + primitiveName != null + ) { + val label: Label = + tw.getLabelFor( + "@\"type;$primitiveName\"", + { tw.writePrimitives(it, primitiveName) } + ) + TypeResult(label, primitiveName, primitiveName) + } else { + addClassLabel(javaClass, listOf()) + } + val kotlinClassId = useClassInstance(kotlinClass, listOf()).typeResult.id + val kotlinResult = + if (true) TypeResult(fakeKotlinType(), "TODO", "TODO") + else if (s.isNullable()) { + val kotlinSignature = + "$kotlinPackageName.$kotlinClassName?" // TODO: Is this right? + val kotlinLabel = "@\"kt_type;nullable;$kotlinPackageName.$kotlinClassName\"" + val kotlinId: Label = + tw.getLabelFor( + kotlinLabel, + { tw.writeKt_nullable_types(it, kotlinClassId) } + ) + TypeResult(kotlinId, kotlinSignature, "TODO") + } else { + val kotlinSignature = + "$kotlinPackageName.$kotlinClassName" // TODO: Is this right? + val kotlinLabel = "@\"kt_type;notnull;$kotlinPackageName.$kotlinClassName\"" + val kotlinId: Label = + tw.getLabelFor(kotlinLabel, { tw.writeKt_notnull_types(it, kotlinClassId) }) + TypeResult(kotlinId, kotlinSignature, "TODO") + } + return TypeResults(javaResult, kotlinResult) + } + + val owner = s.classifier.owner + val primitiveInfo = primitiveTypeMapping.getPrimitiveInfo(s) + + when { + primitiveInfo != null -> { + if (owner is IrClass) { + return primitiveType( + owner, + primitiveInfo.primitiveName, + primitiveInfo.otherIsPrimitive, + primitiveInfo.javaClass, + primitiveInfo.kotlinPackageName, + primitiveInfo.kotlinClassName + ) + } else { + logger.error( + "Got primitive info for non-class (${owner.javaClass}) for ${s.render()}" + ) + return extractErrorType() + } + } + (s.isBoxedArray && s.arguments.isNotEmpty()) || s.isPrimitiveArray() -> { + val arrayInfo = useArrayType(s, false) + return arrayInfo.componentTypeResults + } + owner is IrClass -> { + val args = if (s.isRawType()) null else s.arguments + + return useSimpleTypeClass(owner, args, s.isNullable()) + } + owner is IrTypeParameter -> { + val javaResult = useTypeParameter(owner) + val aClassId = makeClass("kotlin", "TypeParam") // TODO: Wrong + val kotlinResult = + if (true) TypeResult(fakeKotlinType(), "TODO", "TODO") + else if (s.isNullable()) { + val kotlinSignature = "${javaResult.signature}?" // TODO: Wrong + val kotlinLabel = "@\"kt_type;nullable;type_param\"" // TODO: Wrong + val kotlinId: Label = + tw.getLabelFor(kotlinLabel, { tw.writeKt_nullable_types(it, aClassId) }) + TypeResult(kotlinId, kotlinSignature, "TODO") + } else { + val kotlinSignature = javaResult.signature // TODO: Wrong + val kotlinLabel = "@\"kt_type;notnull;type_param\"" // TODO: Wrong + val kotlinId: Label = + tw.getLabelFor(kotlinLabel, { tw.writeKt_notnull_types(it, aClassId) }) + TypeResult(kotlinId, kotlinSignature, "TODO") + } + return TypeResults(javaResult, kotlinResult) + } + else -> { + logger.error("Unrecognised IrSimpleType: " + s.javaClass + ": " + s.render()) + return extractErrorType() + } + } + } + + private fun parentOf(d: IrDeclaration): IrDeclarationParent { + if (d is IrField) { + return getFieldParent(d) + } + return d.parent + } + + fun useDeclarationParentOf( + // The declaration + d: IrDeclaration, + // Whether the type of entity whose parent this is can be a + // top-level entity in the JVM's eyes. If so, then its parent may + // be a file; otherwise, if dp is a file foo.kt, then the parent + // is really the JVM class FooKt. + canBeTopLevel: Boolean, + classTypeArguments: List? = null, + inReceiverContext: Boolean = false + ): Label? { + + val parent = parentOf(d) + if (parent is IrExternalPackageFragment) { + // This is in a file class. + val fqName = getFileClassFqName(d) + if (fqName == null) { + logger.error( + "Can't get FqName for declaration in external package fragment ${d.javaClass}" + ) + return null + } + return extractFileClass(fqName) + } + return useDeclarationParent(parent, canBeTopLevel, classTypeArguments, inReceiverContext) + } + + // Generally, useDeclarationParentOf should be used instead of + // calling this directly, as this cannot handle + // IrExternalPackageFragment + fun useDeclarationParent( + // The declaration parent according to Kotlin + dp: IrDeclarationParent, + // Whether the type of entity whose parent this is can be a + // top-level entity in the JVM's eyes. If so, then its parent may + // be a file; otherwise, if dp is a file foo.kt, then the parent + // is really the JVM class FooKt. + canBeTopLevel: Boolean, + classTypeArguments: List? = null, + inReceiverContext: Boolean = false + ): Label? = + when (dp) { + is IrFile -> + if (canBeTopLevel) { + usePackage(dp.packageFqName.asString()) + } else { + extractFileClass(dp) + } + is IrClass -> + if (classTypeArguments != null) { + useClassInstance(dp, classTypeArguments, inReceiverContext).typeResult.id + } else { + val replacedType = tryReplaceParcelizeRawType(dp) + if (replacedType == null) { + useClassSource(dp) + } else { + useClassInstance(replacedType.first, replacedType.second, inReceiverContext) + .typeResult + .id + } + } + is IrFunction -> useFunction(dp) + is IrExternalPackageFragment -> { + logger.error("Unable to handle IrExternalPackageFragment as an IrDeclarationParent") + null + } + else -> { + logger.error("Unrecognised IrDeclarationParent: " + dp.javaClass) + null + } + } + + private val IrDeclaration.isAnonymousFunction + get() = this is IrSimpleFunction && name == SpecialNames.NO_NAME_PROVIDED + + data class FunctionNames(val nameInDB: String, val kotlinName: String) + + @OptIn(ObsoleteDescriptorBasedAPI::class) + private fun getJvmModuleName(f: IrFunction) = + NameUtils.sanitizeAsJavaIdentifier( + getJvmModuleNameForDeserializedDescriptor(f.descriptor) + ?: JvmCodegenUtil.getModuleName(pluginContext.moduleDescriptor) + ) + + fun getFunctionShortName(f: IrFunction): FunctionNames { + if (f.origin == IrDeclarationOrigin.LOCAL_FUNCTION_FOR_LAMBDA || f.isAnonymousFunction) + return FunctionNames( + OperatorNameConventions.INVOKE.asString(), + OperatorNameConventions.INVOKE.asString() + ) + + fun getSuffixIfInternal() = + if ( + f.visibility == DescriptorVisibilities.INTERNAL && + f !is IrConstructor && + !(f.parent is IrFile || isExternalFileClassMember(f)) + ) { + "\$" + getJvmModuleName(f) + } else { + "" + } + + (f as? IrSimpleFunction)?.correspondingPropertySymbol?.let { + val propName = it.owner.name.asString() + val getter = it.owner.getter + val setter = it.owner.setter + + if (it.owner.parentClassOrNull?.kind == ClassKind.ANNOTATION_CLASS) { + if (getter == null) { + logger.error( + "Expected to find a getter for a property inside an annotation class" + ) + return FunctionNames(propName, propName) + } else { + val jvmName = getJvmName(getter) + return FunctionNames(jvmName ?: propName, propName) + } + } + + val maybeFunctionName = + when (f) { + getter -> JvmAbi.getterName(propName) + setter -> JvmAbi.setterName(propName) + else -> { + logger.error( + "Function has a corresponding property, but is neither the getter nor the setter" + ) + null + } + } + maybeFunctionName?.let { defaultFunctionName -> + val suffix = + if ( + f.visibility == DescriptorVisibilities.PRIVATE && + f.origin == IrDeclarationOrigin.DEFAULT_PROPERTY_ACCESSOR + ) { + "\$private" + } else { + getSuffixIfInternal() + } + return FunctionNames( + getJvmName(f) ?: "$defaultFunctionName$suffix", + defaultFunctionName + ) + } + } + return FunctionNames( + getJvmName(f) ?: "${f.name.asString()}${getSuffixIfInternal()}", + f.name.asString() + ) + } + + // This excludes class type parameters that show up in (at least) constructors' typeParameters + // list. + fun getFunctionTypeParameters(f: IrFunction): List { + return if (f is IrConstructor) f.typeParameters + else f.typeParameters.filter { it.parent == f } + } + + private fun getTypeParameters(dp: IrDeclarationParent): List = + when (dp) { + is IrClass -> dp.typeParameters + is IrFunction -> getFunctionTypeParameters(dp) + else -> listOf() + } + + private fun getEnclosingClass(it: IrDeclarationParent): IrClass? = + when (it) { + is IrClass -> it + is IrFunction -> getEnclosingClass(it.parent) + else -> null + } + + val javaUtilCollection by lazy { referenceExternalClass("java.util.Collection") } + + val wildcardCollectionType by lazy { + javaUtilCollection?.let { it.symbol.typeWithArguments(listOf(IrStarProjectionImpl)) } + } + + private fun makeCovariant(t: IrTypeArgument) = + t.typeOrNull?.let { makeTypeProjection(it, Variance.OUT_VARIANCE) } ?: t + + private fun makeArgumentsCovariant(t: IrType) = + (t as? IrSimpleType)?.let { + t.toBuilder() + .also { b -> b.arguments = b.arguments.map(this::makeCovariant) } + .buildSimpleType() + } ?: t + + fun eraseCollectionsMethodParameterType( + t: IrType, + collectionsMethodName: String, + paramIdx: Int + ) = + when (collectionsMethodName) { + "contains", + "remove", + "containsKey", + "containsValue", + "get", + "indexOf", + "lastIndexOf" -> javaLangObjectType + "getOrDefault" -> if (paramIdx == 0) javaLangObjectType else null + "containsAll", + "removeAll", + "retainAll" -> wildcardCollectionType + // Kotlin defines these like addAll(Collection); Java uses addAll(Collection) + "putAll", + "addAll" -> makeArgumentsCovariant(t) + else -> null + } ?: t + + private fun overridesFunctionDefinedOn(f: IrFunction, packageName: String, className: String) = + (f as? IrSimpleFunction)?.let { + it.overriddenSymbols.any { overridden -> + overridden.owner.parentClassOrNull?.let { defnClass -> + defnClass.name.asString() == className && + defnClass.packageFqName?.asString() == packageName + } ?: false + } + } ?: false + + @OptIn(ObsoleteDescriptorBasedAPI::class) + fun overridesCollectionsMethodWithAlteredParameterTypes(f: IrFunction) = + BuiltinMethodsWithSpecialGenericSignature + .getOverriddenBuiltinFunctionWithErasedValueParametersInJava(f.descriptor) != null || + (f.name.asString() == "putAll" && + overridesFunctionDefinedOn(f, "kotlin.collections", "MutableMap")) || + (f.name.asString() == "addAll" && + overridesFunctionDefinedOn(f, "kotlin.collections", "MutableCollection")) || + (f.name.asString() == "addAll" && + overridesFunctionDefinedOn(f, "kotlin.collections", "MutableList")) + + private val jvmWildcardAnnotation = FqName("kotlin.jvm.JvmWildcard") + private val jvmWildcardSuppressionAnnotation = FqName("kotlin.jvm.JvmSuppressWildcards") + + private fun arrayExtendsAdditionAllowed(t: IrSimpleType): Boolean = + // Note the array special case includes Array<*>, which does permit adding `? extends ...` + // (making `? extends Object[]` in that case) + // Surprisingly Array does permit this as well, though the contravariant array lowers + // to Object[] so this ends up `? extends Object[]` as well. + t.arguments[0].let { + when (it) { + is IrTypeProjection -> + when (it.variance) { + Variance.INVARIANT -> false + Variance.IN_VARIANCE -> !(it.type.isAny() || it.type.isNullableAny()) + Variance.OUT_VARIANCE -> extendsAdditionAllowed(it.type) + } + else -> true + } + } + + private fun extendsAdditionAllowed(t: IrType) = + if (t.isBoxedArray) { + if (t is IrSimpleType) { + arrayExtendsAdditionAllowed(t) + } else { + logger.warn("Boxed array of unexpected kind ${t.javaClass}") + // Return false, for no particular reason + false + } + } else { + ((t as? IrSimpleType)?.classOrNull?.owner?.isFinalClass) != true + } + + private fun wildcardAdditionAllowed( + v: Variance, + t: IrType, + addByDefault: Boolean, + javaVariance: Variance? + ) = + when { + t.hasAnnotation(jvmWildcardAnnotation) -> true + // If a Java declaration specifies a variance, introduce it even if it's pointless (e.g. + // ? extends FinalClass, or ? super Object) + javaVariance == v -> true + !addByDefault -> false + v == Variance.IN_VARIANCE -> !(t.isNullableAny() || t.isAny()) + v == Variance.OUT_VARIANCE -> extendsAdditionAllowed(t) + else -> false + } + + // Returns true if `t` has `@JvmSuppressWildcards` or `@JvmSuppressWildcards(true)`, + // false if it has `@JvmSuppressWildcards(false)`, + // and null if the annotation is not present. + @Suppress("UNCHECKED_CAST") + private fun getWildcardSuppressionDirective(t: IrAnnotationContainer) = + t.getAnnotation(jvmWildcardSuppressionAnnotation)?.let { + (it.getValueArgument(0) as? IrConst)?.value ?: true + } + + private fun addJavaLoweringArgumentWildcards( + p: IrTypeParameter, + t: IrTypeArgument, + addByDefault: Boolean, + javaType: JavaType? + ): IrTypeArgument = + (t as? IrTypeProjection)?.let { + val newAddByDefault = getWildcardSuppressionDirective(it.type)?.not() ?: addByDefault + val newBase = addJavaLoweringWildcards(it.type, newAddByDefault, javaType) + // Note javaVariance == null means we don't have a Java type to conform to -- for + // example if this is a Kotlin source definition. + val javaVariance = + javaType?.let { jType -> + when (jType) { + is JavaWildcardType -> + if (jType.isExtends) Variance.OUT_VARIANCE else Variance.IN_VARIANCE + else -> Variance.INVARIANT + } + } + val newVariance = + if ( + it.variance == Variance.INVARIANT && + p.variance != Variance.INVARIANT && + // The next line forbids inferring a wildcard type when we have a + // corresponding Java type with conflicting variance. + // For example, Java might declare f(Comparable cs), in which + // case we shouldn't add a `? super ...` + // wildcard. Note if javaType is unknown (e.g. this is a Kotlin source + // element), we assume wildcards should be added. + (javaVariance == null || javaVariance == p.variance) && + wildcardAdditionAllowed(p.variance, it.type, newAddByDefault, javaVariance) + ) + p.variance + else it.variance + if (newBase !== it.type || newVariance != it.variance) + makeTypeProjection(newBase, newVariance) + else null + } ?: t + + private fun getJavaTypeArgument(jt: JavaType, idx: Int): JavaType? = + when (jt) { + is JavaWildcardType -> jt.bound?.let { getJavaTypeArgument(it, idx) } + is JavaClassifierType -> jt.typeArguments.getOrNull(idx) + is JavaArrayType -> if (idx == 0) jt.componentType else null + else -> null + } + + fun addJavaLoweringWildcards(t: IrType, addByDefault: Boolean, javaType: JavaType?): IrType = + (t as? IrSimpleType)?.let { + val newAddByDefault = getWildcardSuppressionDirective(t)?.not() ?: addByDefault + val typeParams = it.classOrNull?.owner?.typeParameters ?: return t + val newArgs = + typeParams.zip(it.arguments).mapIndexed { idx, pair -> + addJavaLoweringArgumentWildcards( + pair.first, + pair.second, + newAddByDefault, + javaType?.let { jt -> getJavaTypeArgument(jt, idx) } + ) + } + return if (newArgs.zip(it.arguments).all { pair -> pair.first === pair.second }) t + else it.toBuilder().also { builder -> builder.arguments = newArgs }.buildSimpleType() + } ?: t + + /* + * This is the normal getFunctionLabel function to use. If you want + * to refer to the function in its source class then + * classTypeArgsIncludingOuterClasses should be null. Otherwise, it + * is the list of type arguments that need to be applied to its + * enclosing classes to get the instantiation that this function is + * in. + */ + fun getFunctionLabel( + f: IrFunction, + classTypeArgsIncludingOuterClasses: List? + ): String? { + val parentId = useDeclarationParentOf(f, false, classTypeArgsIncludingOuterClasses, true) + if (parentId == null) { + logger.error("Couldn't get parent ID for function label") + return null + } + return getFunctionLabel(f, parentId, classTypeArgsIncludingOuterClasses) + } + + /* + * There are some pairs of classes (e.g. `kotlin.Throwable` and + * `java.lang.Throwable`) which are really just 2 different names + * for the same class. However, we extract them as separate + * classes. When extracting `kotlin.Throwable`'s methods, if we + * looked up the parent ID ourselves, we would get as ID for + * `java.lang.Throwable`, which isn't what we want. So we have to + * allow it to be passed in. + * + * `maybeParameterList` can be supplied to override the function's + * value parameters; this is used for generating labels of overloads + * that omit one or more parameters that has a default value specified. + */ + @OptIn(ObsoleteDescriptorBasedAPI::class) + fun getFunctionLabel( + f: IrFunction, + parentId: Label, + classTypeArgsIncludingOuterClasses: List?, + maybeParameterList: List? = null + ): String = + getFunctionLabel( + f.parent, + parentId, + getFunctionShortName(f).nameInDB, + (maybeParameterList ?: f.valueParameters).map { it.type }, + getAdjustedReturnType(f), + f.extensionReceiverParameter?.type, + getFunctionTypeParameters(f), + classTypeArgsIncludingOuterClasses, + overridesCollectionsMethodWithAlteredParameterTypes(f), + getJavaCallable(f), + !getInnermostWildcardSupppressionAnnotation(f) + ) + + /* + * This function actually generates the label for a function. + * Sometimes, a function is only generated by kotlinc when writing a + * class file, so there is no corresponding `IrFunction` for it. + * This function therefore takes all the constituent parts of a + * function instead. + */ + fun getFunctionLabel( + // The parent of the function; normally f.parent. + parent: IrDeclarationParent, + // The ID of the function's parent, or null if we should work it out ourselves. + parentId: Label, + // The name of the function; normally f.name.asString(). + name: String, + // The types of the value parameters that the functions takes; normally + // f.valueParameters.map { it.type }. + parameterTypes: List, + // The return type of the function; normally f.returnType. + returnType: IrType, + // The extension receiver of the function, if any; normally + // f.extensionReceiverParameter?.type. + extensionParamType: IrType?, + // The type parameters of the function. This does not include type parameters of enclosing + // classes. + functionTypeParameters: List, + // The type arguments of enclosing classes of the function. + classTypeArgsIncludingOuterClasses: List?, + // If true, this method implements a Java Collections interface (Collection, Map or List) + // and may need + // parameter erasure to match the way this class will appear to an external consumer of the + // .class file. + overridesCollectionsMethod: Boolean, + // The Java signature of this callable, if known. + javaSignature: JavaMember?, + // If true, Java wildcards implied by Kotlin type parameter variance should be added by + // default to this function's value parameters' types. + // (Return-type wildcard addition is always off by default) + addParameterWildcardsByDefault: Boolean, + // The prefix used in the label. "callable", unless a property label is created, then it's + // "property". + prefix: String = "callable" + ): String { + val allParamTypes = + if (extensionParamType == null) parameterTypes + else listOf(extensionParamType) + parameterTypes + + val substitutionMap = + classTypeArgsIncludingOuterClasses?.let { notNullArgs -> + if (notNullArgs.isEmpty()) { + null + } else { + val enclosingClass = getEnclosingClass(parent) + enclosingClass?.let { notNullClass -> + makeTypeGenericSubstitutionMap(notNullClass, notNullArgs) + } + } + } + val getIdForFunctionLabel = { it: IndexedValue -> + // Kotlin rewrites certain Java collections types adding additional generic + // constraints-- for example, + // Collection.remove(Object) because Collection.remove(Collection::E) in the Kotlin + // universe. + // If this has happened, erase the type again to get the correct Java signature. + val maybeAmendedForCollections = + if (overridesCollectionsMethod) + eraseCollectionsMethodParameterType(it.value, name, it.index) + else it.value + // Add any wildcard types that the Kotlin compiler would add in the Java lowering of + // this function: + val withAddedWildcards = + addJavaLoweringWildcards( + maybeAmendedForCollections, + addParameterWildcardsByDefault, + javaSignature?.let { sig -> getJavaValueParameterType(sig, it.index) } + ) + // Now substitute any class type parameters in: + val maybeSubbed = + withAddedWildcards.substituteTypeAndArguments( + substitutionMap, + TypeContext.OTHER, + pluginContext + ) + // Finally, mimic the Java extractor's behaviour by naming functions with type + // parameters for their erased types; + // those without type parameters are named for the generic type. + val maybeErased = + if (functionTypeParameters.isEmpty()) maybeSubbed else erase(maybeSubbed) + "{${useType(maybeErased).javaResult.id}}" + } + val paramTypeIds = + allParamTypes + .withIndex() + .joinToString(separator = ",", transform = getIdForFunctionLabel) + val labelReturnType = + if (name == "") pluginContext.irBuiltIns.unitType + else + erase( + returnType.substituteTypeAndArguments( + substitutionMap, + TypeContext.RETURN, + pluginContext + ) + ) + // Note that `addJavaLoweringWildcards` is not required here because the return type used to + // form the function + // label is always erased. + val returnTypeId = useType(labelReturnType, TypeContext.RETURN).javaResult.id + // This suffix is added to generic methods (and constructors) to match the Java extractor's + // behaviour. + // Comments in that extractor indicates it didn't want the label of the callable to clash + // with the raw + // method (and presumably that disambiguation is never needed when the method belongs to a + // parameterized + // instance of a generic class), but as of now I don't know when the raw method would be + // referred to. + val typeArgSuffix = + if ( + functionTypeParameters.isNotEmpty() && + classTypeArgsIncludingOuterClasses.isNullOrEmpty() + ) + "<${functionTypeParameters.size}>" + else "" + return "@\"$prefix;{$parentId}.$name($paramTypeIds){$returnTypeId}$typeArgSuffix\"" + } + + val javaLangClass by lazy { referenceExternalClass("java.lang.Class") } + + fun kClassToJavaClass(t: IrType): IrType { + when (t) { + is IrSimpleType -> { + if (t.classifier == pluginContext.irBuiltIns.kClassClass) { + javaLangClass?.let { jlc -> + return jlc.symbol.typeWithArguments(t.arguments) + } + } else { + t.classOrNull?.let { tCls -> + if (t.isBoxedArray) { + (t.arguments.singleOrNull() as? IrTypeProjection)?.let { elementTypeArg + -> + val elementType = elementTypeArg.type + val replacedElementType = kClassToJavaClass(elementType) + if (replacedElementType !== elementType) { + val newArg = + makeTypeProjection( + replacedElementType, + elementTypeArg.variance + ) + return tCls + .typeWithArguments(listOf(newArg)) + .codeQlWithHasQuestionMark(t.isNullable()) + } + } + } + } + } + } + is IrDynamicType -> {} + is IrErrorType -> {} + } + return t + } + + fun isAnnotationClassField(f: IrField) = + f.correspondingPropertySymbol?.let { isAnnotationClassProperty(it) } ?: false + + private fun isAnnotationClassProperty(p: IrPropertySymbol) = + p.owner.parentClassOrNull?.kind == ClassKind.ANNOTATION_CLASS + + fun getAdjustedReturnType(f: IrFunction): IrType { + // Replace annotation val accessor types as needed: + (f as? IrSimpleFunction)?.correspondingPropertySymbol?.let { + if (isAnnotationClassProperty(it) && f == it.owner.getter) { + val replaced = kClassToJavaClass(f.returnType) + if (replaced != f.returnType) return replaced + } + } + + // The return type of `java.util.concurrent.ConcurrentHashMap.keySet/0` is defined as + // `Set` in the stubs inside the Android SDK. + // This does not match the Java SDK return type: `ConcurrentHashMap.KeySetView`, so + // it's adjusted here. + // This is a deliberate change in the Android SDK: + // https://github.com/AndroidSDKSources/android-sdk-sources-for-api-level-31/blob/2c56b25f619575bea12f9c5520ed2259620084ac/java/util/concurrent/ConcurrentHashMap.java#L1244-L1249 + // The annotation on the source is not visible in the android.jar, so we can't make the + // change based on that. + // TODO: there are other instances of `dalvik.annotation.codegen.CovariantReturnType` in the + // Android SDK, we should handle those too if they cause DB inconsistencies + val parentClass = f.parentClassOrNull + if ( + parentClass == null || + parentClass.fqNameWhenAvailable?.asString() != + "java.util.concurrent.ConcurrentHashMap" || + getFunctionShortName(f).nameInDB != "keySet" || + f.valueParameters.isNotEmpty() || + f.returnType.classFqName?.asString() != "kotlin.collections.MutableSet" + ) { + return f.returnType + } + + val otherKeySet = + parentClass.declarations.findSubType { + it.name.asString() == "keySet" && it.valueParameters.size == 1 + } ?: return f.returnType + + return otherKeySet.returnType.codeQlWithHasQuestionMark(false) + } + + @OptIn(ObsoleteDescriptorBasedAPI::class) + fun getJavaCallable(f: IrFunction) = + (f.descriptor.source as? JavaSourceElement)?.javaElement as? JavaMember + + fun getJavaValueParameterType(m: JavaMember, idx: Int) = + when (m) { + is JavaMethod -> m.valueParameters[idx].type + is JavaConstructor -> m.valueParameters[idx].type + else -> null + } + + fun getInnermostWildcardSupppressionAnnotation(d: IrDeclaration) = + getWildcardSuppressionDirective(d) + ?: + // Note not using `parentsWithSelf` as that only works if `d` is an IrDeclarationParent + d.parents + .filterIsInstance() + .mapNotNull { getWildcardSuppressionDirective(it) } + .firstOrNull() + ?: false + + /** + * Class to hold labels for generated classes around local functions, lambdas, function + * references, and property references. + */ + open class GeneratedClassLabels( + val type: TypeResults, + val constructor: Label, + val constructorBlock: Label + ) + + /** + * Class to hold labels generated for locally visible functions, such as + * - local functions, + * - lambdas, and + * - wrappers around function references. + */ + class LocallyVisibleFunctionLabels( + type: TypeResults, + constructor: Label, + constructorBlock: Label, + val function: Label + ) : GeneratedClassLabels(type, constructor, constructorBlock) + + /** + * Gets the labels for functions belonging to + * - local functions, and + * - lambdas. + */ + fun getLocallyVisibleFunctionLabels(f: IrFunction): LocallyVisibleFunctionLabels { + if (!f.isLocalFunction()) { + logger.error("Extracting a non-local function as a local one") + } + + var res = tw.lm.locallyVisibleFunctionLabelMapping[f] + if (res == null) { + val javaResult = TypeResult(tw.getFreshIdLabel(), "", "") + val kotlinResult = TypeResult(tw.getFreshIdLabel(), "", "") + tw.writeKt_notnull_types(kotlinResult.id, javaResult.id) + res = + LocallyVisibleFunctionLabels( + TypeResults(javaResult, kotlinResult), + tw.getFreshIdLabel(), + tw.getFreshIdLabel(), + tw.getFreshIdLabel() + ) + tw.lm.locallyVisibleFunctionLabelMapping[f] = res + } + + return res + } + + fun getExistingLocallyVisibleFunctionLabel(f: IrFunction): Label? { + if (!f.isLocalFunction()) { + return null + } + + return tw.lm.locallyVisibleFunctionLabelMapping[f]?.function + } + + private fun kotlinFunctionToJavaEquivalent(f: IrFunction, noReplace: Boolean): IrFunction = + if (noReplace) f + else + f.parentClassOrNull?.let { parentClass -> + getJavaEquivalentClass(parentClass)?.let { javaClass -> + if (javaClass != parentClass) { + var jvmName = getFunctionShortName(f).nameInDB + if ( + f.name.asString() == "get" && + parentClass.fqNameWhenAvailable?.asString() == "kotlin.String" + ) { + // `kotlin.String.get` has an equivalent `java.lang.String.get`, which + // in turn will be stored in the DB as `java.lang.String.charAt`. + // Maybe all operators should be handled the same way, but so far I only + // found this case that needed to be special cased. This is the + // only operator in `JvmNames.specialFunctions` + jvmName = "get" + } + // Look for an exact type match... + javaClass.declarations.findSubType { decl -> + !decl.isFakeOverride && + decl.name.asString() == jvmName && + decl.valueParameters.size == f.valueParameters.size && + decl.valueParameters.zip(f.valueParameters).all { p -> + erase(p.first.type).classifierOrNull == + erase(p.second.type).classifierOrNull + } + } + ?: + // Or check property accessors: + (f.propertyIfAccessor as? IrProperty)?.let { kotlinProp -> + val javaProp = + javaClass.declarations.findSubType { decl -> + decl.name == kotlinProp.name + } + if (javaProp?.getter?.name == f.name) javaProp.getter + else if (javaProp?.setter?.name == f.name) javaProp.setter else null + } + ?: run { + val parentFqName = parentClass.fqNameWhenAvailable?.asString() + logger.warn( + "Couldn't find a Java equivalent function to $parentFqName.${f.name.asString()} in ${javaClass.fqNameWhenAvailable?.asString()}" + ) + null + } + } else null + } + } ?: f + + fun isPrivate(d: IrDeclaration) = + when (d) { + is IrDeclarationWithVisibility -> + d.visibility.let { + it == DescriptorVisibilities.PRIVATE || + it == DescriptorVisibilities.PRIVATE_TO_THIS + } + else -> false + } + + fun useFunction( + f: IrFunction, + classTypeArgsIncludingOuterClasses: List? = null, + noReplace: Boolean = false + ): Label? { + if (f.isLocalFunction()) { + val ids = getLocallyVisibleFunctionLabels(f) + return ids.function.cast() + } + val javaFun = kotlinFunctionToJavaEquivalent(f, noReplace) + val parentId = + useDeclarationParentOf(javaFun, false, classTypeArgsIncludingOuterClasses, true) + if (parentId == null) { + logger.error("Couldn't find parent ID for function ${f.name.asString()}") + return null + } + return useFunction(f, javaFun, parentId, classTypeArgsIncludingOuterClasses) + } + + fun useFunction( + f: IrFunction, + parentId: Label, + classTypeArgsIncludingOuterClasses: List?, + noReplace: Boolean = false + ): Label { + if (f.isLocalFunction()) { + val ids = getLocallyVisibleFunctionLabels(f) + return ids.function.cast() + } + val javaFun = kotlinFunctionToJavaEquivalent(f, noReplace) + return useFunction(f, javaFun, parentId, classTypeArgsIncludingOuterClasses) + } + + private fun useFunction( + f: IrFunction, + javaFun: IrFunction, + parentId: Label, + classTypeArgsIncludingOuterClasses: List? + ): Label { + val label = getFunctionLabel(javaFun, parentId, classTypeArgsIncludingOuterClasses) + val id: Label = + tw.getLabelFor(label) { + extractPrivateSpecialisedDeclaration(f, classTypeArgsIncludingOuterClasses) + } + if (isExternalDeclaration(javaFun)) { + extractFunctionLaterIfExternalFileMember(javaFun) + extractExternalEnclosingClassLater(javaFun) + } + return id + } + + private fun extractPrivateSpecialisedDeclaration( + d: IrDeclaration, + classTypeArgsIncludingOuterClasses: List? + ) { + // Note here `classTypeArgsIncludingOuterClasses` being null doesn't signify a raw receiver + // type but rather that no type args were supplied. + // This is because a call to a private method can only be observed inside Kotlin code, and + // Kotlin can't represent raw types. + if ( + this is KotlinFileExtractor && + isPrivate(d) && + classTypeArgsIncludingOuterClasses != null && + classTypeArgsIncludingOuterClasses.isNotEmpty() + ) { + d.parent.let { + when (it) { + is IrClass -> + this.extractDeclarationPrototype( + d, + useClassInstance(it, classTypeArgsIncludingOuterClasses).typeResult.id, + classTypeArgsIncludingOuterClasses + ) + else -> + logger.warnElement( + "Unable to extract specialised declaration that isn't a member of a class", + d + ) + } + } + } + } + + fun getTypeArgumentLabel(arg: IrTypeArgument): TypeResultWithoutSignature { + + fun extractBoundedWildcard( + wildcardKind: Int, + wildcardLabelStr: String, + wildcardShortName: String, + boundLabel: Label + ): Label = + tw.getLabelFor(wildcardLabelStr) { wildcardLabel -> + tw.writeWildcards(wildcardLabel, wildcardShortName, wildcardKind) + tw.writeHasLocation(wildcardLabel, tw.unknownLocation) + tw.getLabelFor("@\"bound;0;{$wildcardLabel}\"") { + tw.writeTypeBounds(it, boundLabel, 0, wildcardLabel) + } + } + + // Note this function doesn't return a signature because type arguments are never + // incorporated into function signatures. + return when (arg) { + is IrStarProjection -> { + val anyTypeLabel = + useType(pluginContext.irBuiltIns.anyType).javaResult.id.cast() + TypeResultWithoutSignature( + extractBoundedWildcard(1, "@\"wildcard;\"", "?", anyTypeLabel), + Unit, + "?" + ) + } + is IrTypeProjection -> { + val boundResults = useType(arg.type, TypeContext.GENERIC_ARGUMENT) + val boundLabel = boundResults.javaResult.id.cast() + + if (arg.variance == Variance.INVARIANT) + boundResults.javaResult.cast().forgetSignature() + else { + val keyPrefix = if (arg.variance == Variance.IN_VARIANCE) "super" else "extends" + val wildcardKind = if (arg.variance == Variance.IN_VARIANCE) 2 else 1 + val wildcardShortName = "? $keyPrefix ${boundResults.javaResult.shortName}" + TypeResultWithoutSignature( + extractBoundedWildcard( + wildcardKind, + "@\"wildcard;$keyPrefix{$boundLabel}\"", + wildcardShortName, + boundLabel + ), + Unit, + wildcardShortName + ) + } + } + else -> { + logger.error("Unexpected type argument.") + extractJavaErrorType().forgetSignature() + } + } + } + + data class ClassLabelResults(val classLabel: String, val shortName: String) + + /** + * This returns the `X` in c's label `@"class;X"`. + * + * `argsIncludingOuterClasses` can be null to describe a raw generic type. For non-generic types + * it will be zero-length list. + */ + private fun getUnquotedClassLabel( + c: IrClass, + argsIncludingOuterClasses: List? + ): ClassLabelResults { + val pkg = c.packageFqName?.asString() ?: "" + val cls = c.name.asString() + val label = + if (c.isAnonymousObject) "{${useAnonymousClass(c).javaResult.id}}" + else + when (val parent = c.parent) { + is IrClass -> { + "${getUnquotedClassLabel(parent, listOf()).classLabel}\$$cls" + } + is IrFunction -> { + "{${useFunction(parent)}}.$cls" + } + is IrField -> { + "{${useField(parent)}}.$cls" + } + else -> { + if (pkg.isEmpty()) cls else "$pkg.$cls" + } + } + + val reorderedArgs = orderTypeArgsLeftToRight(c, argsIncludingOuterClasses) + val typeArgLabels = reorderedArgs?.map { getTypeArgumentLabel(it) } + val typeArgsShortName = + if (typeArgLabels == null) "<>" + else if (typeArgLabels.isEmpty()) "" + else + typeArgLabels.takeLast(c.typeParameters.size).joinToString( + prefix = "<", + postfix = ">", + separator = "," + ) { + it.shortName + } + val shortNamePrefix = if (c.isAnonymousObject) "" else cls + + return ClassLabelResults( + label + (typeArgLabels?.joinToString(separator = "") { ";{${it.id}}" } ?: "<>"), + shortNamePrefix + typeArgsShortName + ) + } + + // `args` can be null to describe a raw generic type. + // For non-generic types it will be zero-length list. + fun getClassLabel( + c: IrClass, + argsIncludingOuterClasses: List? + ): ClassLabelResults { + val unquotedLabel = getUnquotedClassLabel(c, argsIncludingOuterClasses) + return ClassLabelResults("@\"class;${unquotedLabel.classLabel}\"", unquotedLabel.shortName) + } + + fun useClassSource(c: IrClass): Label { + // For source classes, the label doesn't include any type arguments + val classTypeResult = addClassLabel(c, listOf()) + return classTypeResult.id + } + + fun getTypeParameterParentLabel(param: IrTypeParameter) = + param.parent.let { + when (it) { + is IrClass -> useClassSource(it) + is IrFunction -> + (if (this is KotlinFileExtractor) + this.declarationStack + .findOverriddenAttributes(it) + ?.takeUnless { + // When extracting the `static fun f$default(...)` that accompanies + // `fun f(val x: T? = defaultExpr, ...)`, + // `f$default` has no type parameters, and so there is no + // `f$default::T` to refer to. + // We have no good way to extract references to `T` in + // `defaultExpr`, so we just fall back on describing it + // in terms of `f::T`, even though that type variable ought to be + // out of scope here. + attribs -> + attribs.typeParameters?.isEmpty() == true + } + ?.id + else null) ?: useFunction(it, noReplace = true) + else -> { + logger.error("Unexpected type parameter parent $it") + null + } + } + } + + fun getTypeParameterLabel(param: IrTypeParameter): String { + // Use this instead of `useDeclarationParent` so we can use useFunction with noReplace = + // true, + // ensuring that e.g. a method-scoped type variable declared on kotlin.String.transform + // gets + // a different name to the corresponding java.lang.String.transform , even though + // useFunction + // will usually replace references to one function with the other. + val parentLabel = getTypeParameterParentLabel(param) + return "@\"typevar;{$parentLabel};${param.name}\"" + } + + private fun useTypeParameter(param: IrTypeParameter) = + TypeResult( + tw.getLabelFor(getTypeParameterLabel(param)), + useType(eraseTypeParameter(param)).javaResult.signature, + param.name.asString() + ) + + private fun extractModifier(m: String): Label { + val modifierLabel = "@\"modifier;$m\"" + val id: Label = tw.getLabelFor(modifierLabel, { tw.writeModifiers(it, m) }) + return id + } + + fun addModifiers(modifiable: Label, vararg modifiers: String) = + modifiers.forEach { tw.writeHasModifier(modifiable, extractModifier(it)) } + + sealed class ExtractSupertypesMode { + object Unbound : ExtractSupertypesMode() + + object Raw : ExtractSupertypesMode() + + data class Specialised(val typeArgs: List) : ExtractSupertypesMode() + } + + /** + * Extracts the supertypes of class `c`, either the unbound version, raw version or a + * specialisation to particular type arguments, depending on the value of `mode`. `id` is the + * label of this class or class instantiation. + * + * For example, for type `List` if `mode` `Specialised([String])` then we will extract the + * supertypes of `List`, i.e. `Appendable` etc, or if `mode` is `Unbound` we + * will extract `Appendable` where `E` is the type variable declared as `List`. Finally if + * `mode` is `Raw` we will extract the raw type `Appendable`, represented in QL as + * `Appendable<>`. + * + * Argument `inReceiverContext` will be passed onto the `useClassInstance` invocation for each + * supertype. + */ + fun extractClassSupertypes( + c: IrClass, + id: Label, + mode: ExtractSupertypesMode = ExtractSupertypesMode.Unbound, + inReceiverContext: Boolean = false + ) { + extractClassSupertypes( + c.superTypes, + c.typeParameters, + id, + c.isInterfaceLike, + mode, + inReceiverContext + ) + } + + fun extractClassSupertypes( + superTypes: List, + typeParameters: List, + id: Label, + isInterface: Boolean, + mode: ExtractSupertypesMode = ExtractSupertypesMode.Unbound, + inReceiverContext: Boolean = false + ) { + // Note we only need to substitute type args here because it is illegal to directly extend a + // type variable. + // (For example, we can't have `class A : E`, but can have `class A : Comparable`) + val subbedSupertypes = + when (mode) { + is ExtractSupertypesMode.Specialised -> { + superTypes.map { it.substituteTypeArguments(typeParameters, mode.typeArgs) } + } + else -> superTypes + } + + for (t in subbedSupertypes) { + when (t) { + is IrSimpleType -> { + when (val owner = t.classifier.owner) { + is IrClass -> { + val typeArgs = + if (t.arguments.isNotEmpty() && mode is ExtractSupertypesMode.Raw) + null + else t.arguments + val l = + useClassInstance(owner, typeArgs, inReceiverContext).typeResult.id + if (isInterface || !owner.isInterfaceLike) { + tw.writeExtendsReftype(id, l) + } else { + tw.writeImplInterface(id.cast(), l.cast()) + } + } + else -> { + logger.error( + "Unexpected simple type supertype: " + + t.javaClass + + ": " + + t.render() + ) + } + } + } + else -> { + logger.error("Unexpected supertype: " + t.javaClass + ": " + t.render()) + } + } + } + } + + fun useValueDeclaration(d: IrValueDeclaration): Label? = + when (d) { + is IrValueParameter -> useValueParameter(d, null) + is IrVariable -> useVariable(d) + else -> { + logger.error("Unrecognised IrValueDeclaration: " + d.javaClass) + null + } + } + + /** + * Returns `t` with generic types replaced by raw types, and type parameters replaced by their + * first bound. + * + * Note that `Array` is retained (with `T` itself erased) because these are expected to be + * lowered to Java arrays, which are not generic. + */ + fun erase(t: IrType): IrType { + if (t is IrSimpleType) { + val classifier = t.classifier + val owner = classifier.owner + if (owner is IrTypeParameter) { + return eraseTypeParameter(owner) + } + + if (owner is IrClass) { + if (t.isBoxedArray) { + val elementType = t.getArrayElementType(pluginContext.irBuiltIns) + val erasedElementType = erase(elementType) + return owner + .typeWith(erasedElementType) + .codeQlWithHasQuestionMark(t.isNullable()) + } + + return if (t.arguments.isNotEmpty()) + t.addAnnotations(listOf(RawTypeAnnotation.annotationConstructor)) + else t + } + } + return t + } + + private fun eraseTypeParameter(t: IrTypeParameter) = erase(t.superTypes[0]) + + fun getValueParameterLabel(parentId: Label?, idx: Int) = + "@\"params;{$parentId};$idx\"" + + /** + * Gets the label for `vp` in the context of function instance `parent`, or in that of its + * declaring function if `parent` is null. + */ + fun getValueParameterLabel(vp: IrValueParameter, parent: Label?): String { + val declarationParent = vp.parent + val overriddenParentAttributes = + (declarationParent as? IrFunction)?.let { + (this as? KotlinFileExtractor)?.declarationStack?.findOverriddenAttributes(it) + } + val parentId = parent ?: overriddenParentAttributes?.id ?: useDeclarationParentOf(vp, false) + + val idxBase = overriddenParentAttributes?.valueParameters?.indexOf(vp) ?: vp.index + val idxOffset = + if ( + declarationParent is IrFunction && + declarationParent.extensionReceiverParameter != null + ) + // For extension functions increase the index to match what the java extractor sees: + 1 + else 0 + val idx = idxBase + idxOffset + + if (idx < 0) { + // We're not extracting this and this@TYPE parameters of functions: + logger.error("Unexpected negative index for parameter") + } + + return getValueParameterLabel(parentId, idx) + } + + fun useValueParameter( + vp: IrValueParameter, + parent: Label? + ): Label = tw.getLabelFor(getValueParameterLabel(vp, parent)) + + private fun isDirectlyExposableCompanionObjectField(f: IrField) = + f.hasAnnotation(FqName("kotlin.jvm.JvmField")) || + f.correspondingPropertySymbol?.owner?.let { it.isConst || it.isLateinit } ?: false + + private fun getFieldParent(f: IrField) = + f.parentClassOrNull?.let { + if (it.isCompanion && isDirectlyExposableCompanionObjectField(f)) it.parent else null + } ?: f.parent + + fun isDirectlyExposedCompanionObjectField(f: IrField) = getFieldParent(f) != f.parent + + // Gets a field's corresponding property's extension receiver type, if any + fun getExtensionReceiverType(f: IrField) = + f.correspondingPropertySymbol?.owner?.let { + (it.getter ?: it.setter)?.extensionReceiverParameter?.type + } + + fun getFieldLabel(f: IrField): String { + val parentId = useDeclarationParentOf(f, false) + // Distinguish backing fields of properties based on their extension receiver type; + // otherwise two extension properties declared in the same enclosing context will get + // clashing trap labels. These are always private, so we can just make up a label without + // worrying about their names as seen from Java. + val extensionPropertyDiscriminator = + getExtensionReceiverType(f)?.let { "extension;${useType(it).javaResult.id}" } ?: "" + return "@\"field;{$parentId};$extensionPropertyDiscriminator${f.name.asString()}\"" + } + + fun useField(f: IrField): Label = + tw.getLabelFor(getFieldLabel(f)).also { extractFieldLaterIfExternalFileMember(f) } + + fun getPropertyLabel(p: IrProperty): String? { + val parentId = useDeclarationParentOf(p, false) + if (parentId == null) { + return null + } else { + return getPropertyLabel(p, parentId, null) + } + } + + private fun getPropertyLabel( + p: IrProperty, + parentId: Label, + classTypeArgsIncludingOuterClasses: List? + ): String { + val getter = p.getter + val setter = p.setter + + val func = getter ?: setter + val ext = func?.extensionReceiverParameter + + return if (ext == null) { + "@\"property;{$parentId};${p.name.asString()}\"" + } else { + val returnType = + getter?.returnType + ?: setter?.valueParameters?.singleOrNull()?.type + ?: pluginContext.irBuiltIns.unitType + val typeParams = getFunctionTypeParameters(func) + + getFunctionLabel( + p.parent, + parentId, + p.name.asString(), + listOf(), + returnType, + ext.type, + typeParams, + classTypeArgsIncludingOuterClasses, + overridesCollectionsMethod = false, + javaSignature = null, + addParameterWildcardsByDefault = false, + prefix = "property" + ) + } + } + + fun useProperty( + p: IrProperty, + parentId: Label, + classTypeArgsIncludingOuterClasses: List? + ) = + tw.getLabelFor( + getPropertyLabel(p, parentId, classTypeArgsIncludingOuterClasses) + ) { + extractPropertyLaterIfExternalFileMember(p) + extractPrivateSpecialisedDeclaration(p, classTypeArgsIncludingOuterClasses) + } + + fun getEnumEntryLabel(ee: IrEnumEntry): String { + val parentId = useDeclarationParentOf(ee, false) + return "@\"field;{$parentId};${ee.name.asString()}\"" + } + + fun useEnumEntry(ee: IrEnumEntry): Label = tw.getLabelFor(getEnumEntryLabel(ee)) + + fun getTypeAliasLabel(ta: IrTypeAlias): String { + val parentId = useDeclarationParentOf(ta, true) + return "@\"type_alias;{$parentId};${ta.name.asString()}\"" + } + + fun useTypeAlias(ta: IrTypeAlias): Label = + tw.getLabelFor(getTypeAliasLabel(ta)) + + fun useVariable(v: IrVariable): Label { + return tw.getVariableLabelFor(v) + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/Label.kt b/java/kotlin-extractor2/src/main/kotlin/Label.kt new file mode 100644 index 00000000000..65d5fe72461 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/Label.kt @@ -0,0 +1,21 @@ +package com.github.codeql + +/** This represents a label (`#...`) in a TRAP file. */ +interface Label { + fun cast(): Label { + @Suppress("UNCHECKED_CAST") return this as Label + } +} + +/** The label `#i`, e.g. `#123`. Most labels we generate are of this form. */ +class IntLabel(val i: Int) : Label { + override fun toString(): String = "#$i" +} + +/** + * The label `#name`, e.g. `#compilation`. This is used when labels are shared between different + * components (e.g. when both the interceptor and the extractor need to refer to the same label). + */ +class StringLabel(val name: String) : Label { + override fun toString(): String = "#$name" +} diff --git a/java/kotlin-extractor2/src/main/kotlin/LinesOfCode.kt b/java/kotlin-extractor2/src/main/kotlin/LinesOfCode.kt new file mode 100644 index 00000000000..b8cd198922e --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/LinesOfCode.kt @@ -0,0 +1,30 @@ +package com.github.codeql + +import org.jetbrains.kotlin.ir.declarations.* + +class LinesOfCode(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) { + val linesOfCodePSI = LinesOfCodePSI(logger, tw, file) + val linesOfCodeLighterAST = LinesOfCodeLighterAST(logger, tw, file) + + fun linesOfCodeInFile(id: Label) { + val psiExtracted = linesOfCodePSI.linesOfCodeInFile(id) + val lighterASTExtracted = linesOfCodeLighterAST.linesOfCodeInFile(id) + if (psiExtracted && lighterASTExtracted) { + logger.warnElement( + "Both PSI and LighterAST number-of-lines-in-file information for ${file.path}.", + file + ) + } + } + + fun linesOfCodeInDeclaration(d: IrDeclaration, id: Label) { + val psiExtracted = linesOfCodePSI.linesOfCodeInDeclaration(d, id) + val lighterASTExtracted = linesOfCodeLighterAST.linesOfCodeInDeclaration(d, id) + if (psiExtracted && lighterASTExtracted) { + logger.warnElement( + "Both PSI and LighterAST number-of-lines-in-file information for declaration.", + d + ) + } + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/LinesOfCodePSI.kt b/java/kotlin-extractor2/src/main/kotlin/LinesOfCodePSI.kt new file mode 100644 index 00000000000..452ed13200c --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/LinesOfCodePSI.kt @@ -0,0 +1,150 @@ +package com.github.codeql + +import com.github.codeql.utils.versions.getPsi2Ir +import com.intellij.psi.PsiComment +import com.intellij.psi.PsiElement +import com.intellij.psi.PsiWhiteSpace +import org.jetbrains.kotlin.config.KotlinCompilerVersion +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.psi.KtVisitor + +class LinesOfCodePSI(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) { + val psi2Ir = + getPsi2Ir().also { + if (it == null) { + logger.warn( + "Lines of code will not be populated as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})" + ) + } + } + + fun linesOfCodeInFile(id: Label): Boolean { + if (psi2Ir == null) { + return false + } + val ktFile = psi2Ir.getKtFile(file) + if (ktFile == null) { + return false + } + linesOfCodeInPsi(id, ktFile, file) + // Even if linesOfCodeInPsi didn't manage to extract any + // information, if we got as far as calling it then we have + // PSI info for the file + return true + } + + fun linesOfCodeInDeclaration(d: IrDeclaration, id: Label): Boolean { + if (psi2Ir == null) { + return false + } + val p = psi2Ir.findPsiElement(d, file) + if (p == null) { + return false + } + linesOfCodeInPsi(id, p, d) + // Even if linesOfCodeInPsi didn't manage to extract any + // information, if we got as far as calling it then we have + // PSI info for the declaration + return true + } + + private fun linesOfCodeInPsi(id: Label, root: PsiElement, e: IrElement) { + val document = root.getContainingFile().getViewProvider().getDocument() + if (document == null) { + logger.errorElement("Cannot find document for PSI", e) + tw.writeNumlines(id, 0, 0, 0) + return + } + + val rootRange = root.getTextRange() + val rootStartOffset = rootRange.getStartOffset() + val rootEndOffset = rootRange.getEndOffset() + if (rootStartOffset < 0 || rootEndOffset < 0) { + // This is synthetic, or has an invalid location + tw.writeNumlines(id, 0, 0, 0) + return + } + val rootFirstLine = document.getLineNumber(rootStartOffset) + val rootLastLine = document.getLineNumber(rootEndOffset) + if (rootLastLine < rootFirstLine) { + logger.errorElement("PSI ends before it starts", e) + tw.writeNumlines(id, 0, 0, 0) + return + } + val numLines = 1 + rootLastLine - rootFirstLine + val lineContents = Array(numLines) { LineContent() } + + val visitor = + object : KtVisitor() { + override fun visitElement(element: PsiElement) { + val isComment = element is PsiComment + // Comments may include nodes that aren't PsiComments, + // so we don't want to visit them or we'll think they + // are code. + if (!isComment) { + element.acceptChildren(this) + } + + if (element is PsiWhiteSpace) { + return + } + // Leaf nodes are assumed to be tokens, and + // therefore we count any lines that they are on. + // For comments, we actually need to look at the + // outermost node, as the leaves of KDocs don't + // necessarily cover all lines. + if (isComment || element.getChildren().size == 0) { + val range = element.getTextRange() + val startOffset = range.getStartOffset() + val endOffset = range.getEndOffset() + // The PSI doesn't seem to have anything like + // the IR's UNDEFINED_OFFSET and SYNTHETIC_OFFSET, + // but < 0 still seem to represent bad/unknown + // locations. + if (startOffset < 0 || endOffset < 0) { + logger.errorElement("PSI element has negative offset", e) + return + } + if (startOffset > endOffset) { + logger.errorElement("PSI element has negative size", e) + return + } + // We might get e.g. an import list for a file + // with no imports, which claims to have start + // and end offsets of 0. Anything of 0 width + // we therefore just skip. + if (startOffset == endOffset) { + return + } + val firstLine = document.getLineNumber(startOffset) + val lastLine = document.getLineNumber(endOffset) + if (firstLine < rootFirstLine) { + logger.errorElement("PSI element starts before root", e) + return + } else if (lastLine > rootLastLine) { + logger.errorElement("PSI element ends after root", e) + return + } + for (line in firstLine..lastLine) { + val lineContent = lineContents[line - rootFirstLine] + if (isComment) { + lineContent.containsComment = true + } else { + lineContent.containsCode = true + } + } + } + } + } + root.accept(visitor) + val code = lineContents.count { it.containsCode } + val comment = lineContents.count { it.containsComment } + tw.writeNumlines(id, numLines, code, comment) + } + + private class LineContent { + var containsComment = false + var containsCode = false + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/MetaAnnotationSupport.kt b/java/kotlin-extractor2/src/main/kotlin/MetaAnnotationSupport.kt new file mode 100644 index 00000000000..2dc7382bf16 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/MetaAnnotationSupport.kt @@ -0,0 +1,513 @@ +package com.github.codeql + +import com.github.codeql.utils.versions.copyParameterToFunction +import com.github.codeql.utils.versions.createImplicitParameterDeclarationWithWrappedDescriptor +import com.github.codeql.utils.versions.getAnnotationType +import java.lang.annotation.ElementType +import java.util.HashSet +import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext +import org.jetbrains.kotlin.builtins.StandardNames +import org.jetbrains.kotlin.config.JvmTarget +import org.jetbrains.kotlin.descriptors.ClassKind +import org.jetbrains.kotlin.descriptors.annotations.KotlinRetention +import org.jetbrains.kotlin.descriptors.annotations.KotlinTarget +import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET +import org.jetbrains.kotlin.ir.builders.declarations.addConstructor +import org.jetbrains.kotlin.ir.builders.declarations.addGetter +import org.jetbrains.kotlin.ir.builders.declarations.addProperty +import org.jetbrains.kotlin.ir.builders.declarations.addValueParameter +import org.jetbrains.kotlin.ir.builders.declarations.buildClass +import org.jetbrains.kotlin.ir.builders.declarations.buildField +import org.jetbrains.kotlin.ir.declarations.IrClass +import org.jetbrains.kotlin.ir.declarations.IrConstructor +import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin +import org.jetbrains.kotlin.ir.declarations.IrEnumEntry +import org.jetbrains.kotlin.ir.declarations.IrProperty +import org.jetbrains.kotlin.ir.expressions.IrClassReference +import org.jetbrains.kotlin.ir.expressions.IrConstructorCall +import org.jetbrains.kotlin.ir.expressions.IrGetEnumValue +import org.jetbrains.kotlin.ir.expressions.IrVararg +import org.jetbrains.kotlin.ir.expressions.impl.IrClassReferenceImpl +import org.jetbrains.kotlin.ir.expressions.impl.IrConstructorCallImpl +import org.jetbrains.kotlin.ir.expressions.impl.IrGetEnumValueImpl +import org.jetbrains.kotlin.ir.expressions.impl.IrGetFieldImpl +import org.jetbrains.kotlin.ir.expressions.impl.IrGetValueImpl +import org.jetbrains.kotlin.ir.expressions.impl.IrReturnImpl +import org.jetbrains.kotlin.ir.expressions.impl.IrVarargImpl +import org.jetbrains.kotlin.ir.symbols.IrClassSymbol +import org.jetbrains.kotlin.ir.types.typeWith +import org.jetbrains.kotlin.ir.util.constructedClass +import org.jetbrains.kotlin.ir.util.constructors +import org.jetbrains.kotlin.ir.util.deepCopyWithSymbols +import org.jetbrains.kotlin.ir.util.defaultType +import org.jetbrains.kotlin.ir.util.fqNameWhenAvailable +import org.jetbrains.kotlin.ir.util.getAnnotation +import org.jetbrains.kotlin.ir.util.hasAnnotation +import org.jetbrains.kotlin.ir.util.hasEqualFqName +import org.jetbrains.kotlin.ir.util.parentAsClass +import org.jetbrains.kotlin.ir.util.primaryConstructor +import org.jetbrains.kotlin.load.java.JvmAnnotationNames +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.name.Name +import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull + +class MetaAnnotationSupport( + private val logger: FileLogger, + private val pluginContext: IrPluginContext, + private val extractor: KotlinFileExtractor +) { + + // Taken from AdditionalIrUtils.kt (not available in Kotlin < 1.6) + private val IrConstructorCall.annotationClass + get() = this.symbol.owner.constructedClass + + // Taken from AdditionalIrUtils.kt (not available in Kotlin < 1.6) + private fun IrConstructorCall.isAnnotationWithEqualFqName(fqName: FqName): Boolean = + annotationClass.hasEqualFqName(fqName) + + // Adapted from RepeatedAnnotationLowering.kt + fun groupRepeatableAnnotations(annotations: List): List { + if (annotations.size < 2) return annotations + + val annotationsByClass = annotations.groupByTo(mutableMapOf()) { it.annotationClass } + if (annotationsByClass.values.none { it.size > 1 }) return annotations + + val result = mutableListOf() + for (annotation in annotations) { + val annotationClass = annotation.annotationClass + val grouped = annotationsByClass.remove(annotationClass) ?: continue + if (grouped.size < 2) { + result.add(grouped.single()) + continue + } + + val containerClass = getOrCreateContainerClass(annotationClass) + if (containerClass != null) + wrapAnnotationEntriesInContainer(annotationClass, containerClass, grouped)?.let { + result.add(it) + } + else logger.warnElement("Failed to find an annotation container class", annotationClass) + } + return result + } + + // Adapted from RepeatedAnnotationLowering.kt + private fun getOrCreateContainerClass(annotationClass: IrClass): IrClass? { + val metaAnnotations = annotationClass.annotations + val jvmRepeatable = + metaAnnotations.find { + it.symbol.owner.parentAsClass.fqNameWhenAvailable == + JvmAnnotationNames.REPEATABLE_ANNOTATION + } + return if (jvmRepeatable != null) { + ((jvmRepeatable.getValueArgument(0) as? IrClassReference)?.symbol as? IrClassSymbol) + ?.owner + } else { + getOrCreateSyntheticRepeatableAnnotationContainer(annotationClass) + } + } + + // Adapted from RepeatedAnnotationLowering.kt + private fun wrapAnnotationEntriesInContainer( + annotationClass: IrClass, + containerClass: IrClass, + entries: List + ): IrConstructorCall? { + val annotationType = annotationClass.typeWith() + val containerConstructor = containerClass.primaryConstructor + if (containerConstructor == null) { + logger.warnElement( + "Expected container class to have a primary constructor", + containerClass + ) + return null + } else { + return IrConstructorCallImpl.fromSymbolOwner( + containerClass.defaultType, + containerConstructor.symbol + ) + .apply { + putValueArgument( + 0, + IrVarargImpl( + UNDEFINED_OFFSET, + UNDEFINED_OFFSET, + pluginContext.irBuiltIns.arrayClass.typeWith(annotationType), + annotationType, + entries + ) + ) + } + } + } + + // Taken from AdditionalClassAnnotationLowering.kt + private fun getApplicableTargetSet(c: IrClass): Set? { + val targetEntry = c.getAnnotation(StandardNames.FqNames.target) ?: return null + return loadAnnotationTargets(targetEntry) + } + + // Taken from AdditionalClassAnnotationLowering.kt + private fun loadAnnotationTargets(targetEntry: IrConstructorCall): Set? { + val valueArgument = targetEntry.getValueArgument(0) as? IrVararg ?: return null + return valueArgument.elements + .filterIsInstance() + .mapNotNull { KotlinTarget.valueOrNull(it.symbol.owner.name.asString()) } + .toSet() + } + + private val javaAnnotationTargetElementType by lazy { + extractor.referenceExternalClass("java.lang.annotation.ElementType") + } + + private val javaAnnotationTarget by lazy { + extractor.referenceExternalClass("java.lang.annotation.Target") + } + + private fun findEnumEntry(c: IrClass, name: String) = + c.declarations.filterIsInstance().find { it.name.asString() == name } + + // Adapted from JvmSymbols.kt + private val jvm6TargetMap by lazy { + javaAnnotationTargetElementType?.let { + val etMethod = findEnumEntry(it, "METHOD") + mapOf( + KotlinTarget.CLASS to findEnumEntry(it, "TYPE"), + KotlinTarget.ANNOTATION_CLASS to findEnumEntry(it, "ANNOTATION_TYPE"), + KotlinTarget.CONSTRUCTOR to findEnumEntry(it, "CONSTRUCTOR"), + KotlinTarget.LOCAL_VARIABLE to findEnumEntry(it, "LOCAL_VARIABLE"), + KotlinTarget.FUNCTION to etMethod, + KotlinTarget.PROPERTY_GETTER to etMethod, + KotlinTarget.PROPERTY_SETTER to etMethod, + KotlinTarget.FIELD to findEnumEntry(it, "FIELD"), + KotlinTarget.VALUE_PARAMETER to findEnumEntry(it, "PARAMETER") + ) + } + } + + // Adapted from JvmSymbols.kt + private val jvm8TargetMap by lazy { + javaAnnotationTargetElementType?.let { + jvm6TargetMap?.let { j6Map -> + j6Map + + mapOf( + KotlinTarget.TYPE_PARAMETER to findEnumEntry(it, "TYPE_PARAMETER"), + KotlinTarget.TYPE to findEnumEntry(it, "TYPE_USE") + ) + } + } + } + + private fun getAnnotationTargetMap() = + if (pluginContext.platform?.any { it.targetPlatformVersion == JvmTarget.JVM_1_6 } == true) + jvm6TargetMap + else jvm8TargetMap + + // Adapted from AdditionalClassAnnotationLowering.kt + private fun generateTargetAnnotation(c: IrClass): IrConstructorCall? { + if (c.hasAnnotation(JvmAnnotationNames.TARGET_ANNOTATION)) return null + val elementType = javaAnnotationTargetElementType ?: return null + val targetType = javaAnnotationTarget ?: return null + val targetConstructor = + targetType.declarations.firstIsInstanceOrNull() ?: return null + val targets = getApplicableTargetSet(c) ?: return null + val annotationTargetMap = getAnnotationTargetMap() ?: return null + + val javaTargets = + targets + .mapNotNullTo(HashSet()) { annotationTargetMap[it] } + .sortedBy { ElementType.valueOf(it.symbol.owner.name.asString()) } + val vararg = + IrVarargImpl( + UNDEFINED_OFFSET, + UNDEFINED_OFFSET, + type = pluginContext.irBuiltIns.arrayClass.typeWith(elementType.defaultType), + varargElementType = elementType.defaultType + ) + for (target in javaTargets) { + vararg.elements.add( + IrGetEnumValueImpl( + UNDEFINED_OFFSET, + UNDEFINED_OFFSET, + elementType.defaultType, + target.symbol + ) + ) + } + + return IrConstructorCallImpl.fromSymbolOwner( + UNDEFINED_OFFSET, + UNDEFINED_OFFSET, + targetConstructor.returnType, + targetConstructor.symbol, + 0 + ) + .apply { putValueArgument(0, vararg) } + } + + private val javaAnnotationRetention by lazy { + extractor.referenceExternalClass("java.lang.annotation.Retention") + } + private val javaAnnotationRetentionPolicy by lazy { + extractor.referenceExternalClass("java.lang.annotation.RetentionPolicy") + } + private val javaAnnotationRetentionPolicyRuntime by lazy { + javaAnnotationRetentionPolicy?.let { findEnumEntry(it, "RUNTIME") } + } + + private val annotationRetentionMap by lazy { + javaAnnotationRetentionPolicy?.let { + mapOf( + KotlinRetention.SOURCE to findEnumEntry(it, "SOURCE"), + KotlinRetention.BINARY to findEnumEntry(it, "CLASS"), + KotlinRetention.RUNTIME to javaAnnotationRetentionPolicyRuntime + ) + } + } + + // Taken from AnnotationCodegen.kt (not available in Kotlin < 1.6.20) + private fun IrClass.getAnnotationRetention(): KotlinRetention? { + val retentionArgument = + getAnnotation(StandardNames.FqNames.retention)?.getValueArgument(0) as? IrGetEnumValue + ?: return null + val retentionArgumentValue = retentionArgument.symbol.owner + return KotlinRetention.valueOf(retentionArgumentValue.name.asString()) + } + + // Taken from AdditionalClassAnnotationLowering.kt + private fun generateRetentionAnnotation(irClass: IrClass): IrConstructorCall? { + if (irClass.hasAnnotation(JvmAnnotationNames.RETENTION_ANNOTATION)) return null + val retentionMap = annotationRetentionMap ?: return null + val kotlinRetentionPolicy = irClass.getAnnotationRetention() + val javaRetentionPolicy = + kotlinRetentionPolicy?.let { retentionMap[it] } + ?: javaAnnotationRetentionPolicyRuntime + ?: return null + val retentionPolicyType = javaAnnotationRetentionPolicy ?: return null + val retentionType = javaAnnotationRetention ?: return null + val targetConstructor = + retentionType.declarations.firstIsInstanceOrNull() ?: return null + + return IrConstructorCallImpl.fromSymbolOwner( + UNDEFINED_OFFSET, + UNDEFINED_OFFSET, + targetConstructor.returnType, + targetConstructor.symbol, + 0 + ) + .apply { + putValueArgument( + 0, + IrGetEnumValueImpl( + UNDEFINED_OFFSET, + UNDEFINED_OFFSET, + retentionPolicyType.defaultType, + javaRetentionPolicy.symbol + ) + ) + } + } + + private val javaAnnotationRepeatable by lazy { + extractor.referenceExternalClass("java.lang.annotation.Repeatable") + } + private val kotlinAnnotationRepeatableContainer by lazy { + extractor.referenceExternalClass("kotlin.jvm.internal.RepeatableContainer") + } + + // Taken from declarationBuilders.kt (not available in Kotlin < 1.6): + private fun addDefaultGetter(p: IrProperty, parentClass: IrClass) { + val field = + p.backingField + ?: run { + logger.warnElement("Expected property to have a backing field", p) + return + } + p.addGetter { + origin = IrDeclarationOrigin.DEFAULT_PROPERTY_ACCESSOR + returnType = field.type + } + .apply { + val thisReceiever = + parentClass.thisReceiver + ?: run { + logger.warnElement( + "Expected property's parent class to have a receiver parameter", + parentClass + ) + return + } + val newParam = copyParameterToFunction(thisReceiever, this) + dispatchReceiverParameter = newParam + body = + factory + .createBlockBody(UNDEFINED_OFFSET, UNDEFINED_OFFSET) + .apply({ + this.statements.add( + IrReturnImpl( + UNDEFINED_OFFSET, + UNDEFINED_OFFSET, + pluginContext.irBuiltIns.nothingType, + symbol, + IrGetFieldImpl( + UNDEFINED_OFFSET, + UNDEFINED_OFFSET, + field.symbol, + field.type, + IrGetValueImpl( + UNDEFINED_OFFSET, + UNDEFINED_OFFSET, + newParam.type, + newParam.symbol + ) + ) + ) + ) + }) + } + } + + // Taken from JvmCachedDeclarations.kt + private fun getOrCreateSyntheticRepeatableAnnotationContainer(annotationClass: IrClass) = + extractor.globalExtensionState.syntheticRepeatableAnnotationContainers.getOrPut( + annotationClass + ) { + val containerClass = + pluginContext.irFactory + .buildClass { + kind = ClassKind.ANNOTATION_CLASS + name = Name.identifier("Container") + } + .apply { + createImplicitParameterDeclarationWithWrappedDescriptor() + parent = annotationClass + superTypes = listOf(getAnnotationType(pluginContext)) + } + + val propertyName = Name.identifier("value") + val propertyType = + pluginContext.irBuiltIns.arrayClass.typeWith(annotationClass.typeWith()) + + containerClass + .addConstructor { isPrimary = true } + .apply { addValueParameter(propertyName.identifier, propertyType) } + + containerClass + .addProperty { name = propertyName } + .apply property@{ + backingField = + pluginContext.irFactory + .buildField { + name = propertyName + type = propertyType + } + .apply { + parent = containerClass + correspondingPropertySymbol = this@property.symbol + } + addDefaultGetter(this, containerClass) + } + + val repeatableContainerAnnotation = + kotlinAnnotationRepeatableContainer?.constructors?.single() + + containerClass.annotations = + annotationClass.annotations + .filter { + it.isAnnotationWithEqualFqName(StandardNames.FqNames.retention) || + it.isAnnotationWithEqualFqName(StandardNames.FqNames.target) + } + .map { it.deepCopyWithSymbols(containerClass) } + + listOfNotNull( + repeatableContainerAnnotation?.let { + IrConstructorCallImpl.fromSymbolOwner( + UNDEFINED_OFFSET, + UNDEFINED_OFFSET, + it.returnType, + it.symbol, + 0 + ) + } + ) + + containerClass + } + + // Adapted from AdditionalClassAnnotationLowering.kt + private fun generateRepeatableAnnotation( + irClass: IrClass, + extractAnnotationTypeAccesses: Boolean + ): IrConstructorCall? { + if ( + !irClass.hasAnnotation(StandardNames.FqNames.repeatable) || + irClass.hasAnnotation(JvmAnnotationNames.REPEATABLE_ANNOTATION) + ) + return null + + val repeatableConstructor = + javaAnnotationRepeatable?.declarations?.firstIsInstanceOrNull() + ?: return null + + val containerClass = getOrCreateSyntheticRepeatableAnnotationContainer(irClass) + // Whenever a repeatable annotation with a Kotlin-synthesised container is extracted, + // extract the synthetic container to the same trap file. + extractor.extractClassSource( + containerClass, + extractDeclarations = true, + extractStaticInitializer = true, + extractPrivateMembers = true, + extractFunctionBodies = extractAnnotationTypeAccesses + ) + + val containerReference = + IrClassReferenceImpl( + UNDEFINED_OFFSET, + UNDEFINED_OFFSET, + pluginContext.irBuiltIns.kClassClass.typeWith(containerClass.defaultType), + containerClass.symbol, + containerClass.defaultType + ) + return IrConstructorCallImpl.fromSymbolOwner( + UNDEFINED_OFFSET, + UNDEFINED_OFFSET, + repeatableConstructor.returnType, + repeatableConstructor.symbol, + 0 + ) + .apply { putValueArgument(0, containerReference) } + } + + private val javaAnnotationDocumented by lazy { + extractor.referenceExternalClass("java.lang.annotation.Documented") + } + + // Taken from AdditionalClassAnnotationLowering.kt + private fun generateDocumentedAnnotation(irClass: IrClass): IrConstructorCall? { + if ( + !irClass.hasAnnotation(StandardNames.FqNames.mustBeDocumented) || + irClass.hasAnnotation(JvmAnnotationNames.DOCUMENTED_ANNOTATION) + ) + return null + + val documentedConstructor = + javaAnnotationDocumented?.declarations?.firstIsInstanceOrNull() + ?: return null + + return IrConstructorCallImpl.fromSymbolOwner( + UNDEFINED_OFFSET, + UNDEFINED_OFFSET, + documentedConstructor.returnType, + documentedConstructor.symbol, + 0 + ) + } + + fun generateJavaMetaAnnotations(c: IrClass, extractAnnotationTypeAccesses: Boolean) = + // This is essentially AdditionalClassAnnotationLowering adapted to run outside the backend. + listOfNotNull( + generateTargetAnnotation(c), + generateRetentionAnnotation(c), + generateRepeatableAnnotation(c, extractAnnotationTypeAccesses), + generateDocumentedAnnotation(c) + ) +} diff --git a/java/kotlin-extractor2/src/main/kotlin/PrimitiveTypeInfo.kt b/java/kotlin-extractor2/src/main/kotlin/PrimitiveTypeInfo.kt new file mode 100644 index 00000000000..225fe8509dd --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/PrimitiveTypeInfo.kt @@ -0,0 +1,105 @@ +package com.github.codeql + +import com.github.codeql.utils.* +import com.github.codeql.utils.versions.* +import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext +import org.jetbrains.kotlin.builtins.StandardNames +import org.jetbrains.kotlin.ir.declarations.IrClass +import org.jetbrains.kotlin.ir.declarations.IrPackageFragment +import org.jetbrains.kotlin.ir.types.IrSimpleType +import org.jetbrains.kotlin.ir.types.classOrNull + +class PrimitiveTypeMapping(val logger: Logger, val pluginContext: IrPluginContext) { + fun getPrimitiveInfo(s: IrSimpleType) = + s.classOrNull?.let { + if ( + (it.owner.parent as? IrPackageFragment)?.packageFqName == + StandardNames.BUILT_INS_PACKAGE_FQ_NAME + ) + mapping[it.owner.name] + else null + } + + data class PrimitiveTypeInfo( + val primitiveName: String?, + val otherIsPrimitive: Boolean, + val javaClass: IrClass, + val kotlinPackageName: String, + val kotlinClassName: String + ) + + private fun findClass(fqName: String, fallback: IrClass): IrClass { + val symbol = getClassByFqName(pluginContext, fqName) + if (symbol == null) { + logger.warn("Can't find $fqName") + // Do the best we can + return fallback + } else { + return symbol.owner + } + } + + private val mapping = + { + val kotlinByte = pluginContext.irBuiltIns.byteClass.owner + val javaLangByte = findClass("java.lang.Byte", kotlinByte) + val kotlinShort = pluginContext.irBuiltIns.shortClass.owner + val javaLangShort = findClass("java.lang.Short", kotlinShort) + val kotlinInt = pluginContext.irBuiltIns.intClass.owner + val javaLangInteger = findClass("java.lang.Integer", kotlinInt) + val kotlinLong = pluginContext.irBuiltIns.longClass.owner + val javaLangLong = findClass("java.lang.Long", kotlinLong) + + val kotlinUByte = findClass("kotlin.UByte", kotlinByte) + val kotlinUShort = findClass("kotlin.UShort", kotlinShort) + val kotlinUInt = findClass("kotlin.UInt", kotlinInt) + val kotlinULong = findClass("kotlin.ULong", kotlinLong) + + val kotlinDouble = pluginContext.irBuiltIns.doubleClass.owner + val javaLangDouble = findClass("java.lang.Double", kotlinDouble) + val kotlinFloat = pluginContext.irBuiltIns.floatClass.owner + val javaLangFloat = findClass("java.lang.Float", kotlinFloat) + + val kotlinBoolean = pluginContext.irBuiltIns.booleanClass.owner + val javaLangBoolean = findClass("java.lang.Boolean", kotlinBoolean) + + val kotlinChar = pluginContext.irBuiltIns.charClass.owner + val javaLangCharacter = findClass("java.lang.Character", kotlinChar) + + val kotlinUnit = pluginContext.irBuiltIns.unitClass.owner + + val kotlinNothing = pluginContext.irBuiltIns.nothingClass.owner + val javaLangVoid = findClass("java.lang.Void", kotlinNothing) + + mapOf( + StandardNames.FqNames._byte.shortName() to + PrimitiveTypeInfo("byte", true, javaLangByte, "kotlin", "Byte"), + StandardNames.FqNames._short.shortName() to + PrimitiveTypeInfo("short", true, javaLangShort, "kotlin", "Short"), + StandardNames.FqNames._int.shortName() to + PrimitiveTypeInfo("int", true, javaLangInteger, "kotlin", "Int"), + StandardNames.FqNames._long.shortName() to + PrimitiveTypeInfo("long", true, javaLangLong, "kotlin", "Long"), + StandardNames.FqNames.uByteFqName.shortName() to + PrimitiveTypeInfo("byte", true, kotlinUByte, "kotlin", "UByte"), + StandardNames.FqNames.uShortFqName.shortName() to + PrimitiveTypeInfo("short", true, kotlinUShort, "kotlin", "UShort"), + StandardNames.FqNames.uIntFqName.shortName() to + PrimitiveTypeInfo("int", true, kotlinUInt, "kotlin", "UInt"), + StandardNames.FqNames.uLongFqName.shortName() to + PrimitiveTypeInfo("long", true, kotlinULong, "kotlin", "ULong"), + StandardNames.FqNames._double.shortName() to + PrimitiveTypeInfo("double", true, javaLangDouble, "kotlin", "Double"), + StandardNames.FqNames._float.shortName() to + PrimitiveTypeInfo("float", true, javaLangFloat, "kotlin", "Float"), + StandardNames.FqNames._boolean.shortName() to + PrimitiveTypeInfo("boolean", true, javaLangBoolean, "kotlin", "Boolean"), + StandardNames.FqNames._char.shortName() to + PrimitiveTypeInfo("char", true, javaLangCharacter, "kotlin", "Char"), + StandardNames.FqNames.unit.shortName() to + PrimitiveTypeInfo("void", false, kotlinUnit, "kotlin", "Unit"), + StandardNames.FqNames.nothing.shortName() to + PrimitiveTypeInfo(null, true, javaLangVoid, "kotlin", "Nothing"), + ) + }() +} diff --git a/java/kotlin-extractor2/src/main/kotlin/TrapWriter.kt b/java/kotlin-extractor2/src/main/kotlin/TrapWriter.kt new file mode 100644 index 00000000000..da04893b4d0 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/TrapWriter.kt @@ -0,0 +1,464 @@ +package com.github.codeql + +import com.github.codeql.KotlinUsesExtractor.LocallyVisibleFunctionLabels +import com.semmle.extractor.java.PopulateFile +import com.semmle.util.unicode.UTF8Util +import java.io.BufferedWriter +import java.io.File +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET +import org.jetbrains.kotlin.ir.declarations.IrClass +import org.jetbrains.kotlin.ir.declarations.IrFile +import org.jetbrains.kotlin.ir.declarations.IrFunction +import org.jetbrains.kotlin.ir.declarations.IrVariable +import org.jetbrains.kotlin.ir.declarations.path +import org.jetbrains.kotlin.ir.expressions.IrCall +import org.jetbrains.kotlin.ir.util.SYNTHETIC_OFFSET + +/** + * Each `.trap` file has a `TrapLabelManager` while we are writing it. It provides fresh TRAP label + * names, and maintains a mapping from keys (`@"..."`) to labels. + */ +class TrapLabelManager { + /** The next integer to use as a label name. */ + private var nextInt: Int = 100 + + /** Returns a fresh label. */ + fun getFreshLabel(): Label { + return IntLabel(nextInt++) + } + + /** A mapping from a key (`@"..."`) to the label defined to be that key, if any. */ + val labelMapping: MutableMap> = mutableMapOf>() + + val anonymousTypeMapping: MutableMap = mutableMapOf() + + val locallyVisibleFunctionLabelMapping: MutableMap = + mutableMapOf() + + /** + * The set of labels of generic specialisations that we have extracted in this TRAP file. We + * can't easily avoid duplication between TRAP files, as the labels contain references to other + * labels, so we just accept this duplication. + */ + val genericSpecialisationsExtracted = HashSet() + + /** + * Sometimes, when we extract a file class we don't have the IrFile for it, so we are not able + * to give it a location. This means that the location is written outside of the label creation. + * This allows us to keep track of whether we've written the location already in this TRAP file, + * to avoid duplication. + */ + val fileClassLocationsExtracted = HashSet() +} + +/** + * A `TrapWriter` is used to write TRAP to a particular TRAP file. There may be multiple + * `TrapWriter`s for the same file, as different instances will have different additional state, but + * they must all share the same `TrapLabelManager` and `BufferedWriter`. + */ +// TODO lm was `protected` before anonymousTypeMapping and locallyVisibleFunctionLabelMapping moved +// into it. Should we re-protect it and provide accessors? +abstract class TrapWriter( + protected val loggerBase: LoggerBase, + val lm: TrapLabelManager, + private val bw: BufferedWriter +) { + abstract fun getDiagnosticTrapWriter(): DiagnosticTrapWriter + + /** + * Returns the label that is defined to be the given key, if such a label exists, and `null` + * otherwise. Most users will want to use `getLabelFor` instead, which allows non-existent + * labels to be initialised. + */ + fun getExistingLabelFor(key: String): Label? { + return lm.labelMapping.get(key)?.cast() + } + /** + * Returns the label for the given key, if one exists. Otherwise, a fresh label is bound to that + * key, `initialise` is run on it, and it is returned. + */ + @JvmOverloads // Needed so Java can call a method with an optional argument + fun getLabelFor(key: String, initialise: (Label) -> Unit = {}): Label { + val maybeLabel: Label? = getExistingLabelFor(key) + if (maybeLabel == null) { + val label: Label = lm.getFreshLabel() + lm.labelMapping.put(key, label) + writeTrap("$label = $key\n") + initialise(label) + return label + } else { + return maybeLabel + } + } + + /** Returns a label for a fresh ID (i.e. a new label bound to `*`). */ + fun getFreshIdLabel(): Label { + val label: Label = lm.getFreshLabel() + writeTrap("$label = *\n") + return label + } + + /** + * It is not easy to assign keys to local variables, so they get given `*` IDs. However, the + * same variable may be referred to from distant places in the IR, so we need a way to find out + * which label is used for a given local variable. This information is stored in this mapping. + */ + private val variableLabelMapping: MutableMap> = + mutableMapOf>() + /** This returns the label used for a local variable, creating one if none currently exists. */ + fun getVariableLabelFor(v: IrVariable): Label { + val maybeLabel = variableLabelMapping.get(v) + if (maybeLabel == null) { + val label = getFreshIdLabel() + variableLabelMapping.put(v, label) + return label + } else { + return maybeLabel + } + } + + fun getExistingVariableLabelFor(v: IrVariable): Label? { + return variableLabelMapping.get(v) + } + + /** + * This returns a label for the location described by its arguments. Typically users will not + * want to call this directly, but instead use `unknownLocation`, or overloads of this defined + * by subclasses. + */ + fun getLocation( + fileId: Label, + startLine: Int, + startColumn: Int, + endLine: Int, + endColumn: Int + ): Label { + return getLabelFor("@\"loc,{$fileId},$startLine,$startColumn,$endLine,$endColumn\"") { + writeLocations_default(it, fileId, startLine, startColumn, endLine, endColumn) + } + } + + /** + * The label for the 'unknown' file ID. Users will want to use `unknownLocation` instead. This + * is lazy, as we don't want to define it in a TRAP file unless the TRAP file actually contains + * something in the 'unknown' file. + */ + protected val unknownFileId: Label by lazy { + val unknownFileLabel = "@\";sourcefile\"" + getLabelFor(unknownFileLabel, { writeFiles(it, "") }) + } + + /** + * The label for the 'unknown' location. This is lazy, as we don't want to define it in a TRAP + * file unless the TRAP file actually contains something with an 'unknown' location. + */ + val unknownLocation: Label by lazy { getWholeFileLocation(unknownFileId) } + + /** + * Returns the label for the file `filePath`. If `populateFileTables` is true, then this also + * adds rows to the `files` and `folders` tables for this file. + */ + fun mkFileId(filePath: String, populateFileTables: Boolean): Label { + // If a file is in a jar, then the Kotlin compiler gives + // `!/` as its path. We need to split + // it as appropriate, to make the right file ID. + val populateFile = PopulateFile(this) + val splitFilePath = filePath.split("!/") + if (splitFilePath.size == 1) { + return populateFile.getFileLabel(File(filePath), populateFileTables) + } else { + return populateFile.getFileInJarLabel( + File(splitFilePath.get(0)), + splitFilePath.get(1), + populateFileTables + ) + } + } + + /** + * If you have an ID for a file, then this gets a label for the location representing the whole + * of that file. + */ + fun getWholeFileLocation(fileId: Label): Label { + return getLocation(fileId, 0, 0, 0, 0) + } + + /** + * Write a raw string into the TRAP file. Users should call one of the wrapper functions + * instead. + */ + fun writeTrap(trap: String) { + bw.write(trap) + } + + /** Write a comment into the TRAP file. */ + fun writeComment(comment: String) { + writeTrap("// ${comment.replace("\n", "\n// ")}\n") + } + + /** Flush the TRAP file. */ + fun flush() { + bw.flush() + } + + /** + * Escape a string so that it can be used in a TRAP string literal, i.e. with `"` escaped as + * `""`. + */ + fun escapeTrapString(str: String) = str.replace("\"", "\"\"") + + /** TRAP string literals are limited to 1 megabyte. */ + private val MAX_STRLEN = 1.shl(20) + + /** + * Truncate a string, if necessary, so that it can be used as a TRAP string literal. TRAP string + * literals are limited to 1 megabyte. + */ + fun truncateString(str: String): String { + val len = str.length + val newLen = UTF8Util.encodablePrefixLength(str, MAX_STRLEN) + if (newLen < len) { + loggerBase.warn( + this.getDiagnosticTrapWriter(), + "Truncated string of length $len", + "Truncated string of length $len, starting '${str.take(100)}', ending '${str.takeLast(100)}'" + ) + return str.take(newLen) + } else { + return str + } + } + + /** + * Gets a FileTrapWriter like this one (using the same label manager, writer etc), but using the + * given `filePath` for locations. + */ + fun makeFileTrapWriter(filePath: String, populateFileTables: Boolean) = + FileTrapWriter( + loggerBase, + lm, + bw, + this.getDiagnosticTrapWriter(), + filePath, + populateFileTables + ) + + /** + * Gets a FileTrapWriter like this one (using the same label manager, writer etc), but using the + * given `IrFile` for locations. + */ + fun makeSourceFileTrapWriter(file: IrFile, populateFileTables: Boolean) = + SourceFileTrapWriter( + loggerBase, + lm, + bw, + this.getDiagnosticTrapWriter(), + file, + populateFileTables + ) +} + +/** A `PlainTrapWriter` has no additional context of its own. */ +class PlainTrapWriter( + loggerBase: LoggerBase, + lm: TrapLabelManager, + bw: BufferedWriter, + val dtw: DiagnosticTrapWriter +) : TrapWriter(loggerBase, lm, bw) { + override fun getDiagnosticTrapWriter(): DiagnosticTrapWriter { + return dtw + } +} + +/** + * A `DiagnosticTrapWriter` is a TrapWriter that diagnostics can be written to; i.e. it has + * the #compilation label defined. In practice, this means that it is a TrapWriter for the + * invocation TRAP file. + */ +class DiagnosticTrapWriter(loggerBase: LoggerBase, lm: TrapLabelManager, bw: BufferedWriter) : + TrapWriter(loggerBase, lm, bw) { + override fun getDiagnosticTrapWriter(): DiagnosticTrapWriter { + return this + } +} + +/** + * A `FileTrapWriter` is used when we know which file we are extracting entities from, so we can at + * least give the right file as a location. + * + * An ID for the file will be created, and if `populateFileTables` is true then we will also add + * rows to the `files` and `folders` tables for it. + */ +open class FileTrapWriter( + loggerBase: LoggerBase, + lm: TrapLabelManager, + bw: BufferedWriter, + val dtw: DiagnosticTrapWriter, + val filePath: String, + populateFileTables: Boolean +) : TrapWriter(loggerBase, lm, bw) { + + /** The ID for the file that we are extracting from. */ + val fileId = mkFileId(filePath, populateFileTables) + + override fun getDiagnosticTrapWriter(): DiagnosticTrapWriter { + return dtw + } + + private fun offsetMinOf(default: Int, vararg options: Int?): Int { + if (default == UNDEFINED_OFFSET || default == SYNTHETIC_OFFSET) { + return default + } + + var currentMin = default + for (option in options) { + if ( + option != null && + option != UNDEFINED_OFFSET && + option != SYNTHETIC_OFFSET && + option < currentMin + ) { + currentMin = option + } + } + + return currentMin + } + + private fun getStartOffset(e: IrElement): Int { + return when (e) { + is IrCall -> { + // Calls have incorrect startOffset, so we adjust them: + val dr = e.dispatchReceiver?.let { getStartOffset(it) } + val er = e.extensionReceiver?.let { getStartOffset(it) } + offsetMinOf(e.startOffset, dr, er) + } + else -> e.startOffset + } + } + + private fun getEndOffset(e: IrElement): Int { + return e.endOffset + } + + /** Gets a label for the location of `e`. */ + fun getLocation(e: IrElement): Label { + return getLocation(getStartOffset(e), getEndOffset(e)) + } + /** + * Gets a label for the location corresponding to `startOffset` and `endOffset` within this + * file. + */ + open fun getLocation(startOffset: Int, endOffset: Int): Label { + // We don't have a FileEntry to look up the offsets in, so all + // we can do is return a whole-file location. + return getWholeFileLocation() + } + /** + * Gets the location of `e` as a human-readable string. Only used in log messages and exception + * messages. + */ + open fun getLocationString(e: IrElement): String { + // We don't have a FileEntry to look up the offsets in, so all + // we can do is return a whole-file location. We omit the + // `:0:0:0:0` so that it is easy to distinguish from a location + // where we have actually determined the start/end lines/columns + // to be 0. + return "file://$filePath" + } + /** Gets a label for the location representing the whole of this file. */ + fun getWholeFileLocation(): Label { + return getWholeFileLocation(fileId) + } +} + +/** + * A `SourceFileTrapWriter` is used when not only do we know which file we are extracting entities + * from, but we also have an `IrFileEntry` (from an `IrFile`) which allows us to map byte offsets to + * line and column numbers. + * + * An ID for the file will be created, and if `populateFileTables` is true then we will also add + * rows to the `files` and `folders` tables for it. + */ +class SourceFileTrapWriter( + loggerBase: LoggerBase, + lm: TrapLabelManager, + bw: BufferedWriter, + dtw: DiagnosticTrapWriter, + val irFile: IrFile, + populateFileTables: Boolean +) : FileTrapWriter(loggerBase, lm, bw, dtw, irFile.path, populateFileTables) { + + /** + * The file entry for the file that we are extracting from. Used to map offsets to line/column + * numbers. + */ + private val fileEntry = irFile.fileEntry + + override fun getLocation(startOffset: Int, endOffset: Int): Label { + if (startOffset == UNDEFINED_OFFSET || endOffset == UNDEFINED_OFFSET) { + if (startOffset != endOffset) { + loggerBase.warn( + dtw, + "Location with inconsistent offsets (start $startOffset, end $endOffset)", + null + ) + } + return getWholeFileLocation() + } + + if (startOffset == SYNTHETIC_OFFSET || endOffset == SYNTHETIC_OFFSET) { + if (startOffset != endOffset) { + loggerBase.warn( + dtw, + "Location with inconsistent offsets (start $startOffset, end $endOffset)", + null + ) + } + return getWholeFileLocation() + } + + // If this is the location for a compiler-generated element, then it will + // be a zero-width location. QL doesn't support these, so we translate it + // into a one-width location. + val endColumnOffset = if (startOffset == endOffset) 1 else 0 + return getLocation( + fileId, + fileEntry.getLineNumber(startOffset) + 1, + fileEntry.getColumnNumber(startOffset) + 1, + fileEntry.getLineNumber(endOffset) + 1, + fileEntry.getColumnNumber(endOffset) + endColumnOffset + ) + } + + override fun getLocationString(e: IrElement): String { + if (e.startOffset == UNDEFINED_OFFSET || e.endOffset == UNDEFINED_OFFSET) { + if (e.startOffset != e.endOffset) { + loggerBase.warn( + dtw, + "Location with inconsistent offsets (start ${e.startOffset}, end ${e.endOffset})", + null + ) + } + return "" + } + + if (e.startOffset == SYNTHETIC_OFFSET || e.endOffset == SYNTHETIC_OFFSET) { + if (e.startOffset != e.endOffset) { + loggerBase.warn( + dtw, + "Location with inconsistent offsets (start ${e.startOffset}, end ${e.endOffset})", + null + ) + } + return "" + } + + val startLine = fileEntry.getLineNumber(e.startOffset) + 1 + val startColumn = fileEntry.getColumnNumber(e.startOffset) + 1 + val endLine = fileEntry.getLineNumber(e.endOffset) + 1 + val endColumn = fileEntry.getColumnNumber(e.endOffset) + return "file://$filePath:$startLine:$startColumn:$endLine:$endColumn" + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/comments/CommentExtractor.kt b/java/kotlin-extractor2/src/main/kotlin/comments/CommentExtractor.kt new file mode 100644 index 00000000000..0c6308acd99 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/comments/CommentExtractor.kt @@ -0,0 +1,97 @@ +package com.github.codeql.comments + +import com.github.codeql.* +import com.github.codeql.utils.isLocalFunction +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.ir.expressions.IrBody +import org.jetbrains.kotlin.ir.expressions.IrExpression +import org.jetbrains.kotlin.ir.util.parentClassOrNull + +open class CommentExtractor( + protected val fileExtractor: KotlinFileExtractor, + protected val file: IrFile, + protected val fileLabel: Label +) { + protected val tw = fileExtractor.tw + protected val logger = fileExtractor.logger + + protected fun getLabel(element: IrElement): Label? { + if (element == file) return fileLabel + + if (element is IrValueParameter && element.index == -1) { + // Don't attribute comments to the implicit `this` parameter of a function. + return null + } + + val label: String + val existingLabel = + if (element is IrVariable) { + // local variables are not named globally, so we need to get them from the variable + // label cache + label = "variable ${element.name.asString()}" + tw.getExistingVariableLabelFor(element) + } else if (element is IrFunction && element.isLocalFunction()) { + // local functions are not named globally, so we need to get them from the local + // function label cache + label = "local function ${element.name.asString()}" + fileExtractor.getExistingLocallyVisibleFunctionLabel(element) + } else { + label = getLabelForNamedElement(element) ?: return null + tw.getExistingLabelFor(label) + } + if (existingLabel == null) { + // Sometimes we don't extract elements. + // The actual extractor logic is a bit more nuanced than + // just "isFake", but just checking isFake is good enough + // to not bother with a warning. + if (element !is IrDeclarationWithVisibility || !fileExtractor.isFake(element)) { + logger.warn("Couldn't get existing label for $label") + } + return null + } + return existingLabel + } + + private fun getLabelForNamedElement(element: IrElement): String? { + when (element) { + is IrClass -> return fileExtractor.getClassLabel(element, listOf()).classLabel + is IrTypeParameter -> return fileExtractor.getTypeParameterLabel(element) + is IrFunction -> { + return if (element.isLocalFunction()) { + null + } else { + fileExtractor.getFunctionLabel(element, null) + } + } + is IrValueParameter -> return fileExtractor.getValueParameterLabel(element, null) + is IrProperty -> return fileExtractor.getPropertyLabel(element) + is IrField -> return fileExtractor.getFieldLabel(element) + is IrEnumEntry -> return fileExtractor.getEnumEntryLabel(element) + is IrTypeAlias -> return fileExtractor.getTypeAliasLabel(element) + is IrAnonymousInitializer -> { + val parentClass = element.parentClassOrNull + if (parentClass == null) { + logger.warnElement("Parent of anonymous initializer is not a class", element) + return null + } + // Assign the comment to the class. The content of the `init` blocks might be + // extracted in multiple constructors. + return getLabelForNamedElement(parentClass) + } + + // Fresh entities, not named elements: + is IrBody -> return null + is IrExpression -> return null + + // todo add others: + else -> { + logger.warnElement( + "Unhandled element type found during comment extraction: ${element::class}", + element + ) + return null + } + } + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/comments/CommentExtractorPSI.kt b/java/kotlin-extractor2/src/main/kotlin/comments/CommentExtractorPSI.kt new file mode 100644 index 00000000000..3ca5dc45554 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/comments/CommentExtractorPSI.kt @@ -0,0 +1,122 @@ +package com.github.codeql.comments + +import com.github.codeql.* +import com.github.codeql.utils.IrVisitorLookup +import com.github.codeql.utils.Psi2IrFacade +import com.github.codeql.utils.versions.getPsi2Ir +import com.intellij.psi.PsiComment +import com.intellij.psi.PsiElement +import org.jetbrains.kotlin.config.KotlinCompilerVersion +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.kdoc.psi.api.KDoc +import org.jetbrains.kotlin.lexer.KtTokens +import org.jetbrains.kotlin.psi.KtVisitor +import org.jetbrains.kotlin.psi.psiUtil.endOffset +import org.jetbrains.kotlin.psi.psiUtil.startOffset + +class CommentExtractorPSI( + fileExtractor: KotlinFileExtractor, + file: IrFile, + fileLabel: Label +) : CommentExtractor(fileExtractor, file, fileLabel) { + // Returns true if it extracted the comments; false otherwise. + fun extract(): Boolean { + val psi2Ir = getPsi2Ir() + if (psi2Ir == null) { + logger.warn( + "Comments will not be extracted as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})" + ) + return false + } + val ktFile = psi2Ir.getKtFile(file) + if (ktFile == null) { + return false + } + val commentVisitor = mkCommentVisitor(psi2Ir) + ktFile.accept(commentVisitor) + return true + } + + private fun mkCommentVisitor(psi2Ir: Psi2IrFacade): KtVisitor = + object : KtVisitor() { + override fun visitElement(element: PsiElement) { + element.acceptChildren(this) + + // Slightly hacky, but `visitComment` doesn't seem to visit comments with + // `tokenType` `KtTokens.DOC_COMMENT` + if (element is PsiComment) { + visitCommentElement(element) + } + } + + private fun visitCommentElement(comment: PsiComment) { + val type: CommentType = + when (comment.tokenType) { + KtTokens.EOL_COMMENT -> { + CommentType.SingleLine + } + KtTokens.BLOCK_COMMENT -> { + CommentType.Block + } + KtTokens.DOC_COMMENT -> { + CommentType.Doc + } + else -> { + logger.warn("Unhandled comment token type: ${comment.tokenType}") + return + } + } + + val commentLabel = tw.getFreshIdLabel() + tw.writeKtComments(commentLabel, type.value, comment.text) + val locId = tw.getLocation(comment.startOffset, comment.endOffset) + tw.writeHasLocation(commentLabel, locId) + + if (comment.tokenType != KtTokens.DOC_COMMENT) { + return + } + + if (comment !is KDoc) { + logger.warn("Unexpected comment type with DocComment token type.") + return + } + + for (sec in comment.getAllSections()) { + val commentSectionLabel = tw.getFreshIdLabel() + tw.writeKtCommentSections(commentSectionLabel, commentLabel, sec.getContent()) + val name = sec.name + if (name != null) { + tw.writeKtCommentSectionNames(commentSectionLabel, name) + } + val subjectName = sec.getSubjectName() + if (subjectName != null) { + tw.writeKtCommentSectionSubjectNames(commentSectionLabel, subjectName) + } + } + + // Only storing the owner of doc comments: + val ownerPsi = getKDocOwner(comment) ?: return + + val owners = mutableListOf() + file.accept(IrVisitorLookup(psi2Ir, ownerPsi, file), owners) + + for (ownerIr in owners) { + val ownerLabel = getLabel(ownerIr) + if (ownerLabel != null) { + tw.writeKtCommentOwners(commentLabel, ownerLabel) + } + } + } + + private fun getKDocOwner(comment: KDoc): PsiElement? { + val owner = comment.owner + if (owner == null) { + logger.warn( + "Couldn't get owner of KDoc. The comment is extracted without an owner." + ) + } + return owner + } + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/comments/CommentType.kt b/java/kotlin-extractor2/src/main/kotlin/comments/CommentType.kt new file mode 100644 index 00000000000..1a1cec1aac2 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/comments/CommentType.kt @@ -0,0 +1,7 @@ +package com.github.codeql.comments + +enum class CommentType(val value: Int) { + SingleLine(1), + Block(2), + Doc(3) +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/AutoCloseableUse.kt b/java/kotlin-extractor2/src/main/kotlin/utils/AutoCloseableUse.kt new file mode 100644 index 00000000000..7883d04b8e4 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/AutoCloseableUse.kt @@ -0,0 +1,47 @@ +package com.github.codeql + +// Functions copied from stdlib/jdk7/src/kotlin/AutoCloseable.kt, which is not available within +// kotlinc, +// but allows the `.use` pattern to be applied to JDK7 AutoCloseables: + +/** + * Executes the given [block] function on this resource and then closes it down correctly whether an + * exception is thrown or not. + * + * In case if the resource is being closed due to an exception occurred in [block], and the closing + * also fails with an exception, the latter is added to the + * [suppressed][java.lang.Throwable.addSuppressed] exceptions of the former. + * + * @param block a function to process this [AutoCloseable] resource. + * @return the result of [block] function invoked on this resource. + */ +public inline fun T.useAC(block: (T) -> R): R { + var exception: Throwable? = null + try { + return block(this) + } catch (e: Throwable) { + exception = e + throw e + } finally { + this.closeFinallyAC(exception) + } +} + +/** + * Closes this [AutoCloseable], suppressing possible exception or error thrown by + * [AutoCloseable.close] function when it's being closed due to some other [cause] exception + * occurred. + * + * The suppressed exception is added to the list of suppressed exceptions of [cause] exception. + */ +fun AutoCloseable?.closeFinallyAC(cause: Throwable?) = + when { + this == null -> {} + cause == null -> close() + else -> + try { + close() + } catch (closeException: Throwable) { + cause.addSuppressed(closeException) + } + } diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/ClassNames.kt b/java/kotlin-extractor2/src/main/kotlin/utils/ClassNames.kt new file mode 100644 index 00000000000..54373933b84 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/ClassNames.kt @@ -0,0 +1,150 @@ +package com.github.codeql + +import com.github.codeql.utils.getJvmName +import com.github.codeql.utils.versions.* +import com.intellij.openapi.vfs.StandardFileSystems +import com.intellij.openapi.vfs.VirtualFile +import org.jetbrains.kotlin.builtins.jvm.JavaToKotlinClassMap +import org.jetbrains.kotlin.fir.java.JavaBinarySourceElement +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.ir.util.fqNameWhenAvailable +import org.jetbrains.kotlin.ir.util.parentClassOrNull +import org.jetbrains.kotlin.load.java.sources.JavaSourceElement +import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass +import org.jetbrains.kotlin.load.kotlin.JvmPackagePartSource +import org.jetbrains.kotlin.load.kotlin.KotlinJvmBinarySourceElement +import org.jetbrains.kotlin.load.kotlin.VirtualFileKotlinClass + +// Adapted from Kotlin's interpreter/Utils.kt function 'internalName' +// Translates class names into their JLS section 13.1 binary name, +// and declarations within them into the parent class' JLS 13.1 name as +// specified above, followed by a `$` separator and then the short name +// for `that`. +private fun getName(d: IrDeclarationWithName) = + (d as? IrAnnotationContainer)?.let { getJvmName(it) } ?: d.name.asString() + +fun getFileClassName(f: IrFile) = + getJvmName(f) + ?: ((f.fileEntry.name + .replaceFirst(Regex(""".*[/\\]"""), "") + .replaceFirst(Regex("""\.kt$"""), "") + .replaceFirstChar { it.uppercase() }) + "Kt") + +fun getIrElementBinaryName(that: IrElement): String { + if (that is IrFile) { + val shortName = getFileClassName(that) + val pkg = that.packageFqName.asString() + return if (pkg.isEmpty()) shortName else "$pkg.$shortName" + } + + if (that !is IrDeclaration) { + return "(unknown-name)" + } + + val shortName = + when (that) { + is IrDeclarationWithName -> getName(that) + else -> "(unknown-name)" + } + + val internalName = StringBuilder(shortName) + if (that !is IrClass) { + val parent = that.parent + if (parent is IrFile) { + // Note we'll fall through and do the IrPackageFragment case as well, since IrFile <: + // IrPackageFragment + internalName.insert(0, getFileClassName(parent) + "$") + } + } + + generateSequence(that.parent) { (it as? IrDeclaration)?.parent } + .forEach { + when (it) { + is IrClass -> internalName.insert(0, getName(it) + "$") + is IrPackageFragment -> + it.packageFqName + .asString() + .takeIf { fqName -> fqName.isNotEmpty() } + ?.let { fqName -> internalName.insert(0, "$fqName.") } + } + } + return internalName.toString() +} + +fun getIrClassVirtualFile(irClass: IrClass): VirtualFile? { + val cSource = irClass.source + // Don't emit a location for multi-file classes until we're sure we can cope with different + // declarations + // inside a class disagreeing about their source file. In particular this currently causes + // problems when + // a source-location for a declarations tries to refer to a file-id which is assumed to be + // declared in + // the class trap file. + if (irClass.origin == IrDeclarationOrigin.JVM_MULTIFILE_CLASS) return null + when (cSource) { + is JavaSourceElement -> { + val element = cSource.javaElement + when (element) { + is BinaryJavaClass -> return element.virtualFile + } + } + is JavaBinarySourceElement -> { + return cSource.javaClass.virtualFile + } + is KotlinJvmBinarySourceElement -> { + val binaryClass = cSource.binaryClass + when (binaryClass) { + is VirtualFileKotlinClass -> return binaryClass.file + } + } + is JvmPackagePartSource -> { + val binaryClass = cSource.knownJvmBinaryClass + if (binaryClass != null && binaryClass is VirtualFileKotlinClass) { + return binaryClass.file + } + } + } + return null +} + +private fun getRawIrClassBinaryPath(irClass: IrClass) = + getIrClassVirtualFile(irClass)?.let { + val path = it.path + if (it.fileSystem.protocol == StandardFileSystems.JRT_PROTOCOL) + // For JRT files, which we assume to be the JDK, hide the containing JAR path to match the + // Java extractor's behaviour. + "/${path.split("!/", limit = 2)[1]}" + else path + } + +fun getIrClassBinaryPath(irClass: IrClass): String { + return getRawIrClassBinaryPath(irClass) + // Otherwise, make up a fake location: + ?: getUnknownBinaryLocation(getIrElementBinaryName(irClass)) +} + +fun getIrDeclarationBinaryPath(d: IrDeclaration): String? { + if (d is IrClass) { + return getIrClassBinaryPath(d) + } + val parentClass = d.parentClassOrNull + if (parentClass != null) { + return getIrClassBinaryPath(parentClass) + } + if (d.parent is IrExternalPackageFragment) { + // This is in a file class. + val fqName = getFileClassFqName(d) + if (fqName != null) { + return getUnknownBinaryLocation(fqName.asString()) + } + } + return null +} + +private fun getUnknownBinaryLocation(s: String): String { + return "/!unknown-binary-location/${s.replace(".", "/")}.class" +} + +fun getJavaEquivalentClassId(c: IrClass) = + c.fqNameWhenAvailable?.toUnsafe()?.let { JavaToKotlinClassMap.mapKotlinToJava(it) } diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/ExternalDecls.kt b/java/kotlin-extractor2/src/main/kotlin/utils/ExternalDecls.kt new file mode 100644 index 00000000000..01464efedc1 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/ExternalDecls.kt @@ -0,0 +1,56 @@ +package com.github.codeql.utils + +import org.jetbrains.kotlin.ir.declarations.IrClass +import org.jetbrains.kotlin.ir.declarations.IrDeclaration +import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin +import org.jetbrains.kotlin.ir.declarations.IrExternalPackageFragment +import org.jetbrains.kotlin.ir.util.isFileClass + +fun isExternalDeclaration(d: IrDeclaration): Boolean { + /* + With Kotlin 1 we get things like (from .dump()): + PROPERTY IR_EXTERNAL_JAVA_DECLARATION_STUB name:MIN_VALUE visibility:public modality:FINAL [const,val] + FIELD IR_EXTERNAL_JAVA_DECLARATION_STUB name:MIN_VALUE type:kotlin.Int visibility:public [final,static] + EXPRESSION_BODY + CONST Int type=kotlin.Int value=-2147483648 + */ + if ( + d.origin == IrDeclarationOrigin.IR_EXTERNAL_DECLARATION_STUB || + d.origin == IrDeclarationOrigin.IR_EXTERNAL_JAVA_DECLARATION_STUB || + d.origin.toString() == "FUNCTION_INTERFACE_CLASS" + ) { // Treat kotlin.coroutines.* like ordinary library classes + return true + } + /* + With Kotlin 2, the property itself is not marked as an external stub, but it parent is: + CLASS IR_EXTERNAL_DECLARATION_STUB OBJECT name:Companion modality:OPEN visibility:public [companion] superTypes:[] + PROPERTY name:MIN_VALUE visibility:public modality:FINAL [const,val] + FIELD PROPERTY_BACKING_FIELD name:MIN_VALUE type:kotlin.Int visibility:public [final] + EXPRESSION_BODY + CONST Int type=kotlin.Int value=-2147483648 + */ + val p = d.parent + if (p is IrExternalPackageFragment) { + // This is an external declaration in a (multi)file class + return true + } + if (p is IrDeclaration) { + return isExternalDeclaration(p) + } + return false +} + +/** Returns true if `d` is not itself a class, but is a member of an external file class. */ +fun isExternalFileClassMember(d: IrDeclaration): Boolean { + if (d is IrClass) { + return false + } + val p = d.parent + when (p) { + is IrClass -> return p.isFileClass + is IrExternalPackageFragment -> + // This is an external declaration in a (multi)file class + return true + } + return false +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/GetByFqName.kt b/java/kotlin-extractor2/src/main/kotlin/utils/GetByFqName.kt new file mode 100644 index 00000000000..a3f5bb80a48 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/GetByFqName.kt @@ -0,0 +1,26 @@ +package com.github.codeql.utils + +import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext +import org.jetbrains.kotlin.ir.symbols.* +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.name.Name + +fun getClassByFqName(pluginContext: IrPluginContext, fqName: String): IrClassSymbol? { + return getClassByFqName(pluginContext, FqName(fqName)) +} + +fun getFunctionsByFqName( + pluginContext: IrPluginContext, + pkgName: String, + name: String +): Collection { + return getFunctionsByFqName(pluginContext, FqName(pkgName), Name.identifier(name)) +} + +fun getPropertiesByFqName( + pluginContext: IrPluginContext, + pkgName: String, + name: String +): Collection { + return getPropertiesByFqName(pluginContext, FqName(pkgName), Name.identifier(name)) +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/Helpers.kt b/java/kotlin-extractor2/src/main/kotlin/utils/Helpers.kt new file mode 100644 index 00000000000..aee909b1eb4 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/Helpers.kt @@ -0,0 +1,13 @@ +package com.github.codeql.utils + +import org.jetbrains.kotlin.descriptors.ClassKind +import org.jetbrains.kotlin.descriptors.DescriptorVisibilities +import org.jetbrains.kotlin.ir.declarations.IrClass +import org.jetbrains.kotlin.ir.declarations.IrFunction + +fun IrFunction.isLocalFunction(): Boolean { + return this.visibility == DescriptorVisibilities.LOCAL +} + +val IrClass.isInterfaceLike + get() = kind == ClassKind.INTERFACE || kind == ClassKind.ANNOTATION_CLASS diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/IrVisitorLookup.kt b/java/kotlin-extractor2/src/main/kotlin/utils/IrVisitorLookup.kt new file mode 100644 index 00000000000..2ac55ee7ede --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/IrVisitorLookup.kt @@ -0,0 +1,40 @@ +package com.github.codeql.utils + +import com.intellij.psi.PsiElement +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.declarations.IrDeclaration +import org.jetbrains.kotlin.ir.declarations.IrFile +import org.jetbrains.kotlin.ir.util.isFakeOverride +import org.jetbrains.kotlin.ir.visitors.IrElementVisitor + +class IrVisitorLookup( + private val psi2Ir: Psi2IrFacade, + private val psi: PsiElement, + private val file: IrFile +) : IrElementVisitor> { + private val location = psi.getLocation() + + override fun visitElement(element: IrElement, data: MutableCollection): Unit { + val elementLocation = element.getLocation() + + if (!location.intersects(elementLocation)) { + // No need to visit children. + return + } + + if (element is IrDeclaration && element.isFakeOverride) { + // These aren't extracted, so we don't expect anything to exist + // to which we could ascribe a comment. + return + } + + if (location.contains(elementLocation)) { + val psiElement = psi2Ir.findPsiElement(element, file) + if (psiElement == psi) { + // There can be multiple IrElements that match the same PSI element. + data.add(element) + } + } + element.acceptChildren(this, data) + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/Iterable.kt b/java/kotlin-extractor2/src/main/kotlin/utils/Iterable.kt new file mode 100644 index 00000000000..890ad7c04b9 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/Iterable.kt @@ -0,0 +1,13 @@ +package com.github.codeql + +import org.jetbrains.kotlin.ir.declarations.IrDeclaration + +/** + * This behaves the same as Iterable.find, but requires that the value found is of + * the subtype S, and it casts the result for you appropriately. + */ +inline fun Iterable.findSubType( + predicate: (S) -> Boolean +): S? { + return this.find { it is S && predicate(it) } as S? +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/JvmNames.kt b/java/kotlin-extractor2/src/main/kotlin/utils/JvmNames.kt new file mode 100644 index 00000000000..b6b51fde543 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/JvmNames.kt @@ -0,0 +1,95 @@ +package com.github.codeql.utils + +import com.github.codeql.utils.versions.allOverriddenIncludingSelf +import org.jetbrains.kotlin.builtins.StandardNames +import org.jetbrains.kotlin.ir.declarations.IrAnnotationContainer +import org.jetbrains.kotlin.ir.declarations.IrClass +import org.jetbrains.kotlin.ir.declarations.IrFunction +import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction +import org.jetbrains.kotlin.ir.expressions.IrConst +import org.jetbrains.kotlin.ir.expressions.IrConstructorCall +import org.jetbrains.kotlin.ir.types.IrSimpleType +import org.jetbrains.kotlin.ir.util.fqNameWhenAvailable +import org.jetbrains.kotlin.ir.util.packageFqName +import org.jetbrains.kotlin.ir.util.parentClassOrNull +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.name.Name + +private data class MethodKey(val className: FqName, val functionName: Name) + +private fun makeDescription(className: FqName, functionName: String) = + MethodKey(className, Name.guessByFirstCharacter(functionName)) + +// This essentially mirrors SpecialBridgeMethods.kt, a backend pass which isn't easily available to +// our extractor. +private val specialFunctions = + mapOf( + makeDescription(StandardNames.FqNames.collection, "") to "size", + makeDescription(FqName("java.util.Collection"), "") to "size", + makeDescription(StandardNames.FqNames.map, "") to "size", + makeDescription(FqName("java.util.Map"), "") to "size", + makeDescription(StandardNames.FqNames.charSequence.toSafe(), "") to "length", + makeDescription(FqName("java.lang.CharSequence"), "") to "length", + makeDescription(StandardNames.FqNames.map, "") to "keySet", + makeDescription(FqName("java.util.Map"), "") to "keySet", + makeDescription(StandardNames.FqNames.map, "") to "values", + makeDescription(FqName("java.util.Map"), "") to "values", + makeDescription(StandardNames.FqNames.map, "") to "entrySet", + makeDescription(FqName("java.util.Map"), "") to "entrySet", + makeDescription(StandardNames.FqNames.mutableList, "removeAt") to "remove", + makeDescription(FqName("java.util.List"), "removeAt") to "remove", + makeDescription(StandardNames.FqNames._enum.toSafe(), "") to "ordinal", + makeDescription(FqName("java.lang.Enum"), "") to "ordinal", + makeDescription(StandardNames.FqNames._enum.toSafe(), "") to "name", + makeDescription(FqName("java.lang.Enum"), "") to "name", + makeDescription(StandardNames.FqNames.number.toSafe(), "toByte") to "byteValue", + makeDescription(FqName("java.lang.Number"), "toByte") to "byteValue", + makeDescription(StandardNames.FqNames.number.toSafe(), "toShort") to "shortValue", + makeDescription(FqName("java.lang.Number"), "toShort") to "shortValue", + makeDescription(StandardNames.FqNames.number.toSafe(), "toInt") to "intValue", + makeDescription(FqName("java.lang.Number"), "toInt") to "intValue", + makeDescription(StandardNames.FqNames.number.toSafe(), "toLong") to "longValue", + makeDescription(FqName("java.lang.Number"), "toLong") to "longValue", + makeDescription(StandardNames.FqNames.number.toSafe(), "toFloat") to "floatValue", + makeDescription(FqName("java.lang.Number"), "toFloat") to "floatValue", + makeDescription(StandardNames.FqNames.number.toSafe(), "toDouble") to "doubleValue", + makeDescription(FqName("java.lang.Number"), "toDouble") to "doubleValue", + makeDescription(StandardNames.FqNames.string.toSafe(), "get") to "charAt", + makeDescription(FqName("java.lang.String"), "get") to "charAt", + ) + +private val specialFunctionShortNames = specialFunctions.keys.map { it.functionName }.toSet() + +private fun getSpecialJvmName(f: IrFunction): String? { + if (specialFunctionShortNames.contains(f.name) && f is IrSimpleFunction) { + f.allOverriddenIncludingSelf().forEach { overriddenFunc -> + overriddenFunc.parentClassOrNull?.fqNameWhenAvailable?.let { parentFqName -> + specialFunctions[MethodKey(parentFqName, f.name)]?.let { + return it + } + } + } + } + return null +} + +fun getJvmName(container: IrAnnotationContainer): String? { + for (a: IrConstructorCall in container.annotations) { + val t = a.type + if (t is IrSimpleType && a.valueArgumentsCount == 1) { + val owner = t.classifier.owner + val v = a.getValueArgument(0) + if (owner is IrClass) { + val aPkg = owner.packageFqName?.asString() + val name = owner.name.asString() + if (aPkg == "kotlin.jvm" && name == "JvmName" && v is IrConst<*>) { + val value = v.value + if (value is String) { + return value + } + } + } + } + } + return (container as? IrFunction)?.let { getSpecialJvmName(container) } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/List.kt b/java/kotlin-extractor2/src/main/kotlin/utils/List.kt new file mode 100644 index 00000000000..3ee6e243c76 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/List.kt @@ -0,0 +1,13 @@ +package com.github.codeql + +/** + * Turns this list of nullable elements into a list of non-nullable elements if they are all + * non-null, or returns null otherwise. + */ +public fun List.requireNoNullsOrNull(): List? { + try { + return this.requireNoNulls() + } catch (e: IllegalArgumentException) { + return null + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/Location.kt b/java/kotlin-extractor2/src/main/kotlin/utils/Location.kt new file mode 100644 index 00000000000..31b3a365816 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/Location.kt @@ -0,0 +1,24 @@ +package com.github.codeql.utils + +import com.intellij.psi.PsiElement +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.psi.psiUtil.endOffset +import org.jetbrains.kotlin.psi.psiUtil.startOffset + +data class Location(val startOffset: Int, val endOffset: Int) { + fun contains(location: Location): Boolean { + return this.startOffset <= location.startOffset && this.endOffset >= location.endOffset + } + + fun intersects(location: Location): Boolean { + return this.endOffset >= location.startOffset && this.startOffset <= location.endOffset + } +} + +fun IrElement.getLocation(): Location { + return Location(this.startOffset, this.endOffset) +} + +fun PsiElement.getLocation(): Location { + return Location(this.startOffset, this.endOffset) +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/Logger.kt b/java/kotlin-extractor2/src/main/kotlin/utils/Logger.kt new file mode 100644 index 00000000000..955a34feb1f --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/Logger.kt @@ -0,0 +1,356 @@ +package com.github.codeql + +import java.io.File +import java.io.FileWriter +import java.io.OutputStreamWriter +import java.io.Writer +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Stack +import org.jetbrains.kotlin.ir.IrElement + +class LogCounter() { + public val diagnosticInfo = mutableMapOf>() + public val diagnosticLimit: Int + + init { + diagnosticLimit = + System.getenv("CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT")?.toIntOrNull() ?: 100 + } +} + +enum class Severity(val sev: Int) { + WarnLow(1), + Warn(2), + WarnHigh(3), + /** Minor extractor errors, with minimal impact on analysis. */ + ErrorLow(4), + /** Most extractor errors, with local impact on analysis. */ + Error(5), + /** Javac errors. */ + ErrorHigh(6), + /** Severe extractor errors affecting a single source file. */ + ErrorSevere(7), + /** Severe extractor errors likely to affect multiple source files. */ + ErrorGlobal(8) +} + +class LogMessage(private val kind: String, private val message: String) { + val timestamp: String + + init { + timestamp = "${SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Date())}" + } + + fun toText(): String { + return "[$timestamp K] [$kind] $message" + } + + private fun escape(str: String): String { + return str.replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\u0000", "\\u0000") + .replace("\u0001", "\\u0001") + .replace("\u0002", "\\u0002") + .replace("\u0003", "\\u0003") + .replace("\u0004", "\\u0004") + .replace("\u0005", "\\u0005") + .replace("\u0006", "\\u0006") + .replace("\u0007", "\\u0007") + .replace("\u0008", "\\b") + .replace("\u0009", "\\t") + .replace("\u000A", "\\n") + .replace("\u000B", "\\u000B") + .replace("\u000C", "\\f") + .replace("\u000D", "\\r") + .replace("\u000E", "\\u000E") + .replace("\u000F", "\\u000F") + } + + fun toJsonLine(): String { + val kvs = + listOf( + Pair("origin", "CodeQL Kotlin extractor"), + Pair("timestamp", timestamp), + Pair("kind", kind), + Pair("message", message) + ) + return "{ " + + kvs.map { p -> "\"${p.first}\": \"${escape(p.second)}\"" }.joinToString(", ") + + " }\n" + } +} + +data class ExtractorContext( + val kind: String, + val element: IrElement, + val name: String, + val loc: String +) + +open class LoggerBase(val logCounter: LogCounter) { + val extractorContextStack = Stack() + + private val verbosity: Int + + init { + verbosity = System.getenv("CODEQL_EXTRACTOR_KOTLIN_VERBOSITY")?.toIntOrNull() ?: 3 + } + + private val logStream: Writer + + init { + val extractorLogDir = System.getenv("CODEQL_EXTRACTOR_JAVA_LOG_DIR") + if (extractorLogDir == null || extractorLogDir == "") { + logStream = OutputStreamWriter(System.out) + } else { + val logFile = File.createTempFile("kotlin-extractor.", ".log", File(extractorLogDir)) + logStream = FileWriter(logFile) + } + } + + private fun getDiagnosticLocation(): String? { + val st = Exception().stackTrace + for (x in st) { + when (x.className) { + "com.github.codeql.LoggerBase", + "com.github.codeql.Logger", + "com.github.codeql.FileLogger" -> {} + else -> { + return x.toString() + } + } + } + return null + } + + private var file_number = -1 + private var file_number_diagnostic_number = 0 + + fun setFileNumber(index: Int) { + file_number = index + file_number_diagnostic_number = 0 + } + + fun diagnostic( + dtw: DiagnosticTrapWriter, + severity: Severity, + msg: String, + extraInfo: String?, + locationString: String? = null, + mkLocationId: () -> Label = { dtw.unknownLocation } + ) { + val diagnosticLoc = getDiagnosticLocation() + val diagnosticLocStr = if (diagnosticLoc == null) "" else diagnosticLoc + val suffix = + if (diagnosticLoc == null) { + " Missing caller information.\n" + } else { + val oldInfo = + logCounter.diagnosticInfo.getOrDefault(diagnosticLoc, Pair(severity, 0)) + if (severity != oldInfo.first) { + // We don't want to get in a loop, so just emit this + // directly without going through the diagnostic + // counting machinery + if (verbosity >= 1) { + val message = + "Severity mismatch ($severity vs ${oldInfo.first}) at $diagnosticLoc" + emitDiagnostic(dtw, Severity.Error, "Inconsistency", message, message) + } + } + val newCount = oldInfo.second + 1 + val newInfo = Pair(severity, newCount) + logCounter.diagnosticInfo[diagnosticLoc] = newInfo + when { + logCounter.diagnosticLimit <= 0 -> "" + newCount == logCounter.diagnosticLimit -> + " Limit reached for diagnostics from $diagnosticLoc.\n" + newCount > logCounter.diagnosticLimit -> return + else -> "" + } + } + val fullMsgBuilder = StringBuilder() + fullMsgBuilder.append(msg) + if (extraInfo != null) { + fullMsgBuilder.append('\n') + fullMsgBuilder.append(extraInfo) + } + + val iter = extractorContextStack.listIterator(extractorContextStack.size) + while (iter.hasPrevious()) { + val x = iter.previous() + fullMsgBuilder.append(" ...while extracting a ${x.kind} (${x.name}) at ${x.loc}\n") + } + fullMsgBuilder.append(suffix) + + val fullMsg = fullMsgBuilder.toString() + emitDiagnostic(dtw, severity, diagnosticLocStr, msg, fullMsg, locationString, mkLocationId) + } + + private fun emitDiagnostic( + dtw: DiagnosticTrapWriter, + severity: Severity, + diagnosticLocStr: String, + msg: String, + fullMsg: String, + locationString: String? = null, + mkLocationId: () -> Label = { dtw.unknownLocation } + ) { + val locStr = if (locationString == null) "" else "At " + locationString + ": " + val kind = if (severity <= Severity.WarnHigh) "WARN" else "ERROR" + val logMessage = LogMessage(kind, "Diagnostic($diagnosticLocStr): $locStr$fullMsg") + // We don't actually make the location until after the `return` above + val locationId = mkLocationId() + val diagLabel = dtw.getFreshIdLabel() + dtw.writeDiagnostics( + diagLabel, + "CodeQL Kotlin extractor", + severity.sev, + "", + msg, + "${logMessage.timestamp} $fullMsg", + locationId + ) + dtw.writeDiagnostic_for( + diagLabel, + StringLabel("compilation"), + file_number, + file_number_diagnostic_number++ + ) + logStream.write(logMessage.toJsonLine()) + } + + fun trace(tw: TrapWriter, msg: String) { + if (verbosity >= 4) { + val logMessage = LogMessage("TRACE", msg) + tw.writeComment(logMessage.toText()) + logStream.write(logMessage.toJsonLine()) + } + } + + fun debug(tw: TrapWriter, msg: String) { + if (verbosity >= 4) { + val logMessage = LogMessage("DEBUG", msg) + tw.writeComment(logMessage.toText()) + logStream.write(logMessage.toJsonLine()) + } + } + + fun info(tw: TrapWriter, msg: String) { + if (verbosity >= 3) { + val logMessage = LogMessage("INFO", msg) + tw.writeComment(logMessage.toText()) + logStream.write(logMessage.toJsonLine()) + } + } + + fun warn(dtw: DiagnosticTrapWriter, msg: String, extraInfo: String?) { + if (verbosity >= 2) { + diagnostic(dtw, Severity.Warn, msg, extraInfo) + } + } + + fun error(dtw: DiagnosticTrapWriter, msg: String, extraInfo: String?) { + if (verbosity >= 1) { + diagnostic(dtw, Severity.Error, msg, extraInfo) + } + } + + fun printLimitedDiagnosticCounts(dtw: DiagnosticTrapWriter) { + for ((caller, info) in logCounter.diagnosticInfo) { + val severity = info.first + val count = info.second + if (count >= logCounter.diagnosticLimit) { + val message = + "Total of $count diagnostics (reached limit of ${logCounter.diagnosticLimit}) from $caller." + if (verbosity >= 1) { + emitDiagnostic(dtw, severity, "Limit", message, message) + } + } + } + } + + fun flush() { + logStream.flush() + } + + fun close() { + logStream.close() + } +} + +open class Logger(val loggerBase: LoggerBase, open val dtw: DiagnosticTrapWriter) { + fun flush() { + dtw.flush() + loggerBase.flush() + } + + fun trace(msg: String) { + loggerBase.trace(dtw, msg) + } + + fun trace(msg: String, exn: Throwable) { + trace(msg + "\n" + exn.stackTraceToString()) + } + + fun debug(msg: String) { + loggerBase.debug(dtw, msg) + } + + fun info(msg: String) { + loggerBase.info(dtw, msg) + } + + private fun warn(msg: String, extraInfo: String?) { + loggerBase.warn(dtw, msg, extraInfo) + } + + fun warn(msg: String, exn: Throwable) { + warn(msg, exn.stackTraceToString()) + } + + fun warn(msg: String) { + warn(msg, null) + } + + private fun error(msg: String, extraInfo: String?) { + loggerBase.error(dtw, msg, extraInfo) + } + + fun error(msg: String) { + error(msg, null) + } + + fun error(msg: String, exn: Throwable) { + error(msg, exn.stackTraceToString()) + } +} + +class FileLogger(loggerBase: LoggerBase, val ftw: FileTrapWriter) : + Logger(loggerBase, ftw.getDiagnosticTrapWriter()) { + fun warnElement(msg: String, element: IrElement, exn: Throwable? = null) { + val locationString = ftw.getLocationString(element) + val mkLocationId = { ftw.getLocation(element) } + loggerBase.diagnostic( + ftw.getDiagnosticTrapWriter(), + Severity.Warn, + msg, + exn?.stackTraceToString(), + locationString, + mkLocationId + ) + } + + fun errorElement(msg: String, element: IrElement, exn: Throwable? = null) { + val locationString = ftw.getLocationString(element) + val mkLocationId = { ftw.getLocation(element) } + loggerBase.diagnostic( + ftw.getDiagnosticTrapWriter(), + Severity.Error, + msg, + exn?.stackTraceToString(), + locationString, + mkLocationId + ) + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/Psi2IrFacade.kt b/java/kotlin-extractor2/src/main/kotlin/utils/Psi2IrFacade.kt new file mode 100644 index 00000000000..f623efe2c85 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/Psi2IrFacade.kt @@ -0,0 +1,12 @@ +package com.github.codeql.utils + +import com.intellij.psi.PsiElement +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.declarations.IrFile +import org.jetbrains.kotlin.psi.KtFile + +interface Psi2IrFacade { + fun getKtFile(irFile: IrFile): KtFile? + + fun findPsiElement(irElement: IrElement, irFile: IrFile): PsiElement? +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/TypeResults.kt b/java/kotlin-extractor2/src/main/kotlin/utils/TypeResults.kt new file mode 100644 index 00000000000..ee705bd9ae8 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/TypeResults.kt @@ -0,0 +1,40 @@ +package com.github.codeql + +/** + * A triple of a type's database label, its signature for use in callable signatures, and its short + * name for use in all tables that provide a user-facing type name. + * + * `signature` is a Java primitive name (e.g. "int"), a fully-qualified class name + * ("package.OuterClass.InnerClass"), or an array ("componentSignature[]") Type variables have the + * signature of their upper bound. Type arguments and anonymous types do not have a signature. + * + * `shortName` is a Java primitive name (e.g. "int"), a class short name with Java-style type + * arguments ("InnerClass" or "OuterClass" or "OtherClass") or + * an array ("componentShortName[]"). + */ +data class TypeResultGeneric( + val id: Label, + val signature: SignatureType?, + val shortName: String +) { + fun cast(): TypeResultGeneric { + @Suppress("UNCHECKED_CAST") return this as TypeResultGeneric + } +} + +data class TypeResultsGeneric( + val javaResult: TypeResultGeneric, + val kotlinResult: TypeResultGeneric +) + +typealias TypeResult = TypeResultGeneric + +typealias TypeResultWithoutSignature = TypeResultGeneric + +typealias TypeResults = TypeResultsGeneric + +typealias TypeResultsWithoutSignatures = TypeResultsGeneric + +fun TypeResult.forgetSignature(): TypeResultWithoutSignature { + return TypeResultWithoutSignature(this.id, Unit, this.shortName) +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/TypeSubstitution.kt b/java/kotlin-extractor2/src/main/kotlin/utils/TypeSubstitution.kt new file mode 100644 index 00000000000..91cd5ec9f13 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/TypeSubstitution.kt @@ -0,0 +1,283 @@ +package com.github.codeql.utils + +import com.github.codeql.KotlinUsesExtractor +import com.github.codeql.Logger +import com.github.codeql.getJavaEquivalentClassId +import com.github.codeql.utils.versions.codeQlWithHasQuestionMark +import com.github.codeql.utils.versions.createImplicitParameterDeclarationWithWrappedDescriptor +import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext +import org.jetbrains.kotlin.descriptors.ClassKind +import org.jetbrains.kotlin.ir.builders.declarations.addConstructor +import org.jetbrains.kotlin.ir.builders.declarations.buildClass +import org.jetbrains.kotlin.ir.declarations.IrClass +import org.jetbrains.kotlin.ir.declarations.IrTypeParameter +import org.jetbrains.kotlin.ir.declarations.IrTypeParametersContainer +import org.jetbrains.kotlin.ir.declarations.impl.IrExternalPackageFragmentImpl +import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl +import org.jetbrains.kotlin.ir.expressions.IrConstructorCall +import org.jetbrains.kotlin.ir.expressions.impl.IrConstructorCallImpl +import org.jetbrains.kotlin.ir.symbols.IrTypeParameterSymbol +import org.jetbrains.kotlin.ir.symbols.impl.DescriptorlessExternalPackageFragmentSymbol +import org.jetbrains.kotlin.ir.types.* +import org.jetbrains.kotlin.ir.types.impl.IrSimpleTypeImpl +import org.jetbrains.kotlin.ir.types.impl.IrStarProjectionImpl +import org.jetbrains.kotlin.ir.types.impl.makeTypeProjection +import org.jetbrains.kotlin.ir.util.classId +import org.jetbrains.kotlin.ir.util.constructedClassType +import org.jetbrains.kotlin.ir.util.constructors +import org.jetbrains.kotlin.ir.util.kotlinFqName +import org.jetbrains.kotlin.ir.util.parents +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.name.Name +import org.jetbrains.kotlin.types.Variance +import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull + +fun IrType.substituteTypeArguments(params: List, arguments: List) = + when (this) { + is IrSimpleType -> substituteTypeArguments(params.map { it.symbol }.zip(arguments).toMap()) + else -> this + } + +private fun IrSimpleType.substituteTypeArguments( + substitutionMap: Map +): IrSimpleType { + if (substitutionMap.isEmpty()) return this + + val newArguments = + arguments.map { + if (it is IrTypeProjection) { + val itType = it.type + if (itType is IrSimpleType) { + subProjectedType(substitutionMap, itType, it.variance) + } else { + it + } + } else { + it + } + } + + return IrSimpleTypeImpl(classifier, isNullable(), newArguments, annotations) +} + +/** + * Returns true if substituting `innerVariance T` into the context `outerVariance []` discards all + * knowledge about what T could be. + * + * Note this throws away slightly more information than it could: for example, the projection "in + * (out List)" can refer to any superclass of anything that implements List, which specifically + * excludes e.g. String, but can't be represented as a type projection. The projection "out (in + * List)" on the other hand really is equivalent to "out Any?", which is to say no bound at all. + */ +private fun conflictingVariance(outerVariance: Variance, innerVariance: Variance) = + (outerVariance == Variance.IN_VARIANCE && innerVariance == Variance.OUT_VARIANCE) || + (outerVariance == Variance.OUT_VARIANCE && innerVariance == Variance.IN_VARIANCE) + +/** + * When substituting `innerVariance T` into the context `outerVariance []`, returns the variance + * part of the result `resultVariance T`. We already know they don't conflict. + */ +private fun combineVariance(outerVariance: Variance, innerVariance: Variance) = + when { + outerVariance != Variance.INVARIANT -> outerVariance + innerVariance != Variance.INVARIANT -> innerVariance + else -> Variance.INVARIANT + } + +private fun subProjectedType( + substitutionMap: Map, + t: IrSimpleType, + outerVariance: Variance +): IrTypeArgument = + substitutionMap[t.classifier]?.let { substitutedTypeArg -> + if (substitutedTypeArg is IrTypeProjection) { + if (conflictingVariance(outerVariance, substitutedTypeArg.variance)) + IrStarProjectionImpl + else { + val newProjectedType = + substitutedTypeArg.type.let { + if (t.isNullable()) it.codeQlWithHasQuestionMark(true) else it + } + val newVariance = combineVariance(outerVariance, substitutedTypeArg.variance) + makeTypeProjection(newProjectedType, newVariance) + } + } else { + substitutedTypeArg + } + } ?: makeTypeProjection(t.substituteTypeArguments(substitutionMap), outerVariance) + +private fun IrTypeArgument.upperBound(context: IrPluginContext) = + when (this) { + is IrStarProjection -> context.irBuiltIns.anyNType + is IrTypeProjection -> + when (this.variance) { + Variance.INVARIANT -> this.type + Variance.IN_VARIANCE -> + if (this.type.isNullable()) context.irBuiltIns.anyNType + else context.irBuiltIns.anyType + Variance.OUT_VARIANCE -> this.type + } + else -> context.irBuiltIns.anyNType + } + +private fun IrTypeArgument.lowerBound(context: IrPluginContext) = + when (this) { + is IrStarProjection -> context.irBuiltIns.nothingType + is IrTypeProjection -> + when (this.variance) { + Variance.INVARIANT -> this.type + Variance.IN_VARIANCE -> this.type + Variance.OUT_VARIANCE -> + if (this.type.isNullable()) context.irBuiltIns.nothingNType + else context.irBuiltIns.nothingType + } + else -> context.irBuiltIns.nothingType + } + +fun IrType.substituteTypeAndArguments( + substitutionMap: Map?, + useContext: KotlinUsesExtractor.TypeContext, + pluginContext: IrPluginContext +): IrType = + substitutionMap?.let { substMap -> + if (this is IrSimpleType) { + val typeClassifier = this.classifier + substMap[typeClassifier]?.let { + when (useContext) { + KotlinUsesExtractor.TypeContext.RETURN -> it.upperBound(pluginContext) + else -> it.lowerBound(pluginContext) + } + } ?: this.substituteTypeArguments(substMap) + } else { + this + } + } ?: this + +object RawTypeAnnotation { + // Much of this is taken from JvmGeneratorExtensionsImpl.kt, which is not easily accessible in + // plugin context. + // The constants "kotlin.internal.ir" and "RawType" could be referred to symbolically, but they + // move package + // between different versions of the Kotlin compiler. + val annotationConstructor: IrConstructorCall by lazy { + val irInternalPackage = FqName("kotlin.internal.ir") + val parent = + IrExternalPackageFragmentImpl( + DescriptorlessExternalPackageFragmentSymbol(), + irInternalPackage + ) + val annoClass = + IrFactoryImpl.buildClass { + kind = ClassKind.ANNOTATION_CLASS + name = irInternalPackage.child(Name.identifier("RawType")).shortName() + } + .apply { + createImplicitParameterDeclarationWithWrappedDescriptor() + this.parent = parent + addConstructor { isPrimary = true } + } + val constructor = annoClass.constructors.single() + IrConstructorCallImpl.fromSymbolOwner(constructor.constructedClassType, constructor.symbol) + } +} + +fun IrType.toRawType(): IrType = + when (this) { + is IrSimpleType -> { + when (val owner = this.classifier.owner) { + is IrClass -> { + if (this.arguments.isNotEmpty()) + this.addAnnotations(listOf(RawTypeAnnotation.annotationConstructor)) + else this + } + is IrTypeParameter -> owner.superTypes[0].toRawType() + else -> this + } + } + else -> this + } + +fun IrClass.toRawType(): IrType { + val result = this.typeWith(listOf()) + return if (this.typeParameters.isNotEmpty()) + result.addAnnotations(listOf(RawTypeAnnotation.annotationConstructor)) + else result +} + +fun IrTypeArgument.withQuestionMark(b: Boolean): IrTypeArgument = + when (this) { + is IrStarProjection -> this + is IrTypeProjection -> + this.type.let { + when (it) { + is IrSimpleType -> + if (it.isNullable() == b) this + else makeTypeProjection(it.codeQlWithHasQuestionMark(b), this.variance) + else -> this + } + } + else -> this + } + +typealias TypeSubstitution = (IrType, KotlinUsesExtractor.TypeContext, IrPluginContext) -> IrType + +private fun matchingTypeParameters(l: IrTypeParameter?, r: IrTypeParameter): Boolean { + if (l === r) return true + if (l == null) return false + // Special case: match List's E and MutableList's E, for example, because in the JVM lowering + // they will map to the same thing. + val lParent = l.parent as? IrClass ?: return false + val rParent = r.parent as? IrClass ?: return false + val lJavaId = getJavaEquivalentClassId(lParent) ?: lParent.classId + return (getJavaEquivalentClassId(rParent) ?: rParent.classId) == lJavaId && l.name == r.name +} + +// Returns true if type is C where C is declared `class C { ... }` +fun isUnspecialised( + paramsContainer: IrTypeParametersContainer, + args: List, + logger: Logger +): Boolean { + return isUnspecialised(paramsContainer, args, logger, paramsContainer) +} + +private fun isUnspecialised( + paramsContainer: IrTypeParametersContainer, + args: List, + logger: Logger, + origParamsContainer: IrTypeParametersContainer +): Boolean { + val unspecialisedHere = + paramsContainer.typeParameters.zip(args).all { paramAndArg -> + (paramAndArg.second as? IrTypeProjection)?.let { + // Type arg refers to the class' own type parameter? + it.variance == Variance.INVARIANT && + matchingTypeParameters( + it.type.classifierOrNull?.owner as? IrTypeParameter, + paramAndArg.first + ) + } ?: false + } + val remainingArgs = args.drop(paramsContainer.typeParameters.size) + + val parentTypeContainer = + paramsContainer.parents.firstIsInstanceOrNull() + + val parentUnspecialised = + when { + remainingArgs.isEmpty() -> true + parentTypeContainer == null -> { + logger.error( + "Found more type arguments than parameters: ${origParamsContainer.kotlinFqName.asString()}" + ) + false + } + else -> isUnspecialised(parentTypeContainer, remainingArgs, logger, origParamsContainer) + } + return unspecialisedHere && parentUnspecialised +} + +// Returns true if type is C where C is declared `class C { ... }` +fun isUnspecialised(type: IrSimpleType, logger: Logger) = + (type.classifier.owner as? IrClass)?.let { isUnspecialised(it, type.arguments, logger) } + ?: false diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/CommentExtractorLighterAST.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/CommentExtractorLighterAST.kt new file mode 100644 index 00000000000..59fb1d18d14 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/CommentExtractorLighterAST.kt @@ -0,0 +1,15 @@ +package com.github.codeql.comments + +import com.github.codeql.* +import org.jetbrains.kotlin.ir.declarations.* + +class CommentExtractorLighterAST( + fileExtractor: KotlinFileExtractor, + file: IrFile, + fileLabel: Label +) : CommentExtractor(fileExtractor, file, fileLabel) { + // We don't support LighterAST with old Kotlin versions + fun extract(): Boolean { + return false + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/ExperimentalCompilerApi.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/ExperimentalCompilerApi.kt new file mode 100644 index 00000000000..9b40a26bc51 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/ExperimentalCompilerApi.kt @@ -0,0 +1,4 @@ +package org.jetbrains.kotlin.compiler.plugin + +@RequiresOptIn("This API is experimental. There are no stability guarantees for it") +annotation class ExperimentalCompilerApi diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/FileEntry.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/FileEntry.kt new file mode 100644 index 00000000000..43597b1158c --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/FileEntry.kt @@ -0,0 +1,5 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.ir.SourceManager + +typealias FileEntry = SourceManager.FileEntry diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/FirMetadataSourceFirFile.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/FirMetadataSourceFirFile.kt new file mode 100644 index 00000000000..9c103006099 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/FirMetadataSourceFirFile.kt @@ -0,0 +1 @@ +// Nothing to do diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/Functions.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/Functions.kt new file mode 100644 index 00000000000..2fd45e905d9 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/Functions.kt @@ -0,0 +1,8 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext +import org.jetbrains.kotlin.ir.declarations.IrClass + +fun functionN(pluginContext: IrPluginContext): (Int) -> IrClass { + return { i -> pluginContext.irBuiltIns.functionFactory.functionN(i) } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/IrSymbolInternals.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/IrSymbolInternals.kt new file mode 100644 index 00000000000..47d11ede1f2 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/IrSymbolInternals.kt @@ -0,0 +1,3 @@ +package org.jetbrains.kotlin.ir.symbols + +@RequiresOptIn(level = RequiresOptIn.Level.WARNING) annotation class IrSymbolInternals diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/IsUnderscoreParameter.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/IsUnderscoreParameter.kt new file mode 100644 index 00000000000..cb0fae2555b --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/IsUnderscoreParameter.kt @@ -0,0 +1,21 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI +import org.jetbrains.kotlin.ir.declarations.IrValueParameter +import org.jetbrains.kotlin.psi.KtParameter +import org.jetbrains.kotlin.resolve.DescriptorToSourceUtils +import org.jetbrains.kotlin.resolve.calls.util.isSingleUnderscore +import org.jetbrains.kotlin.utils.addToStdlib.safeAs + +@OptIn(ObsoleteDescriptorBasedAPI::class) +fun isUnderscoreParameter(vp: IrValueParameter) = + try { + DescriptorToSourceUtils.getSourceFromDescriptor(vp.descriptor) + ?.safeAs() + ?.isSingleUnderscore == true + } catch (e: NotImplementedError) { + // Some kinds of descriptor throw in `getSourceFromDescriptor` as that method is not + // normally expected to + // be applied to synthetic functions. + false + } diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/JavaBinarySourceElement.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/JavaBinarySourceElement.kt new file mode 100644 index 00000000000..71b45fa3cbb --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/JavaBinarySourceElement.kt @@ -0,0 +1,11 @@ +package org.jetbrains.kotlin.fir.java + +import org.jetbrains.kotlin.descriptors.SourceElement +import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryJavaClass + +/* +We need this class to exist, but the compiler will never give us an +instance of it. +*/ +abstract class JavaBinarySourceElement private constructor(val javaClass: BinaryJavaClass) : + SourceElement {} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/JvmDefaultModeEnabled.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/JvmDefaultModeEnabled.kt new file mode 100644 index 00000000000..cd849652613 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/JvmDefaultModeEnabled.kt @@ -0,0 +1,7 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.config.JvmDefaultMode + +fun jvmDefaultModeEnabledIsEnabled(jdm: JvmDefaultMode): Boolean { + return jdm.forAllMethodsWithBody +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/Kotlin2ComponentRegistrar.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/Kotlin2ComponentRegistrar.kt new file mode 100644 index 00000000000..84c5fc3bfb6 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/Kotlin2ComponentRegistrar.kt @@ -0,0 +1,12 @@ +// For ComponentRegistrar +@file:Suppress("DEPRECATION") + +package com.github.codeql + +import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar +import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi + +@OptIn(ExperimentalCompilerApi::class) +abstract class Kotlin2ComponentRegistrar : ComponentRegistrar { + /* Nothing to do; supportsK2 doesn't exist yet. */ +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/LinesOfCodeLighterAST.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/LinesOfCodeLighterAST.kt new file mode 100644 index 00000000000..dbda9084a9d --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/LinesOfCodeLighterAST.kt @@ -0,0 +1,18 @@ +package com.github.codeql + +import org.jetbrains.kotlin.ir.declarations.* + +class LinesOfCodeLighterAST(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) { + // We don't support LighterAST with old Kotlin versions + fun linesOfCodeInFile(@Suppress("UNUSED_PARAMETER") id: Label): Boolean { + return false + } + + // We don't support LighterAST with old Kotlin versions + fun linesOfCodeInDeclaration( + @Suppress("UNUSED_PARAMETER") d: IrDeclaration, + @Suppress("UNUSED_PARAMETER") id: Label + ): Boolean { + return false + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/Psi2Ir.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/Psi2Ir.kt new file mode 100644 index 00000000000..2046b507bf8 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/Psi2Ir.kt @@ -0,0 +1,5 @@ +package com.github.codeql.utils.versions + +import com.github.codeql.utils.Psi2IrFacade + +fun getPsi2Ir(): Psi2IrFacade? = null diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/ReferenceEntity.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/ReferenceEntity.kt new file mode 100644 index 00000000000..22e49c97444 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/ReferenceEntity.kt @@ -0,0 +1,33 @@ +package com.github.codeql.utils + +import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext +import org.jetbrains.kotlin.ir.symbols.* +import org.jetbrains.kotlin.name.ClassId +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.name.Name + +fun getClassByFqName(pluginContext: IrPluginContext, fqName: FqName): IrClassSymbol? { + return pluginContext.referenceClass(fqName) +} + +fun getClassByClassId(pluginContext: IrPluginContext, id: ClassId): IrClassSymbol? { + return getClassByFqName(pluginContext, id.asSingleFqName()) +} + +fun getFunctionsByFqName( + pluginContext: IrPluginContext, + pkgName: FqName, + name: Name +): Collection { + val fqName = pkgName.child(name) + return pluginContext.referenceFunctions(fqName) +} + +fun getPropertiesByFqName( + pluginContext: IrPluginContext, + pkgName: FqName, + name: Name +): Collection { + val fqName = pkgName.child(name) + return pluginContext.referenceProperties(fqName) +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/SyntheticBodyKind.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/SyntheticBodyKind.kt new file mode 100644 index 00000000000..2d71b430c9f --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/SyntheticBodyKind.kt @@ -0,0 +1,5 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.ir.expressions.IrSyntheticBodyKind + +val kind_ENUM_ENTRIES: IrSyntheticBodyKind? = null diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/Types.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/Types.kt new file mode 100644 index 00000000000..630d649907c --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/Types.kt @@ -0,0 +1,6 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.backend.jvm.codegen.isRawType +import org.jetbrains.kotlin.ir.types.IrSimpleType + +fun IrSimpleType.isRawType() = this.isRawType() diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/UsesK2.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/UsesK2.kt new file mode 100644 index 00000000000..39219d77730 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/UsesK2.kt @@ -0,0 +1,7 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext + +fun usesK2(@Suppress("UNUSED_PARAMETER") pluginContext: IrPluginContext): Boolean { + return false +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/allOverriddenIncludingSelf.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/allOverriddenIncludingSelf.kt new file mode 100644 index 00000000000..24ac7e40f48 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/allOverriddenIncludingSelf.kt @@ -0,0 +1,6 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.backend.common.ir.allOverridden +import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction + +fun IrSimpleFunction.allOverriddenIncludingSelf() = this.allOverridden(includeSelf = true) diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/annotationType.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/annotationType.kt new file mode 100644 index 00000000000..d4e2f17430f --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/annotationType.kt @@ -0,0 +1,8 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext +import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI + +@OptIn(ObsoleteDescriptorBasedAPI::class) +fun getAnnotationType(context: IrPluginContext) = + context.typeTranslator.translateType(context.builtIns.annotationType) diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/copyTo.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/copyTo.kt new file mode 100644 index 00000000000..a0d46acc56e --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/copyTo.kt @@ -0,0 +1,7 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.backend.common.ir.copyTo +import org.jetbrains.kotlin.ir.declarations.IrFunction +import org.jetbrains.kotlin.ir.declarations.IrValueParameter + +fun copyParameterToFunction(p: IrValueParameter, f: IrFunction) = p.copyTo(f) diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/createImplicitParameterDeclarationWithWrappedDescriptor.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/createImplicitParameterDeclarationWithWrappedDescriptor.kt new file mode 100644 index 00000000000..36016980af1 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/createImplicitParameterDeclarationWithWrappedDescriptor.kt @@ -0,0 +1,7 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.backend.common.ir.createImplicitParameterDeclarationWithWrappedDescriptor +import org.jetbrains.kotlin.ir.declarations.IrClass + +fun IrClass.createImplicitParameterDeclarationWithWrappedDescriptor() = + this.createImplicitParameterDeclarationWithWrappedDescriptor() diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/getFileClassFqName.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/getFileClassFqName.kt new file mode 100644 index 00000000000..4400589ae34 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/getFileClassFqName.kt @@ -0,0 +1,8 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.ir.declarations.IrDeclaration +import org.jetbrains.kotlin.name.FqName + +fun getFileClassFqName(@Suppress("UNUSED_PARAMETER") d: IrDeclaration): FqName? { + return null +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/getKotlinType.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/getKotlinType.kt new file mode 100644 index 00000000000..3f21550f13f --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/getKotlinType.kt @@ -0,0 +1,6 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.ir.types.IrSimpleType +import org.jetbrains.kotlin.ir.types.impl.IrTypeBase + +fun getKotlinType(s: IrSimpleType) = (s as? IrTypeBase)?.kotlinType diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/packageFqName.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/packageFqName.kt new file mode 100644 index 00000000000..87e67cd8368 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/packageFqName.kt @@ -0,0 +1,10 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.name.FqName + +val IrFile.packageFqName: FqName + get() = this.fqName + +val IrPackageFragment.packageFqName: FqName + get() = this.fqName diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/parents.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/parents.kt new file mode 100644 index 00000000000..d30fb104c03 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/parents.kt @@ -0,0 +1,11 @@ +package org.jetbrains.kotlin.ir.util + +import org.jetbrains.kotlin.backend.common.lower.parents as kParents +import org.jetbrains.kotlin.backend.common.lower.parentsWithSelf as kParentsWithSelf +import org.jetbrains.kotlin.ir.declarations.* + +val IrDeclaration.parents: Sequence + get() = this.kParents + +val IrDeclaration.parentsWithSelf: Sequence + get() = this.kParentsWithSelf diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/withHasQuestionMark.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/withHasQuestionMark.kt new file mode 100644 index 00000000000..114391ae6bf --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_0/withHasQuestionMark.kt @@ -0,0 +1,8 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.ir.types.IrType +import org.jetbrains.kotlin.ir.types.withHasQuestionMark + +fun IrType.codeQlWithHasQuestionMark(b: Boolean): IrType { + return this.withHasQuestionMark(b) +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_20/FileEntry.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_20/FileEntry.kt new file mode 100644 index 00000000000..18308780bff --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_20/FileEntry.kt @@ -0,0 +1,5 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.ir.IrFileEntry + +typealias FileEntry = IrFileEntry diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_20/Psi2Ir.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_20/Psi2Ir.kt new file mode 100644 index 00000000000..6bd99e06ef2 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_5_20/Psi2Ir.kt @@ -0,0 +1,21 @@ +package com.github.codeql.utils.versions + +import com.github.codeql.utils.Psi2IrFacade +import com.intellij.psi.PsiElement +import org.jetbrains.kotlin.backend.common.psi.PsiSourceManager +import org.jetbrains.kotlin.backend.jvm.ir.getKtFile +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.declarations.IrFile +import org.jetbrains.kotlin.psi.KtFile + +fun getPsi2Ir(): Psi2IrFacade? = Psi2Ir() + +private class Psi2Ir() : Psi2IrFacade { + override fun getKtFile(irFile: IrFile): KtFile? { + return irFile.getKtFile() + } + + override fun findPsiElement(irElement: IrElement, irFile: IrFile): PsiElement? { + return PsiSourceManager.findPsiElement(irElement, irFile) + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_6_0/Functions.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_6_0/Functions.kt new file mode 100644 index 00000000000..7d6c3eda8c3 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_6_0/Functions.kt @@ -0,0 +1,5 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext + +fun functionN(pluginContext: IrPluginContext) = pluginContext.irBuiltIns::functionN diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_6_0/annotationType.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_6_0/annotationType.kt new file mode 100644 index 00000000000..d51db5eb05c --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_6_0/annotationType.kt @@ -0,0 +1,5 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext + +fun getAnnotationType(context: IrPluginContext) = context.irBuiltIns.annotationType diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_6_20/IsUnderscoreParameter.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_6_20/IsUnderscoreParameter.kt new file mode 100644 index 00000000000..d727126ffc4 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_6_20/IsUnderscoreParameter.kt @@ -0,0 +1,7 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin +import org.jetbrains.kotlin.ir.declarations.IrValueParameter + +fun isUnderscoreParameter(vp: IrValueParameter) = + vp.origin == IrDeclarationOrigin.UNDERSCORE_PARAMETER diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_6_20/Types.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_6_20/Types.kt new file mode 100644 index 00000000000..d44e4407400 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_6_20/Types.kt @@ -0,0 +1,6 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.backend.jvm.ir.isRawType +import org.jetbrains.kotlin.ir.types.IrSimpleType + +fun IrSimpleType.isRawType() = this.isRawType() diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_0/getFileClassFqName.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_0/getFileClassFqName.kt new file mode 100644 index 00000000000..82275ba0342 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_0/getFileClassFqName.kt @@ -0,0 +1,41 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.ir.declarations.IrDeclaration +import org.jetbrains.kotlin.ir.declarations.IrField +import org.jetbrains.kotlin.ir.declarations.IrMemberWithContainerSource +import org.jetbrains.kotlin.load.kotlin.FacadeClassSource +import org.jetbrains.kotlin.name.FqName + +fun getFileClassFqName(d: IrDeclaration): FqName? { + // d is in a file class. + // Get the name in a similar way to the compiler's ExternalPackageParentPatcherLowering + // visitMemberAccess/generateOrGetFacadeClass. + + // But first, fields aren't IrMemberWithContainerSource, so we need + // to get back to the property (if there is one) + if (d is IrField) { + val propSym = d.correspondingPropertySymbol + if (propSym != null) { + return getFileClassFqName(propSym.owner) + } + } + + // Now the main code + if (d is IrMemberWithContainerSource) { + val containerSource = d.containerSource + if (containerSource is FacadeClassSource) { + val facadeClassName = containerSource.facadeClassName + if (facadeClassName != null) { + // TODO: This is really a multifile-class rather than a file-class, + // but for now we treat them the same. + return facadeClassName.fqNameForTopLevelClassMaybeWithDollars + } else { + return containerSource.className.fqNameForTopLevelClassMaybeWithDollars + } + } else { + return null + } + } else { + return null + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_0/getKotlinType.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_0/getKotlinType.kt new file mode 100644 index 00000000000..2f53392fba1 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_0/getKotlinType.kt @@ -0,0 +1,5 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.ir.types.IrSimpleType + +fun getKotlinType(s: IrSimpleType) = s.kotlinType diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_0/withHasQuestionMark.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_0/withHasQuestionMark.kt new file mode 100644 index 00000000000..f4e171f8fb2 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_0/withHasQuestionMark.kt @@ -0,0 +1,13 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.ir.types.IrType +import org.jetbrains.kotlin.ir.types.makeNotNull +import org.jetbrains.kotlin.ir.types.makeNullable + +fun IrType.codeQlWithHasQuestionMark(b: Boolean): IrType { + if (b) { + return this.makeNullable() + } else { + return this.makeNotNull() + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_20/allOverriddenIncludingSelf.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_20/allOverriddenIncludingSelf.kt new file mode 100644 index 00000000000..2d33930ecbc --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_20/allOverriddenIncludingSelf.kt @@ -0,0 +1,6 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction +import org.jetbrains.kotlin.ir.util.allOverridden + +fun IrSimpleFunction.allOverriddenIncludingSelf() = this.allOverridden(includeSelf = true) diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_20/copyTo.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_20/copyTo.kt new file mode 100644 index 00000000000..a68e9343247 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_20/copyTo.kt @@ -0,0 +1,7 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.ir.declarations.IrFunction +import org.jetbrains.kotlin.ir.declarations.IrValueParameter +import org.jetbrains.kotlin.ir.util.copyTo + +fun copyParameterToFunction(p: IrValueParameter, f: IrFunction) = p.copyTo(f) diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_20/createImplicitParameterDeclarationWithWrappedDescriptor.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_20/createImplicitParameterDeclarationWithWrappedDescriptor.kt new file mode 100644 index 00000000000..f262ff226a0 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_7_20/createImplicitParameterDeclarationWithWrappedDescriptor.kt @@ -0,0 +1,7 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.ir.declarations.IrClass +import org.jetbrains.kotlin.ir.util.createImplicitParameterDeclarationWithWrappedDescriptor + +fun IrClass.createImplicitParameterDeclarationWithWrappedDescriptor() = + this.createImplicitParameterDeclarationWithWrappedDescriptor() diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_8_0/ExperimentalCompilerApi.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_8_0/ExperimentalCompilerApi.kt new file mode 100644 index 00000000000..48829cc30c5 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_8_0/ExperimentalCompilerApi.kt @@ -0,0 +1,4 @@ +package com.github.codeql + +// The compiler provides the annotation class, so we don't need to do +// anything diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_8_0/ReferenceEntity.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_8_0/ReferenceEntity.kt new file mode 100644 index 00000000000..666e4e38386 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_8_0/ReferenceEntity.kt @@ -0,0 +1,35 @@ +package com.github.codeql.utils + +import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext +import org.jetbrains.kotlin.ir.symbols.* +import org.jetbrains.kotlin.name.CallableId +import org.jetbrains.kotlin.name.ClassId +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.name.Name + +fun getClassByFqName(pluginContext: IrPluginContext, fqName: FqName): IrClassSymbol? { + val id = ClassId.topLevel(fqName) + return getClassByClassId(pluginContext, id) +} + +fun getClassByClassId(pluginContext: IrPluginContext, id: ClassId): IrClassSymbol? { + return pluginContext.referenceClass(id) +} + +fun getFunctionsByFqName( + pluginContext: IrPluginContext, + pkgName: FqName, + name: Name +): Collection { + val id = CallableId(pkgName, name) + return pluginContext.referenceFunctions(id) +} + +fun getPropertiesByFqName( + pluginContext: IrPluginContext, + pkgName: FqName, + name: Name +): Collection { + val id = CallableId(pkgName, name) + return pluginContext.referenceProperties(id) +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_8_0/SyntheticBodyKind.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_8_0/SyntheticBodyKind.kt new file mode 100644 index 00000000000..4c654282ca7 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_8_0/SyntheticBodyKind.kt @@ -0,0 +1,5 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.ir.expressions.IrSyntheticBodyKind + +val kind_ENUM_ENTRIES: IrSyntheticBodyKind? = IrSyntheticBodyKind.ENUM_ENTRIES diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/CommentExtractorLighterAST.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/CommentExtractorLighterAST.kt new file mode 100644 index 00000000000..ecaa5e23287 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/CommentExtractorLighterAST.kt @@ -0,0 +1,131 @@ +package com.github.codeql.comments + +import com.github.codeql.* +import com.github.codeql.utils.versions.* +import com.intellij.lang.LighterASTNode +import com.intellij.util.diff.FlyweightCapableTreeStructure +import org.jetbrains.kotlin.fir.backend.FirMetadataSource +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.ir.util.SYNTHETIC_OFFSET +import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid +import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid +import org.jetbrains.kotlin.ir.visitors.acceptVoid +import org.jetbrains.kotlin.kdoc.lexer.KDocTokens +import org.jetbrains.kotlin.lexer.KtTokens +import org.jetbrains.kotlin.util.getChildren + +// TODO: This doesn't give owners to as many comments as the PSI extractor does. +// See the library-tests/comments tests for details. + +class CommentExtractorLighterAST( + fileExtractor: KotlinFileExtractor, + file: IrFile, + fileLabel: Label +) : CommentExtractor(fileExtractor, file, fileLabel) { + // Returns true if it extracted the comments; false otherwise. + fun extract(): Boolean { + val sourceElement = + (file.metadata as? FirMetadataSource.File)?.firFile?.source + val treeStructure = sourceElement?.treeStructure + if (treeStructure == null) { + return false + } + + val owners = findKDocOwners(file) + extractComments(treeStructure.root, treeStructure, owners) + return true + } + + private fun findKDocOwners(file: IrFile): Map> { + fun LighterASTNode.isKDocComment() = this.tokenType == KDocTokens.KDOC + + val kDocOwners = mutableMapOf>() + val visitor = + object : IrElementVisitorVoid { + override fun visitElement(element: IrElement) { + val metadata = (element as? IrMetadataSourceOwner)?.metadata + val sourceElement = (metadata as? FirMetadataSource)?.fir?.source + val treeStructure = sourceElement?.treeStructure + + if (treeStructure != null) { + sourceElement.lighterASTNode + .getChildren(treeStructure) + .firstOrNull { it.isKDocComment() } + ?.let { kDoc -> + // LighterASTNodes are not stable, so we can't + // use the node itself as the key. But the + // startOffset should uniquely identify them + // anyway. + val startOffset = kDoc.startOffset + if ( + startOffset != UNDEFINED_OFFSET && + startOffset != SYNTHETIC_OFFSET + ) { + kDocOwners + .getOrPut(startOffset, { mutableListOf() }) + .add(element) + } + } + } + + element.acceptChildrenVoid(this) + } + } + file.acceptVoid(visitor) + return kDocOwners + } + + private fun extractComments( + node: LighterASTNode, + treeStructure: FlyweightCapableTreeStructure, + owners: Map> + ) { + node.getChildren(treeStructure).forEach { + if (KtTokens.COMMENTS.contains(it.tokenType)) { + extractComment(it, owners) + } else { + extractComments(it, treeStructure, owners) + } + } + } + + private fun extractComment(comment: LighterASTNode, owners: Map>) { + val type: CommentType = + when (comment.tokenType) { + KtTokens.EOL_COMMENT -> { + CommentType.SingleLine + } + KtTokens.BLOCK_COMMENT -> { + CommentType.Block + } + KtTokens.DOC_COMMENT -> { + CommentType.Doc + } + else -> { + logger.warn("Unhandled comment token type: ${comment.tokenType}") + return + } + } + + val commentLabel = tw.getFreshIdLabel() + tw.writeKtComments(commentLabel, type.value, comment.toString()) + val locId = tw.getLocation(comment.startOffset, comment.endOffset) + tw.writeHasLocation(commentLabel, locId) + + if (comment.tokenType != KtTokens.DOC_COMMENT) { + return + } + + // TODO: The PSI comment extractor extracts comment.getAllSections() + // here, so we should too + + for (owner in owners.getOrDefault(comment.startOffset, listOf())) { + val ownerLabel = getLabel(owner) + if (ownerLabel != null) { + tw.writeKtCommentOwners(commentLabel, ownerLabel) + } + } + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/FirMetadataSourceFirFile.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/FirMetadataSourceFirFile.kt new file mode 100644 index 00000000000..53fefb30817 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/FirMetadataSourceFirFile.kt @@ -0,0 +1,7 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.fir.backend.FirMetadataSource +import org.jetbrains.kotlin.fir.declarations.FirFile + +val FirMetadataSource.File.firFile: FirFile? + get() = this.files.elementAtOrNull(0) diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/Kotlin2ComponentRegistrar.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/Kotlin2ComponentRegistrar.kt new file mode 100644 index 00000000000..323d8e2c283 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/Kotlin2ComponentRegistrar.kt @@ -0,0 +1,13 @@ +// For ComponentRegistrar +@file:Suppress("DEPRECATION") + +package com.github.codeql + +import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar +import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi + +@OptIn(ExperimentalCompilerApi::class) +abstract class Kotlin2ComponentRegistrar : ComponentRegistrar { + override val supportsK2: Boolean + get() = true +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/LinesOfCodeLighterAST.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/LinesOfCodeLighterAST.kt new file mode 100644 index 00000000000..82357d9dfbd --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/LinesOfCodeLighterAST.kt @@ -0,0 +1,148 @@ +package com.github.codeql + +import com.github.codeql.utils.versions.* +import com.intellij.lang.LighterASTNode +import com.intellij.util.diff.FlyweightCapableTreeStructure +import org.jetbrains.kotlin.KtSourceElement +import org.jetbrains.kotlin.fir.backend.FirMetadataSource +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.lexer.KtTokens +import org.jetbrains.kotlin.util.getChildren + +class LinesOfCodeLighterAST(val logger: FileLogger, val tw: FileTrapWriter, val file: IrFile) { + val fileEntry = file.fileEntry + + fun linesOfCodeInFile(id: Label): Boolean { + val sourceElement = + (file.metadata as? FirMetadataSource.File)?.firFile?.source + if (sourceElement == null) { + return false + } + linesOfCodeInLighterAST(id, file, sourceElement) + // Even if linesOfCodeInLighterAST didn't manage to extract any + // information, if we got as far as calling it then we have + // LighterAST info for the file + return true + } + + fun linesOfCodeInDeclaration(d: IrDeclaration, id: Label): Boolean { + val metadata = (d as? IrMetadataSourceOwner)?.metadata + val sourceElement = (metadata as? FirMetadataSource)?.fir?.source + if (sourceElement == null) { + return false + } + linesOfCodeInLighterAST(id, d, sourceElement) + // Even if linesOfCodeInLighterAST didn't manage to extract any + // information, if we got as far as calling it then we have + // LighterAST info for the declaration + return true + } + + private fun linesOfCodeInLighterAST( + id: Label, + e: IrElement, + s: KtSourceElement + ) { + val rootStartOffset = s.startOffset + val rootEndOffset = s.endOffset + if (rootStartOffset < 0 || rootEndOffset < 0) { + // This is synthetic, or has an invalid location + tw.writeNumlines(id, 0, 0, 0) + return + } + val rootFirstLine = fileEntry.getLineNumber(rootStartOffset) + val rootLastLine = fileEntry.getLineNumber(rootEndOffset) + if (rootLastLine < rootFirstLine) { + logger.errorElement("Source element ends before it starts", e) + tw.writeNumlines(id, 0, 0, 0) + return + } + + val numLines = 1 + rootLastLine - rootFirstLine + val lineContents = Array(numLines) { LineContent() } + + val treeStructure = s.treeStructure + + processSubtree( + e, + treeStructure, + rootFirstLine, + rootLastLine, + lineContents, + s.lighterASTNode + ) + + val code = lineContents.count { it.containsCode } + val comment = lineContents.count { it.containsComment } + tw.writeNumlines(id, numLines, code, comment) + } + + private fun processSubtree( + e: IrElement, + treeStructure: FlyweightCapableTreeStructure, + rootFirstLine: Int, + rootLastLine: Int, + lineContents: Array, + node: LighterASTNode + ) { + if (KtTokens.WHITESPACES.contains(node.tokenType)) { + return + } + + val isComment = KtTokens.COMMENTS.contains(node.tokenType) + val children = node.getChildren(treeStructure) + + // Leaf nodes are assumed to be tokens, and + // therefore we count any lines that they are on. + // For comments, we actually need to look at the + // outermost node, as the leaves of KDocs don't + // necessarily cover all lines. + if (isComment || children.isEmpty()) { + val startOffset = node.getStartOffset() + val endOffset = node.getEndOffset() + if (startOffset < 0 || endOffset < 0) { + logger.errorElement("LighterAST node has negative offset", e) + return + } + if (startOffset > endOffset) { + logger.errorElement("LighterAST node has negative size", e) + return + } + // This may not be possible with LighterAST, but: + // We might get e.g. an import list for a file + // with no imports, which claims to have start + // and end offsets of 0. Anything of 0 width + // we therefore just skip. + if (startOffset == endOffset) { + return + } + val firstLine = fileEntry.getLineNumber(startOffset) + val lastLine = fileEntry.getLineNumber(endOffset) + if (firstLine < rootFirstLine) { + logger.errorElement("LighterAST element starts before root", e) + return + } else if (lastLine > rootLastLine) { + logger.errorElement("LighterAST element ends after root", e) + return + } + for (line in firstLine..lastLine) { + val lineContent = lineContents[line - rootFirstLine] + if (isComment) { + lineContent.containsComment = true + } else { + lineContent.containsCode = true + } + } + } else { + for (child in children) { + processSubtree(e, treeStructure, rootFirstLine, rootLastLine, lineContents, child) + } + } + } + + private class LineContent { + var containsComment = false + var containsCode = false + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/UsesK2.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/UsesK2.kt new file mode 100644 index 00000000000..8d5e3345565 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_0-Beta/UsesK2.kt @@ -0,0 +1,7 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext + +fun usesK2(pluginContext: IrPluginContext): Boolean { + return pluginContext.languageVersionSettings.languageVersion.usesK2 +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_20-Beta/packageFqName.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_20-Beta/packageFqName.kt new file mode 100644 index 00000000000..613d51c7cbf --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_1_9_20-Beta/packageFqName.kt @@ -0,0 +1,3 @@ +package com.github.codeql + +// The compiler provides packageFqName, so we don't need to do anything diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/FirMetadataSourceFirFile.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/FirMetadataSourceFirFile.kt new file mode 100644 index 00000000000..dc47af858b2 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/FirMetadataSourceFirFile.kt @@ -0,0 +1,7 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.fir.backend.FirMetadataSource +import org.jetbrains.kotlin.fir.declarations.FirFile + +val FirMetadataSource.File.firFile: FirFile? + get() = this.fir diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/IrSymbolInternals.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/IrSymbolInternals.kt new file mode 100644 index 00000000000..48829cc30c5 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/IrSymbolInternals.kt @@ -0,0 +1,4 @@ +package com.github.codeql + +// The compiler provides the annotation class, so we don't need to do +// anything diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/JavaBinarySourceElement.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/JavaBinarySourceElement.kt new file mode 100644 index 00000000000..80167cbcc8f --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/JavaBinarySourceElement.kt @@ -0,0 +1,3 @@ +/* +The compiler provides this class, so we don't have to do anything. +*/ diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/JvmDefaultModeEnabled.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/JvmDefaultModeEnabled.kt new file mode 100644 index 00000000000..10a936ed909 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/JvmDefaultModeEnabled.kt @@ -0,0 +1,7 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.config.JvmDefaultMode + +fun jvmDefaultModeEnabledIsEnabled(jdm: JvmDefaultMode): Boolean { + return jdm.isEnabled +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/Psi2Ir.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/Psi2Ir.kt new file mode 100644 index 00000000000..09c4f42e021 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/Psi2Ir.kt @@ -0,0 +1,21 @@ +package com.github.codeql.utils.versions + +import com.github.codeql.utils.Psi2IrFacade +import com.intellij.psi.PsiElement +import org.jetbrains.kotlin.ir.PsiSourceManager +import org.jetbrains.kotlin.backend.jvm.ir.getKtFile +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.declarations.IrFile +import org.jetbrains.kotlin.psi.KtFile + +fun getPsi2Ir(): Psi2IrFacade? = Psi2Ir() + +private class Psi2Ir() : Psi2IrFacade { + override fun getKtFile(irFile: IrFile): KtFile? { + return irFile.getKtFile() + } + + override fun findPsiElement(irElement: IrElement, irFile: IrFile): PsiElement? { + return PsiSourceManager.findPsiElement(irElement, irFile) + } +} diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/parents.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/parents.kt new file mode 100644 index 00000000000..9c103006099 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_0-RC1/parents.kt @@ -0,0 +1 @@ +// Nothing to do diff --git a/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_20-Beta2/getKotlinType.kt b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_20-Beta2/getKotlinType.kt new file mode 100644 index 00000000000..37fb2a70041 --- /dev/null +++ b/java/kotlin-extractor2/src/main/kotlin/utils/versions/v_2_0_20-Beta2/getKotlinType.kt @@ -0,0 +1,5 @@ +package com.github.codeql.utils.versions + +import org.jetbrains.kotlin.ir.types.IrSimpleType + +fun getKotlinType(s: IrSimpleType) = s.originalKotlinType diff --git a/java/kotlin-extractor2/src/main/resources/META-INF/services/org.jetbrains.kotlin.compiler.plugin.CommandLineProcessor b/java/kotlin-extractor2/src/main/resources/META-INF/services/org.jetbrains.kotlin.compiler.plugin.CommandLineProcessor new file mode 100644 index 00000000000..2d0055c74d4 --- /dev/null +++ b/java/kotlin-extractor2/src/main/resources/META-INF/services/org.jetbrains.kotlin.compiler.plugin.CommandLineProcessor @@ -0,0 +1 @@ +com.github.codeql.KotlinExtractorCommandLineProcessor diff --git a/java/kotlin-extractor2/src/main/resources/META-INF/services/org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar b/java/kotlin-extractor2/src/main/resources/META-INF/services/org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar new file mode 100644 index 00000000000..564ed6bfe25 --- /dev/null +++ b/java/kotlin-extractor2/src/main/resources/META-INF/services/org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar @@ -0,0 +1 @@ +com.github.codeql.KotlinExtractorComponentRegistrar diff --git a/java/kotlin-extractor2/versions.bzl b/java/kotlin-extractor2/versions.bzl new file mode 100644 index 00000000000..e124d6e150a --- /dev/null +++ b/java/kotlin-extractor2/versions.bzl @@ -0,0 +1,47 @@ +# when updating this list, `bazel mod tidy` should be run from `codeql` to update `MODULE.bazel` +VERSIONS = [ + "1.5.0", + "1.5.10", + "1.5.20", + "1.5.30", + "1.6.0", + "1.6.20", + "1.7.0", + "1.7.20", + "1.8.0", + "1.9.0-Beta", + "1.9.20-Beta", + "2.0.0-RC1", + "2.0.20-Beta2", +] + +def _version_to_tuple(v): + # we ignore the tag when comparing versions, for example 1.9.0-Beta <= 1.9.0 + v, _, ignored_tag = v.partition("-") + return tuple([int(x) for x in v.split(".")]) + +def version_less(lhs, rhs): + return _version_to_tuple(lhs) < _version_to_tuple(rhs) + +def get_language_version(version): + major, minor, _ = _version_to_tuple(version) + return "%s.%s" % (major, minor) + +def _basename(path): + if "/" not in path: + return path + return path[path.rindex("/") + 1:] + +def get_compatilibity_sources(version, dir): + prefix = "%s/v_" % dir + available = native.glob(["%s*" % prefix], exclude_directories = 0) + + # we want files with the same base name to replace ones for previous versions, hence the map + srcs = {} + for d in available: + compat_version = d[len(prefix):].replace("_", ".") + if version_less(version, compat_version): + break + files = native.glob(["%s/*.kt" % d]) + srcs |= {_basename(f): f for f in files} + return srcs.values()