mirror of
https://github.com/github/codeql.git
synced 2026-05-26 00:51:25 +02:00
Compare commits
2 Commits
asgerf/js-
...
tausbn/pyt
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5e34778d72 | ||
|
|
adbc4a7777 |
@@ -27,7 +27,7 @@ bazel_dep(name = "abseil-cpp", version = "20260107.1", repo_name = "absl")
|
||||
bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json")
|
||||
bazel_dep(name = "fmt", version = "12.1.0-codeql.1")
|
||||
bazel_dep(name = "rules_kotlin", version = "2.2.2-codeql.1")
|
||||
bazel_dep(name = "gazelle", version = "0.50.0")
|
||||
bazel_dep(name = "gazelle", version = "0.47.0")
|
||||
bazel_dep(name = "rules_dotnet", version = "0.21.5-codeql.1")
|
||||
bazel_dep(name = "googletest", version = "1.17.0.bcr.2")
|
||||
bazel_dep(name = "rules_rust", version = "0.69.0")
|
||||
|
||||
@@ -20,6 +20,6 @@ from ArtifactPoisoningFlow::PathNode source, ArtifactPoisoningFlow::PathNode sin
|
||||
where
|
||||
ArtifactPoisoningFlow::flowPath(source, sink) and
|
||||
event = getRelevantEventInPrivilegedContext(sink.getNode())
|
||||
select source.getNode(), source, sink,
|
||||
"Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@).",
|
||||
event, event.getName()
|
||||
select sink.getNode(), source, sink,
|
||||
"Potential artifact poisoning in $@, which may be controlled by an external user ($@).", sink,
|
||||
sink.getNode().toString(), event, event.getName()
|
||||
|
||||
@@ -20,5 +20,6 @@ from ArtifactPoisoningFlow::PathNode source, ArtifactPoisoningFlow::PathNode sin
|
||||
where
|
||||
ArtifactPoisoningFlow::flowPath(source, sink) and
|
||||
inNonPrivilegedContext(sink.getNode().asExpr())
|
||||
select source.getNode(), source, sink,
|
||||
"Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user."
|
||||
select sink.getNode(), source, sink,
|
||||
"Potential artifact poisoning in $@, which may be controlled by an external user.", sink,
|
||||
sink.getNode().toString()
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
---
|
||||
category: majorAnalysis
|
||||
---
|
||||
* Fixed alert messages in `actions/artifact-poisoning/critical` and `actions/artifact-poisoning/medium` as they previously included a redundant placeholder in the alert message that would on occasion contain a long block of yml that makes the alert difficult to understand. Also clarify the wording to make it clear that it is not the artifact that is being poisoned, but instead a potentially untrusted artifact that is consumed. Also change the alert location to be the source, to align more with other queries reporting an artifact (e.g. zipslip) which is more useful.
|
||||
@@ -55,21 +55,21 @@ nodes
|
||||
| .github/workflows/test25.yml:39:14:40:45 | ./gradlew buildScanPublishPrevious\n | semmle.label | ./gradlew buildScanPublishPrevious\n |
|
||||
subpaths
|
||||
#select
|
||||
| .github/actions/download-artifact-2/action.yaml:6:7:25:4 | Uses Step | .github/actions/download-artifact-2/action.yaml:6:7:25:4 | Uses Step | .github/workflows/artifactpoisoning92.yml:28:9:29:6 | Uses Step | Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@). | .github/workflows/artifactpoisoning92.yml:3:3:3:14 | workflow_run | workflow_run |
|
||||
| .github/actions/download-artifact-2/action.yaml:6:7:25:4 | Uses Step | .github/actions/download-artifact-2/action.yaml:6:7:25:4 | Uses Step | .github/workflows/artifactpoisoning92.yml:29:14:29:26 | make snapshot | Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@). | .github/workflows/artifactpoisoning92.yml:3:3:3:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning11.yml:13:9:32:6 | Uses Step | .github/workflows/artifactpoisoning11.yml:13:9:32:6 | Uses Step | .github/workflows/artifactpoisoning11.yml:38:11:38:77 | ./sonarcloud-data/x.py build -j$(nproc) --compiler gcc --skip-build | Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@). | .github/workflows/artifactpoisoning11.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning12.yml:13:9:32:6 | Uses Step | .github/workflows/artifactpoisoning12.yml:13:9:32:6 | Uses Step | .github/workflows/artifactpoisoning12.yml:38:11:38:25 | python foo/x.py | Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@). | .github/workflows/artifactpoisoning12.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning21.yml:13:9:18:6 | Uses Step | .github/workflows/artifactpoisoning21.yml:13:9:18:6 | Uses Step | .github/workflows/artifactpoisoning21.yml:19:14:20:21 | sh foo/cmd\n | Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@). | .github/workflows/artifactpoisoning21.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning22.yml:13:9:17:6 | Uses Step | .github/workflows/artifactpoisoning22.yml:13:9:17:6 | Uses Step | .github/workflows/artifactpoisoning22.yml:18:14:18:19 | sh cmd | Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@). | .github/workflows/artifactpoisoning22.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning31.yml:13:9:15:6 | Run Step | .github/workflows/artifactpoisoning31.yml:13:9:15:6 | Run Step | .github/workflows/artifactpoisoning31.yml:19:14:19:22 | ./foo/cmd | Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@). | .github/workflows/artifactpoisoning31.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning32.yml:13:9:16:6 | Run Step | .github/workflows/artifactpoisoning32.yml:13:9:16:6 | Run Step | .github/workflows/artifactpoisoning32.yml:17:14:18:20 | ./bar/cmd\n | Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@). | .github/workflows/artifactpoisoning32.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning33.yml:13:9:16:6 | Run Step | .github/workflows/artifactpoisoning33.yml:13:9:16:6 | Run Step | .github/workflows/artifactpoisoning33.yml:17:14:18:20 | ./bar/cmd\n | Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@). | .github/workflows/artifactpoisoning33.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning34.yml:13:9:16:6 | Run Step | .github/workflows/artifactpoisoning34.yml:13:9:16:6 | Run Step | .github/workflows/artifactpoisoning34.yml:20:14:22:23 | npm install\nnpm run lint\n | Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@). | .github/workflows/artifactpoisoning34.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning41.yml:13:9:21:6 | Run Step | .github/workflows/artifactpoisoning41.yml:13:9:21:6 | Run Step | .github/workflows/artifactpoisoning41.yml:22:14:22:22 | ./foo/cmd | Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@). | .github/workflows/artifactpoisoning41.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning42.yml:13:9:21:6 | Run Step | .github/workflows/artifactpoisoning42.yml:13:9:21:6 | Run Step | .github/workflows/artifactpoisoning42.yml:22:14:22:18 | ./cmd | Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@). | .github/workflows/artifactpoisoning42.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning71.yml:9:9:16:6 | Uses Step | .github/workflows/artifactpoisoning71.yml:9:9:16:6 | Uses Step | .github/workflows/artifactpoisoning71.yml:17:14:18:40 | sed -f config foo.md > bar.md\n | Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@). | .github/workflows/artifactpoisoning71.yml:4:5:4:16 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning81.yml:28:9:31:6 | Uses Step | .github/workflows/artifactpoisoning81.yml:28:9:31:6 | Uses Step | .github/workflows/artifactpoisoning81.yml:31:14:31:27 | python test.py | Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@). | .github/workflows/artifactpoisoning81.yml:3:5:3:23 | pull_request_target | pull_request_target |
|
||||
| .github/workflows/artifactpoisoning96.yml:13:9:18:6 | Uses Step | .github/workflows/artifactpoisoning96.yml:13:9:18:6 | Uses Step | .github/workflows/artifactpoisoning96.yml:18:14:18:24 | npm install | Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@). | .github/workflows/artifactpoisoning96.yml:2:3:2:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning101.yml:10:9:16:6 | Uses Step | .github/workflows/artifactpoisoning101.yml:10:9:16:6 | Uses Step | .github/workflows/artifactpoisoning101.yml:17:14:19:59 | PR_NUMBER=$(./get_pull_request_number.sh pr_number.txt)\necho "PR_NUMBER=$PR_NUMBER" >> $GITHUB_OUTPUT \n | Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@). | .github/workflows/artifactpoisoning101.yml:4:3:4:21 | pull_request_target | pull_request_target |
|
||||
| .github/workflows/test18.yml:12:15:33:12 | Uses Step | .github/workflows/test18.yml:12:15:33:12 | Uses Step | .github/workflows/test18.yml:36:15:40:58 | Uses Step | Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@). | .github/workflows/test18.yml:3:5:3:16 | workflow_run | workflow_run |
|
||||
| .github/workflows/test25.yml:22:9:32:6 | Uses Step: downloadBuildScan | .github/workflows/test25.yml:22:9:32:6 | Uses Step: downloadBuildScan | .github/workflows/test25.yml:39:14:40:45 | ./gradlew buildScanPublishPrevious\n | Potential artifact poisoning; the artifact being consumed has contents that may be controlled by an external user ($@). | .github/workflows/test25.yml:2:3:2:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning11.yml:38:11:38:77 | ./sonarcloud-data/x.py build -j$(nproc) --compiler gcc --skip-build | .github/workflows/artifactpoisoning11.yml:13:9:32:6 | Uses Step | .github/workflows/artifactpoisoning11.yml:38:11:38:77 | ./sonarcloud-data/x.py build -j$(nproc) --compiler gcc --skip-build | Potential artifact poisoning in $@, which may be controlled by an external user ($@). | .github/workflows/artifactpoisoning11.yml:38:11:38:77 | ./sonarcloud-data/x.py build -j$(nproc) --compiler gcc --skip-build | ./sonarcloud-data/x.py build -j$(nproc) --compiler gcc --skip-build | .github/workflows/artifactpoisoning11.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning12.yml:38:11:38:25 | python foo/x.py | .github/workflows/artifactpoisoning12.yml:13:9:32:6 | Uses Step | .github/workflows/artifactpoisoning12.yml:38:11:38:25 | python foo/x.py | Potential artifact poisoning in $@, which may be controlled by an external user ($@). | .github/workflows/artifactpoisoning12.yml:38:11:38:25 | python foo/x.py | python foo/x.py | .github/workflows/artifactpoisoning12.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning21.yml:19:14:20:21 | sh foo/cmd\n | .github/workflows/artifactpoisoning21.yml:13:9:18:6 | Uses Step | .github/workflows/artifactpoisoning21.yml:19:14:20:21 | sh foo/cmd\n | Potential artifact poisoning in $@, which may be controlled by an external user ($@). | .github/workflows/artifactpoisoning21.yml:19:14:20:21 | sh foo/cmd\n | sh foo/cmd\n | .github/workflows/artifactpoisoning21.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning22.yml:18:14:18:19 | sh cmd | .github/workflows/artifactpoisoning22.yml:13:9:17:6 | Uses Step | .github/workflows/artifactpoisoning22.yml:18:14:18:19 | sh cmd | Potential artifact poisoning in $@, which may be controlled by an external user ($@). | .github/workflows/artifactpoisoning22.yml:18:14:18:19 | sh cmd | sh cmd | .github/workflows/artifactpoisoning22.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning31.yml:19:14:19:22 | ./foo/cmd | .github/workflows/artifactpoisoning31.yml:13:9:15:6 | Run Step | .github/workflows/artifactpoisoning31.yml:19:14:19:22 | ./foo/cmd | Potential artifact poisoning in $@, which may be controlled by an external user ($@). | .github/workflows/artifactpoisoning31.yml:19:14:19:22 | ./foo/cmd | ./foo/cmd | .github/workflows/artifactpoisoning31.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning32.yml:17:14:18:20 | ./bar/cmd\n | .github/workflows/artifactpoisoning32.yml:13:9:16:6 | Run Step | .github/workflows/artifactpoisoning32.yml:17:14:18:20 | ./bar/cmd\n | Potential artifact poisoning in $@, which may be controlled by an external user ($@). | .github/workflows/artifactpoisoning32.yml:17:14:18:20 | ./bar/cmd\n | ./bar/cmd\n | .github/workflows/artifactpoisoning32.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning33.yml:17:14:18:20 | ./bar/cmd\n | .github/workflows/artifactpoisoning33.yml:13:9:16:6 | Run Step | .github/workflows/artifactpoisoning33.yml:17:14:18:20 | ./bar/cmd\n | Potential artifact poisoning in $@, which may be controlled by an external user ($@). | .github/workflows/artifactpoisoning33.yml:17:14:18:20 | ./bar/cmd\n | ./bar/cmd\n | .github/workflows/artifactpoisoning33.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning34.yml:20:14:22:23 | npm install\nnpm run lint\n | .github/workflows/artifactpoisoning34.yml:13:9:16:6 | Run Step | .github/workflows/artifactpoisoning34.yml:20:14:22:23 | npm install\nnpm run lint\n | Potential artifact poisoning in $@, which may be controlled by an external user ($@). | .github/workflows/artifactpoisoning34.yml:20:14:22:23 | npm install\nnpm run lint\n | npm install\nnpm run lint\n | .github/workflows/artifactpoisoning34.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning41.yml:22:14:22:22 | ./foo/cmd | .github/workflows/artifactpoisoning41.yml:13:9:21:6 | Run Step | .github/workflows/artifactpoisoning41.yml:22:14:22:22 | ./foo/cmd | Potential artifact poisoning in $@, which may be controlled by an external user ($@). | .github/workflows/artifactpoisoning41.yml:22:14:22:22 | ./foo/cmd | ./foo/cmd | .github/workflows/artifactpoisoning41.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning42.yml:22:14:22:18 | ./cmd | .github/workflows/artifactpoisoning42.yml:13:9:21:6 | Run Step | .github/workflows/artifactpoisoning42.yml:22:14:22:18 | ./cmd | Potential artifact poisoning in $@, which may be controlled by an external user ($@). | .github/workflows/artifactpoisoning42.yml:22:14:22:18 | ./cmd | ./cmd | .github/workflows/artifactpoisoning42.yml:4:3:4:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning71.yml:17:14:18:40 | sed -f config foo.md > bar.md\n | .github/workflows/artifactpoisoning71.yml:9:9:16:6 | Uses Step | .github/workflows/artifactpoisoning71.yml:17:14:18:40 | sed -f config foo.md > bar.md\n | Potential artifact poisoning in $@, which may be controlled by an external user ($@). | .github/workflows/artifactpoisoning71.yml:17:14:18:40 | sed -f config foo.md > bar.md\n | sed -f config foo.md > bar.md\n | .github/workflows/artifactpoisoning71.yml:4:5:4:16 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning81.yml:31:14:31:27 | python test.py | .github/workflows/artifactpoisoning81.yml:28:9:31:6 | Uses Step | .github/workflows/artifactpoisoning81.yml:31:14:31:27 | python test.py | Potential artifact poisoning in $@, which may be controlled by an external user ($@). | .github/workflows/artifactpoisoning81.yml:31:14:31:27 | python test.py | python test.py | .github/workflows/artifactpoisoning81.yml:3:5:3:23 | pull_request_target | pull_request_target |
|
||||
| .github/workflows/artifactpoisoning92.yml:28:9:29:6 | Uses Step | .github/actions/download-artifact-2/action.yaml:6:7:25:4 | Uses Step | .github/workflows/artifactpoisoning92.yml:28:9:29:6 | Uses Step | Potential artifact poisoning in $@, which may be controlled by an external user ($@). | .github/workflows/artifactpoisoning92.yml:28:9:29:6 | Uses Step | Uses Step | .github/workflows/artifactpoisoning92.yml:3:3:3:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning92.yml:29:14:29:26 | make snapshot | .github/actions/download-artifact-2/action.yaml:6:7:25:4 | Uses Step | .github/workflows/artifactpoisoning92.yml:29:14:29:26 | make snapshot | Potential artifact poisoning in $@, which may be controlled by an external user ($@). | .github/workflows/artifactpoisoning92.yml:29:14:29:26 | make snapshot | make snapshot | .github/workflows/artifactpoisoning92.yml:3:3:3:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning96.yml:18:14:18:24 | npm install | .github/workflows/artifactpoisoning96.yml:13:9:18:6 | Uses Step | .github/workflows/artifactpoisoning96.yml:18:14:18:24 | npm install | Potential artifact poisoning in $@, which may be controlled by an external user ($@). | .github/workflows/artifactpoisoning96.yml:18:14:18:24 | npm install | npm install | .github/workflows/artifactpoisoning96.yml:2:3:2:14 | workflow_run | workflow_run |
|
||||
| .github/workflows/artifactpoisoning101.yml:17:14:19:59 | PR_NUMBER=$(./get_pull_request_number.sh pr_number.txt)\necho "PR_NUMBER=$PR_NUMBER" >> $GITHUB_OUTPUT \n | .github/workflows/artifactpoisoning101.yml:10:9:16:6 | Uses Step | .github/workflows/artifactpoisoning101.yml:17:14:19:59 | PR_NUMBER=$(./get_pull_request_number.sh pr_number.txt)\necho "PR_NUMBER=$PR_NUMBER" >> $GITHUB_OUTPUT \n | Potential artifact poisoning in $@, which may be controlled by an external user ($@). | .github/workflows/artifactpoisoning101.yml:17:14:19:59 | PR_NUMBER=$(./get_pull_request_number.sh pr_number.txt)\necho "PR_NUMBER=$PR_NUMBER" >> $GITHUB_OUTPUT \n | PR_NUMBER=$(./get_pull_request_number.sh pr_number.txt)\necho "PR_NUMBER=$PR_NUMBER" >> $GITHUB_OUTPUT \n | .github/workflows/artifactpoisoning101.yml:4:3:4:21 | pull_request_target | pull_request_target |
|
||||
| .github/workflows/test18.yml:36:15:40:58 | Uses Step | .github/workflows/test18.yml:12:15:33:12 | Uses Step | .github/workflows/test18.yml:36:15:40:58 | Uses Step | Potential artifact poisoning in $@, which may be controlled by an external user ($@). | .github/workflows/test18.yml:36:15:40:58 | Uses Step | Uses Step | .github/workflows/test18.yml:3:5:3:16 | workflow_run | workflow_run |
|
||||
| .github/workflows/test25.yml:39:14:40:45 | ./gradlew buildScanPublishPrevious\n | .github/workflows/test25.yml:22:9:32:6 | Uses Step: downloadBuildScan | .github/workflows/test25.yml:39:14:40:45 | ./gradlew buildScanPublishPrevious\n | Potential artifact poisoning in $@, which may be controlled by an external user ($@). | .github/workflows/test25.yml:39:14:40:45 | ./gradlew buildScanPublishPrevious\n | ./gradlew buildScanPublishPrevious\n | .github/workflows/test25.yml:2:3:2:14 | workflow_run | workflow_run |
|
||||
|
||||
@@ -29,8 +29,4 @@ module CsharpDataFlow implements InputSig<Location> {
|
||||
predicate neverSkipInPathGraph(Node n) {
|
||||
exists(n.(AssignableDefinitionNode).getDefinition().getTargetAccess())
|
||||
}
|
||||
|
||||
DataFlowType getSourceContextParameterNodeType(Node p) {
|
||||
exists(p) and result.isSourceContextParameterType()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1179,8 +1179,7 @@ private module Cached {
|
||||
cached
|
||||
newtype TDataFlowType =
|
||||
TGvnDataFlowType(Gvn::GvnType t) or
|
||||
TDelegateDataFlowType(Callable lambda) { lambdaCreationExpr(_, lambda) } or
|
||||
TSourceContextParameterType()
|
||||
TDelegateDataFlowType(Callable lambda) { lambdaCreationExpr(_, lambda) }
|
||||
}
|
||||
|
||||
import Cached
|
||||
@@ -2395,8 +2394,6 @@ class DataFlowType extends TDataFlowType {
|
||||
|
||||
Callable asDelegate() { this = TDelegateDataFlowType(result) }
|
||||
|
||||
predicate isSourceContextParameterType() { this = TSourceContextParameterType() }
|
||||
|
||||
/**
|
||||
* Gets an expression that creates a delegate of this type.
|
||||
*
|
||||
@@ -2415,9 +2412,6 @@ class DataFlowType extends TDataFlowType {
|
||||
result = this.asGvnType().toString()
|
||||
or
|
||||
result = this.asDelegate().toString()
|
||||
or
|
||||
this.isSourceContextParameterType() and
|
||||
result = "<source context parameter type>"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2475,11 +2469,6 @@ private predicate compatibleTypesDelegateLeft(DataFlowType dt1, DataFlowType dt2
|
||||
)
|
||||
}
|
||||
|
||||
pragma[nomagic]
|
||||
private predicate compatibleTypesSourceContextParameterTypeLeft(DataFlowType dt1, DataFlowType dt2) {
|
||||
dt1.isSourceContextParameterType() and not exists(dt2.asDelegate())
|
||||
}
|
||||
|
||||
/**
|
||||
* Holds if `t1` and `t2` are compatible, that is, whether data can flow from
|
||||
* a node of type `t1` to a node of type `t2`.
|
||||
@@ -2510,10 +2499,6 @@ predicate compatibleTypes(DataFlowType dt1, DataFlowType dt2) {
|
||||
compatibleTypesDelegateLeft(dt2, dt1)
|
||||
or
|
||||
dt1.asDelegate() = dt2.asDelegate()
|
||||
or
|
||||
compatibleTypesSourceContextParameterTypeLeft(dt1, dt2)
|
||||
or
|
||||
compatibleTypesSourceContextParameterTypeLeft(dt2, dt1)
|
||||
}
|
||||
|
||||
pragma[nomagic]
|
||||
@@ -2526,8 +2511,6 @@ predicate typeStrongerThan(DataFlowType t1, DataFlowType t2) {
|
||||
uselessTypebound(t2)
|
||||
or
|
||||
compatibleTypesDelegateLeft(t1, t2)
|
||||
or
|
||||
compatibleTypesSourceContextParameterTypeLeft(t1, t2)
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -536,12 +536,6 @@ public class HigherOrderParameters
|
||||
{
|
||||
a(o);
|
||||
}
|
||||
|
||||
private void CallApply()
|
||||
{
|
||||
// Test that this call to `Apply` does not interfere with the flow summaries generated for `Apply`
|
||||
Apply(x => x, null);
|
||||
}
|
||||
}
|
||||
|
||||
public static class HigherOrderExtensionMethods
|
||||
|
||||
@@ -11,23 +11,23 @@
|
||||
Microsoft extensions (up to VS 2022),
|
||||
|
||||
Arm Compiler 5 [5]_","``.cpp``, ``.c++``, ``.cxx``, ``.hpp``, ``.hh``, ``.h++``, ``.hxx``, ``.c``, ``.cc``, ``.h``"
|
||||
C#,C# up to 14 [6]_,"Microsoft Visual Studio up to 2019 with .NET up to 4.8,
|
||||
C#,C# up to 13,"Microsoft Visual Studio up to 2019 with .NET up to 4.8,
|
||||
|
||||
.NET Core up to 3.1
|
||||
|
||||
.NET 5, .NET 6, .NET 7, .NET 8, .NET 9, .NET 10 [6]_","``.sln``, ``.slnx``, ``.csproj``, ``.cs``, ``.cshtml``, ``.xaml``"
|
||||
.NET 5, .NET 6, .NET 7, .NET 8, .NET 9","``.sln``, ``.slnx``, ``.csproj``, ``.cs``, ``.cshtml``, ``.xaml``"
|
||||
GitHub Actions,"Not applicable",Not applicable,"``.github/workflows/*.yml``, ``.github/workflows/*.yaml``, ``**/action.yml``, ``**/action.yaml``"
|
||||
Go (aka Golang), "Go up to 1.26", "Go 1.11 or more recent", ``.go``
|
||||
Java,"Java 7 to 26 [7]_","javac (OpenJDK and Oracle JDK),
|
||||
Java,"Java 7 to 26 [6]_","javac (OpenJDK and Oracle JDK),
|
||||
|
||||
Eclipse compiler for Java (ECJ) [8]_",``.java``
|
||||
Eclipse compiler for Java (ECJ) [7]_",``.java``
|
||||
Kotlin,"Kotlin 1.8.0 to 2.3.2\ *x*","kotlinc",``.kt``
|
||||
JavaScript,ECMAScript 2022 or lower,Not applicable,"``.js``, ``.jsx``, ``.mjs``, ``.es``, ``.es6``, ``.htm``, ``.html``, ``.xhtm``, ``.xhtml``, ``.vue``, ``.hbs``, ``.ejs``, ``.njk``, ``.json``, ``.yaml``, ``.yml``, ``.raml``, ``.xml`` [9]_"
|
||||
Python [10]_,"2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13",Not applicable,``.py``
|
||||
Ruby [11]_,"up to 3.3",Not applicable,"``.rb``, ``.erb``, ``.gemspec``, ``Gemfile``"
|
||||
Rust [12]_,"Rust editions 2021 and 2024","Rust compiler","``.rs``, ``Cargo.toml``"
|
||||
Swift [13]_ [14]_,"Swift 5.4-6.2","Swift compiler","``.swift``"
|
||||
TypeScript [15]_,"2.6-5.9",Standard TypeScript compiler,"``.ts``, ``.tsx``, ``.mts``, ``.cts``"
|
||||
JavaScript,ECMAScript 2022 or lower,Not applicable,"``.js``, ``.jsx``, ``.mjs``, ``.es``, ``.es6``, ``.htm``, ``.html``, ``.xhtm``, ``.xhtml``, ``.vue``, ``.hbs``, ``.ejs``, ``.njk``, ``.json``, ``.yaml``, ``.yml``, ``.raml``, ``.xml`` [8]_"
|
||||
Python [9]_,"2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13",Not applicable,``.py``
|
||||
Ruby [10]_,"up to 3.3",Not applicable,"``.rb``, ``.erb``, ``.gemspec``, ``Gemfile``"
|
||||
Rust [11]_,"Rust editions 2021 and 2024","Rust compiler","``.rs``, ``Cargo.toml``"
|
||||
Swift [12]_ [13]_,"Swift 5.4-6.2","Swift compiler","``.swift``"
|
||||
TypeScript [14]_,"2.6-5.9",Standard TypeScript compiler,"``.ts``, ``.tsx``, ``.mts``, ``.cts``"
|
||||
|
||||
.. container:: footnote-group
|
||||
|
||||
@@ -36,13 +36,12 @@
|
||||
.. [3] Objective-C, Objective-C++, C++/CLI, and C++/CX are not supported.
|
||||
.. [4] Support for the clang-cl compiler is preliminary.
|
||||
.. [5] Support for the Arm Compiler (armcc) is preliminary.
|
||||
.. [6] Support for .NET 10 is preliminary and code that uses language features new to C# 14 is not yet fully supported for extraction and analysis.
|
||||
.. [7] Builds that execute on Java 7 to 26 can be analyzed. The analysis understands standard language features in Java 8 to 26; "preview" and "incubator" features are not supported. Source code using Java language versions older than Java 8 are analyzed as Java 8 code.
|
||||
.. [8] ECJ is supported when the build invokes it via the Maven Compiler plugin or the Takari Lifecycle plugin.
|
||||
.. [9] JSX and Flow code, YAML, JSON, HTML, and XML files may also be analyzed with JavaScript files.
|
||||
.. [10] The extractor requires Python 3 to run. To analyze Python 2.7 you should install both versions of Python.
|
||||
.. [11] Requires glibc 2.17.
|
||||
.. [12] Requires ``rustup`` and ``cargo`` to be installed. Features from nightly toolchains are not supported.
|
||||
.. [13] Support for the analysis of Swift requires macOS.
|
||||
.. [14] Embedded Swift is not supported.
|
||||
.. [15] TypeScript analysis is performed by running the JavaScript extractor with TypeScript enabled. This is the default.
|
||||
.. [6] Builds that execute on Java 7 to 26 can be analyzed. The analysis understands standard language features in Java 8 to 26; "preview" and "incubator" features are not supported. Source code using Java language versions older than Java 8 are analyzed as Java 8 code.
|
||||
.. [7] ECJ is supported when the build invokes it via the Maven Compiler plugin or the Takari Lifecycle plugin.
|
||||
.. [8] JSX and Flow code, YAML, JSON, HTML, and XML files may also be analyzed with JavaScript files.
|
||||
.. [9] The extractor requires Python 3 to run. To analyze Python 2.7 you should install both versions of Python.
|
||||
.. [10] Requires glibc 2.17.
|
||||
.. [11] Requires ``rustup`` and ``cargo`` to be installed. Features from nightly toolchains are not supported.
|
||||
.. [12] Support for the analysis of Swift requires macOS.
|
||||
.. [13] Embedded Swift is not supported.
|
||||
.. [14] TypeScript analysis is performed by running the JavaScript extractor with TypeScript enabled. This is the default.
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
# Build outputs
|
||||
bin/
|
||||
/typescript-parser-wrapper
|
||||
|
||||
# Validation output (generated during test comparison)
|
||||
validation-output/
|
||||
|
||||
# Go build cache
|
||||
*.test
|
||||
*.out
|
||||
@@ -1,94 +0,0 @@
|
||||
# TypeScript Parser Wrapper (Go)
|
||||
|
||||
Drop-in replacement for the Node.js TypeScript parser wrapper
|
||||
(`lib/typescript/src/main.ts`) that uses the TypeScript 7 Go-based
|
||||
compiler (`tsgo`) for parsing.
|
||||
|
||||
## Architecture
|
||||
|
||||
The Go wrapper implements the same stdin/stdout JSON protocol as the
|
||||
Node.js wrapper, making it a transparent replacement from the Java
|
||||
extractor's perspective.
|
||||
|
||||
```
|
||||
Java Extractor ──stdin/stdout JSON──▶ Go Wrapper ──▶ tsgo (TypeScript 7)
|
||||
```
|
||||
|
||||
### Protocol
|
||||
|
||||
Commands are sent as one JSON object per line on stdin:
|
||||
|
||||
| Command | Response Type | Description |
|
||||
|-----------------|---------------|------------------------------------|
|
||||
| `get-metadata` | `metadata` | Returns syntax kind/flag mappings |
|
||||
| `prepare-files` | `ok` | Hints about upcoming parse order |
|
||||
| `parse` | `ast` | Parses a file and returns the AST |
|
||||
| `reset` | `reset-done` | Resets state to fresh |
|
||||
| `quit` | *(exits)* | Shuts down the process |
|
||||
|
||||
### Package Structure
|
||||
|
||||
```
|
||||
cmd/typescript-parser-wrapper/ Entry point (server + single-file modes)
|
||||
internal/
|
||||
protocol/ JSON protocol handler
|
||||
tsparser/ Parser backend interface + tsgo impl
|
||||
astconv/ AST property whitelist + conversion
|
||||
validation/ Comparison tests (Node.js vs Go)
|
||||
scripts/
|
||||
validate-output.sh Shell script for bulk comparison
|
||||
testdata/ Sample TypeScript files for testing
|
||||
```
|
||||
|
||||
## Building
|
||||
|
||||
```bash
|
||||
go build -o bin/typescript-parser-wrapper ./cmd/typescript-parser-wrapper/
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
```bash
|
||||
# Unit tests
|
||||
go test ./...
|
||||
|
||||
# Validation against Node.js wrapper
|
||||
go test ./internal/validation/ -v
|
||||
|
||||
# Or via shell script
|
||||
./scripts/validate-output.sh testdata/sample.ts
|
||||
```
|
||||
|
||||
## Status
|
||||
|
||||
This is initial scaffolding. The parser backend communicates with the
|
||||
`tsgo` binary (from `@typescript/native-preview`) via its `--api --async`
|
||||
mode, which uses JSON-RPC 2.0 with LSP-style Content-Length framing.
|
||||
|
||||
**Validated so far:**
|
||||
- ✅ Successfully initialize the tsgo API subprocess
|
||||
- ✅ Open a project via `updateSnapshot` with a tsconfig
|
||||
- ✅ Retrieve binary-encoded source file data via `getSourceFile`
|
||||
- ✅ Protocol handler matches the Node.js wrapper's command set
|
||||
- ✅ Validation framework compares outputs (skips gracefully when Go can't parse yet)
|
||||
|
||||
**Key discovery: tsgo API returns binary-encoded ASTs**, not JSON.
|
||||
The `getSourceFile` response is a custom binary format (base64-encoded
|
||||
when using JSON protocol). This means the AST conversion layer needs
|
||||
to decode this binary format rather than transform JSON. See
|
||||
`microsoft/typescript-go/internal/api/encoder/encoder.go` for the
|
||||
format specification.
|
||||
|
||||
### Next Steps
|
||||
|
||||
1. **Decode binary AST format** — Implement a decoder for the tsgo
|
||||
encoder format (flat node array with string tables and sibling pointers)
|
||||
2. **Convert decoded AST to JSON** — Map decoded nodes to the JSON
|
||||
format expected by the Java extractor (property whitelist, `$pos`/`$end`,
|
||||
`$lineStarts`, `$tokens`, `$declarationKind`, string kind names)
|
||||
3. **Wire up end-to-end** — Connect the decoded AST through the protocol
|
||||
handler so `parse` commands return valid AST JSON
|
||||
4. **Validate against Node.js wrapper** — Run the comparison tests
|
||||
5. **Consider alternative: build from source** — Since all typescript-go
|
||||
packages are `internal/`, building our wrapper as a cmd inside a fork
|
||||
would give direct parser access without binary encoding overhead
|
||||
@@ -1,47 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/github/codeql/javascript/extractor/lib/typescript-go/internal/protocol"
|
||||
"github.com/github/codeql/javascript/extractor/lib/typescript-go/internal/tsparser"
|
||||
)
|
||||
|
||||
// Handler implements protocol.Handler by delegating to a tsparser.Parser.
|
||||
type Handler struct {
|
||||
parser tsparser.Parser
|
||||
pendingFiles []string
|
||||
}
|
||||
|
||||
// NewHandler creates a new Handler backed by the given parser.
|
||||
func NewHandler(parser tsparser.Parser) *Handler {
|
||||
return &Handler{parser: parser}
|
||||
}
|
||||
|
||||
func (h *Handler) HandleParse(filename string) (interface{}, error) {
|
||||
result, err := h.parser.Parse(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result.AST, nil
|
||||
}
|
||||
|
||||
func (h *Handler) HandlePrepareFiles(filenames []string) error {
|
||||
h.pendingFiles = filenames
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *Handler) HandleReset() error {
|
||||
h.pendingFiles = nil
|
||||
return h.parser.Reset()
|
||||
}
|
||||
|
||||
func (h *Handler) HandleGetMetadata() (*protocol.MetadataResponse, error) {
|
||||
meta, err := h.parser.GetMetadata()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &protocol.MetadataResponse{
|
||||
Type: "metadata",
|
||||
SyntaxKinds: meta.SyntaxKinds,
|
||||
NodeFlags: meta.NodeFlags,
|
||||
}, nil
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
package main
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
func marshalJSON(v interface{}) ([]byte, error) {
|
||||
return json.Marshal(v)
|
||||
}
|
||||
@@ -1,96 +0,0 @@
|
||||
// typescript-parser-wrapper is a drop-in replacement for the Node.js
|
||||
// TypeScript parser wrapper (lib/typescript/src/main.ts).
|
||||
//
|
||||
// It implements the same stdin/stdout JSON protocol, allowing the Java
|
||||
// extractor to use the TypeScript 7 (Go-based) compiler for parsing
|
||||
// TypeScript files.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// # Server mode (reads commands from stdin):
|
||||
// typescript-parser-wrapper
|
||||
//
|
||||
// # Parse a single file:
|
||||
// typescript-parser-wrapper file.ts
|
||||
//
|
||||
// # Print version:
|
||||
// typescript-parser-wrapper --version
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/github/codeql/javascript/extractor/lib/typescript-go/internal/protocol"
|
||||
"github.com/github/codeql/javascript/extractor/lib/typescript-go/internal/tsparser"
|
||||
)
|
||||
|
||||
const version = "0.1.0"
|
||||
|
||||
func main() {
|
||||
if len(os.Args) > 1 {
|
||||
arg := os.Args[1]
|
||||
switch {
|
||||
case arg == "--version":
|
||||
fmt.Println("typescript-parser-wrapper (Go) version " + version + " with TypeScript 7")
|
||||
os.Exit(0)
|
||||
case filepath.Ext(arg) == ".ts" || filepath.Ext(arg) == ".tsx":
|
||||
if err := parseSingleFile(arg); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
os.Exit(0)
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "Unrecognized file or flag: %s\n", arg)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Server mode
|
||||
if err := runServer(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func parseSingleFile(filename string) error {
|
||||
parser := createParser()
|
||||
defer parser.Close()
|
||||
|
||||
result, err := parser.Parse(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp := &protocol.ASTResponse{
|
||||
Type: "ast",
|
||||
AST: result.AST,
|
||||
}
|
||||
|
||||
data, err := marshalJSON(resp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
os.Stdout.Write(data)
|
||||
os.Stdout.Write([]byte("\n"))
|
||||
return nil
|
||||
}
|
||||
|
||||
func runServer() error {
|
||||
parser := createParser()
|
||||
defer parser.Close()
|
||||
|
||||
handler := NewHandler(parser)
|
||||
server := protocol.NewServer(handler)
|
||||
return server.Run()
|
||||
}
|
||||
|
||||
func createParser() tsparser.Parser {
|
||||
config := tsparser.Config{
|
||||
TsgoBinary: os.Getenv("SEMMLE_TYPESCRIPT_TSGO_BINARY"),
|
||||
Stderr: os.Stderr,
|
||||
}
|
||||
return tsparser.NewTsgoParser(config)
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
module github.com/github/codeql/javascript/extractor/lib/typescript-go
|
||||
|
||||
go 1.22
|
||||
@@ -1,211 +0,0 @@
|
||||
package astconv
|
||||
|
||||
// childProps maps SyntaxKind string names to ordered lists of child property names.
|
||||
// The order corresponds to the bitmask order in the binary encoder. When a node
|
||||
// uses the Children data type (top 2 bits = 0b00), the low byte is a bitmask
|
||||
// indicating which of these properties are present. Children are consumed in order.
|
||||
//
|
||||
// These names must match the property names expected by the Java extractor.
|
||||
// Derived from microsoft/typescript-go/internal/api/encoder/encoder.go.
|
||||
var childProps = map[string][]string{
|
||||
// Multi-child nodes with property mask
|
||||
"QualifiedName": {"left", "right"},
|
||||
"TypeParameter": {"modifiers", "name", "constraint", "default"},
|
||||
"IfStatement": {"expression", "thenStatement", "elseStatement"},
|
||||
"DoStatement": {"statement", "expression"},
|
||||
"WhileStatement": {"expression", "statement"},
|
||||
"ForStatement": {"initializer", "condition", "incrementor", "statement"},
|
||||
"ForInStatement": {"awaitModifier", "initializer", "expression", "statement"},
|
||||
"ForOfStatement": {"awaitModifier", "initializer", "expression", "statement"},
|
||||
"WithStatement": {"expression", "statement"},
|
||||
"SwitchStatement": {"expression", "caseBlock"},
|
||||
"CaseClause": {"expression", "statements"},
|
||||
"DefaultClause": {"expression", "statements"},
|
||||
"TryStatement": {"tryBlock", "catchClause", "finallyBlock"},
|
||||
"CatchClause": {"variableDeclaration", "block"},
|
||||
"LabeledStatement": {"label", "statement"},
|
||||
"VariableStatement": {"modifiers", "declarationList"},
|
||||
"VariableDeclarationList": {"declarations"},
|
||||
"VariableDeclaration": {"name", "exclamationToken", "type", "initializer"},
|
||||
"Parameter": {"modifiers", "dotDotDotToken", "name", "questionToken", "type", "initializer"},
|
||||
"BindingElement": {"dotDotDotToken", "propertyName", "name", "initializer"},
|
||||
"FunctionDeclaration": {"modifiers", "asteriskToken", "name", "typeParameters", "parameters", "type", "body"},
|
||||
"InterfaceDeclaration": {"modifiers", "name", "typeParameters", "heritageClauses", "members"},
|
||||
"TypeAliasDeclaration": {"modifiers", "name", "typeParameters", "type"},
|
||||
"EnumMember": {"name", "initializer"},
|
||||
"EnumDeclaration": {"modifiers", "name", "members"},
|
||||
"ModuleDeclaration": {"modifiers", "name", "body"},
|
||||
"ImportEqualsDeclaration": {"modifiers", "name", "moduleReference"},
|
||||
"ImportDeclaration": {"modifiers", "importClause", "moduleSpecifier", "attributes"},
|
||||
"JSImportDeclaration": {"modifiers", "importClause", "moduleSpecifier", "attributes"},
|
||||
"ImportSpecifier": {"propertyName", "name"},
|
||||
"ImportClause": {"name", "namedBindings"},
|
||||
"ExportAssignment": {"modifiers", "expression"},
|
||||
"JSExportAssignment": {"modifiers", "expression"},
|
||||
"NamespaceExportDeclaration": {"modifiers", "name"},
|
||||
"ExportDeclaration": {"modifiers", "exportClause", "moduleSpecifier", "attributes"},
|
||||
"ExportSpecifier": {"propertyName", "name"},
|
||||
"CallSignature": {"typeParameters", "parameters", "type"},
|
||||
"ConstructSignature": {"typeParameters", "parameters", "type"},
|
||||
"Constructor": {"modifiers", "typeParameters", "parameters", "type", "body"},
|
||||
"GetAccessor": {"modifiers", "name", "typeParameters", "parameters", "type", "body"},
|
||||
"SetAccessor": {"modifiers", "name", "typeParameters", "parameters", "type", "body"},
|
||||
"IndexSignature": {"modifiers", "parameters", "type"},
|
||||
"MethodSignature": {"modifiers", "name", "questionToken", "typeParameters", "parameters", "type"},
|
||||
"MethodDeclaration": {"modifiers", "asteriskToken", "name", "questionToken", "typeParameters", "parameters", "type", "body"},
|
||||
"PropertySignature": {"modifiers", "name", "questionToken", "type", "initializer"},
|
||||
"PropertyDeclaration": {"modifiers", "name", "questionToken", "type", "initializer"},
|
||||
"BinaryExpression": {"left", "operatorToken", "right"},
|
||||
"YieldExpression": {"asteriskToken", "expression"},
|
||||
"ArrowFunction": {"modifiers", "typeParameters", "parameters", "type", "equalsGreaterThanToken", "body"},
|
||||
"FunctionExpression": {"modifiers", "asteriskToken", "name", "typeParameters", "parameters", "type", "body"},
|
||||
"AsExpression": {"expression", "type"},
|
||||
"SatisfiesExpression": {"expression", "type"},
|
||||
"ConditionalExpression": {"condition", "questionToken", "whenTrue", "colonToken", "whenFalse"},
|
||||
"PropertyAccessExpression": {"expression", "questionDotToken", "name"},
|
||||
"ElementAccessExpression": {"expression", "questionDotToken", "argumentExpression"},
|
||||
"CallExpression": {"expression", "questionDotToken", "typeArguments", "arguments"},
|
||||
"NewExpression": {"expression", "typeArguments", "arguments"},
|
||||
"TemplateExpression": {"head", "templateSpans"},
|
||||
"TemplateSpan": {"expression", "literal"},
|
||||
"TaggedTemplateExpression": {"tag", "questionDotToken", "typeArguments", "template"},
|
||||
"PropertyAssignment": {"modifiers", "name", "questionToken", "initializer"},
|
||||
"ShorthandPropertyAssignment": {"modifiers", "name", "questionToken", "equalsToken", "objectAssignmentInitializer"},
|
||||
"TypeAssertionExpression": {"type", "expression"},
|
||||
"ConditionalType": {"checkType", "extendsType", "trueType", "falseType"},
|
||||
"IndexedAccessType": {"objectType", "indexType"},
|
||||
"TypeReference": {"typeName", "typeArguments"},
|
||||
"ExpressionWithTypeArguments": {"expression", "typeArguments"},
|
||||
"TypePredicate": {"assertsModifier", "parameterName", "type"},
|
||||
"ImportType": {"argument", "attributes", "qualifier", "typeArguments"},
|
||||
"ImportAttribute": {"name", "value"},
|
||||
"TypeQuery": {"exprName", "typeArguments"},
|
||||
"MappedType": {"readonlyToken", "typeParameter", "nameType", "questionToken", "type", "members"},
|
||||
"NamedTupleMember": {"dotDotDotToken", "name", "questionToken", "type"},
|
||||
"FunctionType": {"typeParameters", "parameters", "type"},
|
||||
"ConstructorType": {"modifiers", "typeParameters", "parameters", "type"},
|
||||
"TemplateLiteralType": {"head", "templateSpans"},
|
||||
"TemplateLiteralTypeSpan": {"type", "literal"},
|
||||
"JsxElement": {"openingElement", "children", "closingElement"},
|
||||
"JsxNamespacedName": {"name", "namespace"},
|
||||
"JsxOpeningElement": {"tagName", "typeArguments", "attributes"},
|
||||
"JsxSelfClosingElement": {"tagName", "typeArguments", "attributes"},
|
||||
"JsxFragment": {"openingFragment", "children", "closingFragment"},
|
||||
"JsxAttribute": {"name", "initializer"},
|
||||
"JsxExpression": {"dotDotDotToken", "expression"},
|
||||
"JSDoc": {"comment", "tags"},
|
||||
"JSDocTypeTag": {"tagName", "typeExpression", "comment"},
|
||||
"JSDocTag": {"tagName", "comment"},
|
||||
"JSDocTemplateTag": {"tagName", "constraint", "typeParameters", "comment"},
|
||||
"JSDocReturnTag": {"tagName", "typeExpression", "comment"},
|
||||
"JSDocPublicTag": {"tagName", "comment"},
|
||||
"JSDocPrivateTag": {"tagName", "comment"},
|
||||
"JSDocProtectedTag": {"tagName", "comment"},
|
||||
"JSDocReadonlyTag": {"tagName", "comment"},
|
||||
"JSDocOverrideTag": {"tagName", "comment"},
|
||||
"JSDocDeprecatedTag": {"tagName", "comment"},
|
||||
"JSDocSeeTag": {"tagName", "nameExpression", "comment"},
|
||||
"JSDocImplementsTag": {"tagName", "className", "comment"},
|
||||
"JSDocAugmentsTag": {"tagName", "className", "comment"},
|
||||
"JSDocSatisfiesTag": {"tagName", "typeExpression", "comment"},
|
||||
"JSDocThrowsTag": {"tagName", "typeExpression", "comment"},
|
||||
"JSDocThisTag": {"tagName", "typeExpression", "comment"},
|
||||
"JSDocImportTag": {"tagName", "importClause", "moduleSpecifier", "attributes", "comment"},
|
||||
"JSDocCallbackTag": {"tagName", "typeExpression", "fullName", "comment"},
|
||||
"JSDocOverloadTag": {"tagName", "typeExpression", "comment"},
|
||||
"JSDocTypedefTag": {"tagName", "typeExpression", "name", "comment"},
|
||||
"JSDocSignature": {"typeParameters", "parameters", "type"},
|
||||
"ClassStaticBlockDeclaration": {"modifiers", "body"},
|
||||
"ClassDeclaration": {"modifiers", "name", "typeParameters", "heritageClauses", "members"},
|
||||
"ClassExpression": {"modifiers", "name", "typeParameters", "heritageClauses", "members"},
|
||||
|
||||
// JSDocParameterTag and JSDocPropertyTag have order-dependent children
|
||||
// (handled specially in the converter based on isNameFirst defined bit).
|
||||
// Default order (isNameFirst=false):
|
||||
"JSDocParameterTag": {"tagName", "typeExpression", "name", "comment"},
|
||||
"JSDocPropertyTag": {"tagName", "typeExpression", "name", "comment"},
|
||||
}
|
||||
|
||||
// singleChildProp maps node kinds that have exactly one Node child to
|
||||
// the property name for that child.
|
||||
var singleChildProp = map[string]string{
|
||||
"ReturnStatement": "expression",
|
||||
"ThrowStatement": "expression",
|
||||
"ExpressionStatement": "expression",
|
||||
"BreakStatement": "label",
|
||||
"ContinueStatement": "label",
|
||||
"ParenthesizedExpression": "expression",
|
||||
"ComputedPropertyName": "expression",
|
||||
"Decorator": "expression",
|
||||
"SpreadElement": "expression",
|
||||
"SpreadAssignment": "expression",
|
||||
"DeleteExpression": "expression",
|
||||
"TypeOfExpression": "expression",
|
||||
"VoidExpression": "expression",
|
||||
"AwaitExpression": "expression",
|
||||
"NonNullExpression": "expression",
|
||||
"ExternalModuleReference": "expression",
|
||||
"NamespaceImport": "name",
|
||||
"NamespaceExport": "name",
|
||||
"JsxClosingElement": "tagName",
|
||||
"ArrayType": "elementType",
|
||||
"LiteralType": "literal",
|
||||
"InferType": "typeParameter",
|
||||
"OptionalType": "type",
|
||||
"RestType": "type",
|
||||
"ParenthesizedType": "type",
|
||||
"JSDocTypeExpression": "type",
|
||||
"JSDocNonNullableType": "type",
|
||||
"JSDocNullableType": "type",
|
||||
"JSDocVariadicType": "type",
|
||||
"JSDocOptionalType": "type",
|
||||
"JSDocNameReference": "name",
|
||||
}
|
||||
|
||||
// singleNodeListProp maps node kinds that have exactly one NodeList child
|
||||
// to the property name for that child.
|
||||
var singleNodeListProp = map[string]string{
|
||||
"Block": "statements",
|
||||
"ArrayLiteralExpression": "elements",
|
||||
"ObjectLiteralExpression": "properties",
|
||||
"UnionType": "types",
|
||||
"IntersectionType": "types",
|
||||
"TupleType": "elements",
|
||||
"NamedImports": "elements",
|
||||
"NamedExports": "elements",
|
||||
"ModuleBlock": "statements",
|
||||
"CaseBlock": "clauses",
|
||||
"TypeLiteral": "members",
|
||||
"JsxAttributes": "properties",
|
||||
"ArrayBindingPattern": "elements",
|
||||
"ObjectBindingPattern": "elements",
|
||||
"HeritageClause": "types",
|
||||
"ImportAttributes": "elements",
|
||||
"JSDocTypeLiteral": "jsDocPropertyTags",
|
||||
}
|
||||
|
||||
// operandKinds are node kinds where the single child is called "operand"
|
||||
// and the operator is encoded in the defined bits.
|
||||
var operandKinds = map[string]bool{
|
||||
"PrefixUnaryExpression": true,
|
||||
"PostfixUnaryExpression": true,
|
||||
}
|
||||
|
||||
// GetChildProperties returns the ordered child property names for the given
|
||||
// SyntaxKind name. Returns nil if the kind has no registered child properties
|
||||
// (leaf node, single-child, or NodeList-child).
|
||||
func GetChildProperties(kindName string) []string {
|
||||
return childProps[kindName]
|
||||
}
|
||||
|
||||
// GetSingleChildProperty returns the property name for a single-child node.
|
||||
// Returns "" if the kind is not a single-child node.
|
||||
func GetSingleChildProperty(kindName string) string {
|
||||
return singleChildProp[kindName]
|
||||
}
|
||||
|
||||
// GetSingleNodeListProperty returns the property name for a single-NodeList-child node.
|
||||
// Returns "" if the kind is not a single-NodeList-child node.
|
||||
func GetSingleNodeListProperty(kindName string) string {
|
||||
return singleNodeListProp[kindName]
|
||||
}
|
||||
@@ -1,842 +0,0 @@
|
||||
package astconv
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Converter transforms a BinaryAST into the JSON format expected by the
|
||||
// Java extractor.
|
||||
type Converter struct {
|
||||
ast *BinaryAST
|
||||
kindNames map[uint32]string // numeric kind → string name
|
||||
sourceText string // source file text for $lineStarts / $pos augmentation
|
||||
utf16Offsets []int // maps byte offset → UTF-16 code unit offset
|
||||
byteOffsets []int // maps UTF-16 code unit offset → byte offset
|
||||
parseDiagnostics []ParseDiagnostic // syntactic diagnostics from the compiler
|
||||
}
|
||||
|
||||
// ParseDiagnostic represents a syntactic error reported by the TypeScript compiler.
|
||||
type ParseDiagnostic struct {
|
||||
Pos int // UTF-16 offset of error start
|
||||
End int // UTF-16 offset of error end
|
||||
MessageText string // human-readable error message
|
||||
}
|
||||
|
||||
// NewConverter creates a Converter for the given binary AST.
|
||||
// kindToName maps numeric SyntaxKind values to their string names.
|
||||
func NewConverter(ast *BinaryAST, kindToName map[uint32]string) *Converter {
|
||||
text := ast.SourceText()
|
||||
utf16Table, byteTable := buildOffsetTables(text)
|
||||
return &Converter{
|
||||
ast: ast,
|
||||
kindNames: kindToName,
|
||||
sourceText: text,
|
||||
utf16Offsets: utf16Table,
|
||||
byteOffsets: byteTable,
|
||||
}
|
||||
}
|
||||
|
||||
// SetParseDiagnostics sets the syntactic diagnostics to include in the output.
|
||||
func (c *Converter) SetParseDiagnostics(diags []ParseDiagnostic) {
|
||||
c.parseDiagnostics = diags
|
||||
}
|
||||
|
||||
// Convert transforms the binary AST into a JSON-serializable map.
|
||||
// The root node is at index 1.
|
||||
func (c *Converter) Convert() (map[string]interface{}, error) {
|
||||
if c.ast.NodeCount() < 2 {
|
||||
return nil, fmt.Errorf("no nodes to convert")
|
||||
}
|
||||
return c.convertNode(1)
|
||||
}
|
||||
|
||||
// ConvertJSON is a convenience method that converts to JSON bytes.
|
||||
func (c *Converter) ConvertJSON() (json.RawMessage, error) {
|
||||
obj, err := c.Convert()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(obj)
|
||||
}
|
||||
|
||||
func (c *Converter) convertNode(i int) (map[string]interface{}, error) {
|
||||
kind := c.ast.Kind(i)
|
||||
kindName := c.kindNames[kind]
|
||||
if kindName == "" {
|
||||
kindName = fmt.Sprintf("Unknown_%d", kind)
|
||||
}
|
||||
|
||||
node := map[string]interface{}{
|
||||
"kind": int(kind),
|
||||
"flags": int(c.ast.Flags(i)),
|
||||
"$pos": c.augmentPos(int(c.ast.Pos(i)), true),
|
||||
"$end": int(c.ast.End(i)),
|
||||
}
|
||||
|
||||
dataType := c.ast.DataType(i)
|
||||
|
||||
switch dataType {
|
||||
case nodeDataTypeString:
|
||||
c.handleStringNode(i, kindName, node)
|
||||
|
||||
case nodeDataTypeExtended:
|
||||
if err := c.handleExtendedNode(i, kindName, node); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
default: // nodeDataTypeChildren
|
||||
if err := c.handleChildrenNode(i, kindName, node); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Add defined-bits-based properties
|
||||
c.addDefinedBitProperties(i, kindName, node)
|
||||
|
||||
if kindName == "ModuleDeclaration" {
|
||||
// TS7 doesn't set the NestedNamespace flag in the binary AST, but the Java
|
||||
// extractor needs it to wrap inner namespace declarations in ExportNamedDeclaration.
|
||||
// Detect nested namespaces (ModuleDeclaration whose body is another ModuleDeclaration)
|
||||
// and add the flag to the inner declaration.
|
||||
if body, ok := node["body"].(map[string]interface{}); ok {
|
||||
if bodyKind, ok := body["kind"].(int); ok && bodyKind == 268 { // 268 = ModuleDeclaration
|
||||
if flags, ok := body["flags"].(int); ok {
|
||||
body["flags"] = flags | 8 // NestedNamespace = 8
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TS7 binary AST doesn't have a GlobalAugmentation flag. Detect `declare global {}`
|
||||
// by checking if the name is "global" (Identifier), and set a synthetic flag bit
|
||||
// so the Java extractor can distinguish it from regular namespace declarations.
|
||||
if name, ok := node["name"].(map[string]interface{}); ok {
|
||||
if nameKind, ok := name["kind"].(int); ok && nameKind == 79 { // 79 = Identifier
|
||||
if text, _ := name["escapedText"].(string); text == "global" {
|
||||
if flags, ok := node["flags"].(int); ok {
|
||||
node["flags"] = flags | (1 << 30) // synthetic GlobalAugmentation
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return node, nil
|
||||
}
|
||||
|
||||
// handleStringNode handles nodes with a string property (Identifier, StringLiteral, etc.)
|
||||
func (c *Converter) handleStringNode(i int, kindName string, node map[string]interface{}) {
|
||||
strIdx := c.ast.StringIndex(i)
|
||||
text := c.ast.GetString(strIdx)
|
||||
|
||||
switch kindName {
|
||||
case "Identifier", "PrivateIdentifier":
|
||||
node["escapedText"] = text
|
||||
default:
|
||||
node["text"] = text
|
||||
}
|
||||
}
|
||||
|
||||
// handleExtendedNode handles SourceFile and template literal nodes.
|
||||
func (c *Converter) handleExtendedNode(i int, kindName string, node map[string]interface{}) error {
|
||||
extOff := c.ast.ExtOffset(i)
|
||||
|
||||
switch kindName {
|
||||
case "SourceFile":
|
||||
return c.handleSourceFile(i, extOff, node)
|
||||
case "TemplateHead", "TemplateMiddle", "TemplateTail":
|
||||
c.handleTemplateLiteral(extOff, node)
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("unknown extended data node kind: %s", kindName)
|
||||
}
|
||||
}
|
||||
|
||||
// handleSourceFile extracts SourceFile-specific data from extended data.
|
||||
func (c *Converter) handleSourceFile(i int, extOff uint32, node map[string]interface{}) error {
|
||||
// SourceFile extended data layout:
|
||||
// [0-4] textIdx, [4-8] fileNameIdx, [8-12] pathIdx,
|
||||
// [12-16] languageVariant, [16-20] scriptKind,
|
||||
// [20-24] referencedFiles, [24-28] typeReferenceDirectives, [28-32] libReferenceDirectives
|
||||
// [32-36] imports, [36-40] moduleAugmentations, [40-44] ambientModuleNames
|
||||
// [44-48] externalModuleIndicator
|
||||
|
||||
fileNameIdx := c.ast.ExtUint32(extOff + 4)
|
||||
node["fileName"] = c.ast.GetString(fileNameIdx)
|
||||
|
||||
// Add source text
|
||||
if c.sourceText != "" {
|
||||
node["text"] = c.sourceText
|
||||
node["$lineStarts"] = computeLineStarts(c.sourceText, c.utf16Offsets)
|
||||
}
|
||||
|
||||
// Add parseDiagnostics (expected by Java extractor).
|
||||
// The Java extractor uses these to report syntax errors and skip full extraction.
|
||||
diagArray := make([]interface{}, 0, len(c.parseDiagnostics))
|
||||
for _, d := range c.parseDiagnostics {
|
||||
diagArray = append(diagArray, map[string]interface{}{
|
||||
"$pos": d.Pos,
|
||||
"messageText": d.MessageText,
|
||||
})
|
||||
}
|
||||
node["parseDiagnostics"] = diagArray
|
||||
|
||||
// Add children (statements + EndOfFile)
|
||||
children := c.ast.Children(i)
|
||||
statementsFound := false
|
||||
for _, ci := range children {
|
||||
if c.ast.IsNodeList(ci) {
|
||||
arr, err := c.convertNodeList(ci)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
node["statements"] = arr
|
||||
statementsFound = true
|
||||
}
|
||||
// Skip EndOfFile token — the Java extractor doesn't use it
|
||||
}
|
||||
if !statementsFound {
|
||||
node["statements"] = []interface{}{}
|
||||
}
|
||||
|
||||
// Generate $tokens by scanning the source text.
|
||||
if c.sourceText != "" {
|
||||
events := c.collectRescanEvents(i)
|
||||
scanner := NewScanner(c.sourceText, events)
|
||||
rawTokens := scanner.ScanAll()
|
||||
tokenArr := make([]interface{}, len(rawTokens))
|
||||
for ti, tok := range rawTokens {
|
||||
tokenArr[ti] = map[string]interface{}{
|
||||
"kind": tok.Kind,
|
||||
"tokenPos": byteToUTF16(tok.TokenPos, c.utf16Offsets),
|
||||
"text": tok.Text,
|
||||
}
|
||||
}
|
||||
node["$tokens"] = tokenArr
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleTemplateLiteral extracts template literal data from extended data.
|
||||
func (c *Converter) handleTemplateLiteral(extOff uint32, node map[string]interface{}) {
|
||||
textIdx := c.ast.ExtUint32(extOff)
|
||||
rawTextIdx := c.ast.ExtUint32(extOff + 4)
|
||||
node["text"] = c.ast.GetString(textIdx)
|
||||
node["rawText"] = c.ast.GetString(rawTextIdx)
|
||||
}
|
||||
|
||||
// handleChildrenNode handles nodes with child properties determined by a bitmask.
|
||||
func (c *Converter) handleChildrenNode(i int, kindName string, node map[string]interface{}) error {
|
||||
children := c.ast.Children(i)
|
||||
|
||||
// Check for single-child nodes
|
||||
if prop := GetSingleChildProperty(kindName); prop != "" {
|
||||
if len(children) > 0 {
|
||||
child, err := c.convertNode(children[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
node[prop] = child
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check for single NodeList child nodes
|
||||
if prop := GetSingleNodeListProperty(kindName); prop != "" {
|
||||
if len(children) > 0 && c.ast.IsNodeList(children[0]) {
|
||||
arr, err := c.convertNodeList(children[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
node[prop] = arr
|
||||
} else if len(children) > 0 {
|
||||
// Some single-NodeList nodes may not have a NodeList child
|
||||
// (e.g., JSDocTypeLiteral). Fall through to multi-child handling.
|
||||
} else {
|
||||
node[prop] = []interface{}{}
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check for operator-in-definedBits nodes (PrefixUnaryExpression, PostfixUnaryExpression)
|
||||
if operandKinds[kindName] {
|
||||
if len(children) > 0 {
|
||||
child, err := c.convertNode(children[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
node["operand"] = child
|
||||
}
|
||||
node["operator"] = int(c.ast.DefinedBits(i))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Multi-child nodes with property mask
|
||||
props := GetChildProperties(kindName)
|
||||
if props != nil {
|
||||
return c.assignChildProperties(i, kindName, props, children, node)
|
||||
}
|
||||
|
||||
// Token/keyword nodes with no children — nothing to add
|
||||
if len(children) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// MetaProperty: keywordToken + name
|
||||
if kindName == "MetaProperty" {
|
||||
if len(children) > 0 {
|
||||
child, err := c.convertNode(children[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
node["name"] = child
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TypeOperator: operator keyword kind inferred from source text + type child
|
||||
if kindName == "TypeOperator" {
|
||||
// Operator (keyof/unique/readonly) is not in the binary encoding.
|
||||
bytePos := utf16ToByte(int(c.ast.Pos(i)), c.byteOffsets)
|
||||
if c.sourceText != "" && bytePos < len(c.sourceText) {
|
||||
text := c.sourceText[bytePos:]
|
||||
// Skip leading trivia
|
||||
for len(text) > 0 && (text[0] == ' ' || text[0] == '\t' || text[0] == '\n' || text[0] == '\r') {
|
||||
text = text[1:]
|
||||
}
|
||||
if len(text) >= 5 && text[:5] == "keyof" {
|
||||
node["operator"] = int(c.kindForName("KeyOfKeyword"))
|
||||
} else if len(text) >= 6 && text[:6] == "unique" {
|
||||
node["operator"] = int(c.kindForName("UniqueKeyword"))
|
||||
} else if len(text) >= 8 && text[:8] == "readonly" {
|
||||
node["operator"] = int(c.kindForName("ReadonlyKeyword"))
|
||||
}
|
||||
}
|
||||
if len(children) > 0 {
|
||||
child, err := c.convertNode(children[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
node["type"] = child
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MissingDeclaration: optional modifiers child
|
||||
if kindName == "MissingDeclaration" {
|
||||
if len(children) > 0 && c.ast.IsNodeList(children[0]) {
|
||||
arr, err := c.convertNodeList(children[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
node["modifiers"] = arr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unknown node kind with children — emit them as a generic "children" array
|
||||
arr := make([]interface{}, 0, len(children))
|
||||
for _, ci := range children {
|
||||
if c.ast.IsNodeList(ci) {
|
||||
nlArr, err := c.convertNodeList(ci)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range nlArr {
|
||||
arr = append(arr, item)
|
||||
}
|
||||
} else {
|
||||
child, err := c.convertNode(ci)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
arr = append(arr, child)
|
||||
}
|
||||
}
|
||||
if len(arr) > 0 {
|
||||
node["children"] = arr
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// assignChildProperties distributes children to named properties based on
|
||||
// the bitmask in the node's data field.
|
||||
func (c *Converter) assignChildProperties(nodeIdx int, kindName string, props []string, children []int, node map[string]interface{}) error {
|
||||
mask := c.ast.ChildMask(nodeIdx)
|
||||
definedBits := c.ast.DefinedBits(nodeIdx)
|
||||
|
||||
// Special handling for JSDocParameterTag/JSDocPropertyTag where
|
||||
// child order depends on isNameFirst
|
||||
if (kindName == "JSDocParameterTag" || kindName == "JSDocPropertyTag") && definedBits&2 != 0 {
|
||||
// isNameFirst=true: order is tagName, name, typeExpression, comment
|
||||
props = []string{"tagName", "name", "typeExpression", "comment"}
|
||||
}
|
||||
|
||||
childIdx := 0
|
||||
for bit, prop := range props {
|
||||
if bit < 8 && mask != 0 && mask&(1<<uint(bit)) == 0 {
|
||||
// Property not present per bitmask. For array properties,
|
||||
// emit an empty array (the Java extractor expects them).
|
||||
if isArrayProperty(prop) {
|
||||
node[prop] = []interface{}{}
|
||||
}
|
||||
continue
|
||||
}
|
||||
// If mask is 0 (single-child or no disambiguation needed), consume sequentially
|
||||
if childIdx >= len(children) {
|
||||
// No more children — emit empty arrays for remaining array properties
|
||||
if isArrayProperty(prop) {
|
||||
node[prop] = []interface{}{}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
ci := children[childIdx]
|
||||
childIdx++
|
||||
|
||||
if c.ast.IsNodeList(ci) {
|
||||
arr, err := c.convertNodeList(ci)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Filter out zero-width synthetic modifiers (TS7 adds these for
|
||||
// nested namespace bodies, but TS5/Node.js doesn't emit them).
|
||||
if prop == "modifiers" {
|
||||
filtered := make([]interface{}, 0, len(arr))
|
||||
for _, elem := range arr {
|
||||
if m, ok := elem.(map[string]interface{}); ok {
|
||||
pos, _ := m["$pos"].(int)
|
||||
end, _ := m["$end"].(int)
|
||||
if pos == end {
|
||||
continue // zero-width synthetic node
|
||||
}
|
||||
}
|
||||
filtered = append(filtered, elem)
|
||||
}
|
||||
if len(filtered) == 0 {
|
||||
continue // drop entirely
|
||||
}
|
||||
arr = filtered
|
||||
}
|
||||
node[prop] = arr
|
||||
} else {
|
||||
child, err := c.convertNode(ci)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Remap TS7 "postfixToken" (questionToken property) to the correct name
|
||||
// based on the actual token kind. TS7 uses a single PostfixToken
|
||||
// for what TS5 had as separate questionToken/exclamationToken.
|
||||
if prop == "questionToken" {
|
||||
childKind := c.ast.Kind(ci)
|
||||
exclamationKind := c.kindForName("ExclamationToken")
|
||||
if exclamationKind != 0 && childKind == exclamationKind {
|
||||
prop = "exclamationToken"
|
||||
}
|
||||
}
|
||||
node[prop] = child
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// isArrayProperty returns true for property names that should be empty arrays
|
||||
// (not omitted) when absent in the binary AST.
|
||||
func isArrayProperty(prop string) bool {
|
||||
return arrayProperties[prop]
|
||||
}
|
||||
|
||||
var arrayProperties = map[string]bool{
|
||||
"arguments": true,
|
||||
"declarations": true,
|
||||
"elements": true,
|
||||
"members": true,
|
||||
"parameters": true,
|
||||
"properties": true,
|
||||
}
|
||||
|
||||
// convertNodeList converts a NodeList into a JSON array.
|
||||
func (c *Converter) convertNodeList(i int) ([]interface{}, error) {
|
||||
children := c.ast.Children(i)
|
||||
arr := make([]interface{}, 0, len(children))
|
||||
for _, ci := range children {
|
||||
child, err := c.convertNode(ci)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
arr = append(arr, child)
|
||||
}
|
||||
return arr, nil
|
||||
}
|
||||
|
||||
// addDefinedBitProperties adds properties derived from the defined bits
|
||||
// (bits 24-29 of the data field) that aren't part of the child tree.
|
||||
func (c *Converter) addDefinedBitProperties(i int, kindName string, node map[string]interface{}) {
|
||||
definedBits := c.ast.DefinedBits(i)
|
||||
|
||||
switch kindName {
|
||||
case "ImportSpecifier", "ImportEqualsDeclaration", "ExportSpecifier", "ExportDeclaration":
|
||||
node["isTypeOnly"] = definedBits&1 != 0
|
||||
case "ImportClause":
|
||||
node["isTypeOnly"] = definedBits&1 != 0
|
||||
if definedBits&2 != 0 {
|
||||
node["phaseModifier"] = "defer"
|
||||
}
|
||||
case "ImportType":
|
||||
if definedBits&1 != 0 {
|
||||
node["isTypeOf"] = true
|
||||
} else {
|
||||
node["isTypeOf"] = false
|
||||
}
|
||||
case "ExportAssignment", "JSExportAssignment":
|
||||
if definedBits&1 != 0 {
|
||||
node["isExportEquals"] = true
|
||||
}
|
||||
case "VariableDeclarationList":
|
||||
// Determine $declarationKind from defined bits
|
||||
if definedBits&2 != 0 {
|
||||
node["$declarationKind"] = "const"
|
||||
} else if definedBits&1 != 0 {
|
||||
node["$declarationKind"] = "let"
|
||||
} else {
|
||||
node["$declarationKind"] = "var"
|
||||
}
|
||||
case "ImportAttributes":
|
||||
if definedBits&2 != 0 {
|
||||
node["token"] = c.kindForName("AssertKeyword")
|
||||
} else {
|
||||
node["token"] = c.kindForName("WithKeyword")
|
||||
}
|
||||
case "HeritageClause":
|
||||
// Token (extends/implements) is not in the binary encoding.
|
||||
// Infer from source text, skipping leading trivia.
|
||||
bytePos := utf16ToByte(int(c.ast.Pos(i)), c.byteOffsets)
|
||||
if c.sourceText != "" && bytePos < len(c.sourceText) {
|
||||
text := c.sourceText[bytePos:]
|
||||
// Skip whitespace/newlines
|
||||
for len(text) > 0 && (text[0] == ' ' || text[0] == '\t' || text[0] == '\n' || text[0] == '\r') {
|
||||
text = text[1:]
|
||||
}
|
||||
if len(text) >= 10 && text[:10] == "implements" {
|
||||
node["token"] = int(c.kindForName("ImplementsKeyword"))
|
||||
} else {
|
||||
node["token"] = int(c.kindForName("ExtendsKeyword"))
|
||||
}
|
||||
}
|
||||
case "JSDocParameterTag", "JSDocPropertyTag":
|
||||
if definedBits&1 != 0 {
|
||||
node["isBracketed"] = true
|
||||
}
|
||||
if definedBits&2 != 0 {
|
||||
node["isNameFirst"] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// augmentPos replicates the Node.js wrapper's $pos augmentation:
|
||||
// if skip is true, advances past leading whitespace, single-line comments (//),
|
||||
// and multi-line comments (/* */). This matches the TS5 Node.js wrapper regex:
|
||||
// /(?:\s|\/\/.*|\/\*[^]*?\*\/)*/g
|
||||
// Note: shebangs (#!) are NOT skipped — the TS5 regex does not match them.
|
||||
// Input pos is a UTF-16 code unit offset; returns a UTF-16 code unit offset.
|
||||
func (c *Converter) augmentPos(pos int, skip bool) int {
|
||||
if !skip || c.sourceText == "" {
|
||||
return pos
|
||||
}
|
||||
return byteToUTF16(c.skipTrivia(utf16ToByte(pos, c.byteOffsets)), c.utf16Offsets)
|
||||
}
|
||||
|
||||
// augmentBytePos converts a UTF-16 offset to byte offset then skips trivia,
|
||||
// returning the result as a byte offset. Used for scanner rescan events.
|
||||
func (c *Converter) augmentBytePos(utf16Pos int) int {
|
||||
return c.skipTrivia(utf16ToByte(utf16Pos, c.byteOffsets))
|
||||
}
|
||||
|
||||
// skipTrivia advances past whitespace, single-line comments (//), and
|
||||
// multi-line comments (/* */), starting at byte offset i.
|
||||
func (c *Converter) skipTrivia(i int) int {
|
||||
n := len(c.sourceText)
|
||||
for i < n {
|
||||
ch := c.sourceText[i]
|
||||
if ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == '\f' || ch == '\v' {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if ch == '/' && i+1 < n {
|
||||
next := c.sourceText[i+1]
|
||||
if next == '/' {
|
||||
// Single-line comment — skip to end of line
|
||||
i += 2
|
||||
for i < n && c.sourceText[i] != '\n' {
|
||||
i++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if next == '*' {
|
||||
// Multi-line comment — skip to */
|
||||
i += 2
|
||||
for i+1 < n {
|
||||
if c.sourceText[i] == '*' && c.sourceText[i+1] == '/' {
|
||||
i += 2
|
||||
break
|
||||
}
|
||||
i++
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
// computeLineStarts returns an array of UTF-16 code unit offsets where each line starts.
|
||||
func computeLineStarts(text string, utf16Offsets []int) []int {
|
||||
starts := []int{0}
|
||||
for i := 0; i < len(text); i++ {
|
||||
ch := text[i]
|
||||
if ch == '\n' {
|
||||
starts = append(starts, byteToUTF16(i+1, utf16Offsets))
|
||||
} else if ch == '\r' {
|
||||
if i+1 < len(text) && text[i+1] == '\n' {
|
||||
i++
|
||||
}
|
||||
starts = append(starts, byteToUTF16(i+1, utf16Offsets))
|
||||
}
|
||||
}
|
||||
return starts
|
||||
}
|
||||
|
||||
// buildOffsetTables builds bidirectional mapping tables between byte offsets
|
||||
// and UTF-16 code unit offsets.
|
||||
func buildOffsetTables(text string) (byteToUTF16Table []int, utf16ToByteTable []int) {
|
||||
byteToUTF16Table = make([]int, len(text)+1)
|
||||
// First pass: compute total UTF-16 length and byte→UTF-16 mapping
|
||||
utf16Pos := 0
|
||||
i := 0
|
||||
for i < len(text) {
|
||||
byteToUTF16Table[i] = utf16Pos
|
||||
b := text[i]
|
||||
if b < 0x80 {
|
||||
i++
|
||||
utf16Pos++
|
||||
} else if b < 0xE0 {
|
||||
if i+1 < len(byteToUTF16Table) {
|
||||
byteToUTF16Table[i+1] = utf16Pos
|
||||
}
|
||||
i += 2
|
||||
utf16Pos++
|
||||
} else if b < 0xF0 {
|
||||
if i+1 < len(byteToUTF16Table) {
|
||||
byteToUTF16Table[i+1] = utf16Pos
|
||||
}
|
||||
if i+2 < len(byteToUTF16Table) {
|
||||
byteToUTF16Table[i+2] = utf16Pos
|
||||
}
|
||||
i += 3
|
||||
utf16Pos++
|
||||
} else {
|
||||
// 4-byte UTF-8 = 2 UTF-16 code units (surrogate pair)
|
||||
for j := 1; j < 4 && i+j < len(byteToUTF16Table); j++ {
|
||||
byteToUTF16Table[i+j] = utf16Pos
|
||||
}
|
||||
i += 4
|
||||
utf16Pos += 2
|
||||
}
|
||||
}
|
||||
byteToUTF16Table[len(text)] = utf16Pos
|
||||
|
||||
// Second pass: build UTF-16→byte mapping
|
||||
utf16ToByteTable = make([]int, utf16Pos+1)
|
||||
i = 0
|
||||
utf16Pos = 0
|
||||
for i < len(text) {
|
||||
utf16ToByteTable[utf16Pos] = i
|
||||
b := text[i]
|
||||
if b < 0x80 {
|
||||
i++
|
||||
utf16Pos++
|
||||
} else if b < 0xE0 {
|
||||
i += 2
|
||||
utf16Pos++
|
||||
} else if b < 0xF0 {
|
||||
i += 3
|
||||
utf16Pos++
|
||||
} else {
|
||||
utf16ToByteTable[utf16Pos+1] = i
|
||||
i += 4
|
||||
utf16Pos += 2
|
||||
}
|
||||
}
|
||||
utf16ToByteTable[utf16Pos] = i
|
||||
return
|
||||
}
|
||||
|
||||
// byteToUTF16 converts a byte offset to a UTF-16 code unit offset.
|
||||
func byteToUTF16(byteOff int, table []int) int {
|
||||
if len(table) == 0 {
|
||||
return byteOff
|
||||
}
|
||||
if byteOff >= len(table) {
|
||||
return table[len(table)-1]
|
||||
}
|
||||
if byteOff < 0 {
|
||||
return 0
|
||||
}
|
||||
return table[byteOff]
|
||||
}
|
||||
|
||||
// utf16ToByte converts a UTF-16 code unit offset to a byte offset.
|
||||
func utf16ToByte(utf16Off int, table []int) int {
|
||||
if len(table) == 0 {
|
||||
return utf16Off
|
||||
}
|
||||
if utf16Off >= len(table) {
|
||||
return table[len(table)-1]
|
||||
}
|
||||
if utf16Off < 0 {
|
||||
return 0
|
||||
}
|
||||
return table[utf16Off]
|
||||
}
|
||||
|
||||
// kindForName returns the numeric kind for a given string name.
|
||||
// This is the reverse of kindNames. Returns 0 if not found.
|
||||
func (c *Converter) kindForName(name string) uint32 {
|
||||
for k, v := range c.kindNames {
|
||||
if v == name {
|
||||
return k
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// collectRescanEvents walks the AST to find positions that need rescanning.
|
||||
// This matches the Node.js wrapper's rescan logic in ast_extractor.ts.
|
||||
func (c *Converter) collectRescanEvents(root int) []RescanEvent {
|
||||
var events []RescanEvent
|
||||
c.walkForRescan(root, &events)
|
||||
// Sort by position
|
||||
sortRescanEvents(events)
|
||||
return events
|
||||
}
|
||||
|
||||
func (c *Converter) walkForRescan(i int, events *[]RescanEvent) {
|
||||
if i <= 0 || i >= c.ast.NodeCount() {
|
||||
return
|
||||
}
|
||||
if c.ast.IsNodeList(i) {
|
||||
for _, ci := range c.ast.Children(i) {
|
||||
c.walkForRescan(ci, events)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
kind := c.ast.Kind(i)
|
||||
kindName := c.kindNames[kind]
|
||||
|
||||
// RegularExpressionLiteral needs rescan (scanner sees / as SlashToken)
|
||||
if kindName == "RegularExpressionLiteral" {
|
||||
pos := c.augmentBytePos(int(c.ast.Pos(i)))
|
||||
*events = append(*events, RescanEvent{Pos: pos, Kind: "regex"})
|
||||
}
|
||||
|
||||
// TemplateMiddle and TemplateTail need rescan (scanner sees } as CloseBraceToken)
|
||||
if kindName == "TemplateMiddle" || kindName == "TemplateTail" {
|
||||
pos := c.augmentBytePos(int(c.ast.Pos(i)))
|
||||
*events = append(*events, RescanEvent{Pos: pos, Kind: "template"})
|
||||
}
|
||||
|
||||
// BinaryExpression with >>= or >>> etc. needs rescan (scanner may see > separately)
|
||||
if kindName == "BinaryExpression" {
|
||||
children := c.ast.Children(i)
|
||||
if len(children) >= 3 {
|
||||
// BinaryExpression children: left, operatorToken, right
|
||||
opKind := c.kindNames[c.ast.Kind(children[1])]
|
||||
switch opKind {
|
||||
case "GreaterThanEqualsToken", "GreaterThanGreaterThanEqualsToken",
|
||||
"GreaterThanGreaterThanGreaterThanEqualsToken",
|
||||
"GreaterThanGreaterThanGreaterThanToken", "GreaterThanGreaterThanToken":
|
||||
pos := c.augmentBytePos(int(c.ast.Pos(children[1])))
|
||||
*events = append(*events, RescanEvent{Pos: pos, Kind: "greater"})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Recurse into children
|
||||
for _, ci := range c.ast.Children(i) {
|
||||
c.walkForRescan(ci, events)
|
||||
}
|
||||
}
|
||||
|
||||
func sortRescanEvents(events []RescanEvent) {
|
||||
// Simple insertion sort — events are typically few
|
||||
for i := 1; i < len(events); i++ {
|
||||
key := events[i]
|
||||
j := i - 1
|
||||
for j >= 0 && events[j].Pos > key.Pos {
|
||||
events[j+1] = events[j]
|
||||
j--
|
||||
}
|
||||
events[j+1] = key
|
||||
}
|
||||
}
|
||||
|
||||
// FilterWhitelist removes properties from the converted AST that are not
|
||||
// in the property whitelist. This is applied recursively.
|
||||
func FilterWhitelist(obj map[string]interface{}) map[string]interface{} {
|
||||
result := make(map[string]interface{}, len(obj))
|
||||
for k, v := range obj {
|
||||
if !IsAllowedProperty(k) {
|
||||
continue
|
||||
}
|
||||
switch val := v.(type) {
|
||||
case map[string]interface{}:
|
||||
result[k] = FilterWhitelist(val)
|
||||
case []interface{}:
|
||||
result[k] = filterWhitelistArray(val)
|
||||
default:
|
||||
result[k] = v
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func filterWhitelistArray(arr []interface{}) []interface{} {
|
||||
result := make([]interface{}, len(arr))
|
||||
for i, v := range arr {
|
||||
if obj, ok := v.(map[string]interface{}); ok {
|
||||
result[i] = FilterWhitelist(obj)
|
||||
} else {
|
||||
result[i] = v
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// BuildKindToNameMap builds a reverse mapping from numeric kind to string name
|
||||
// from a SyntaxKinds metadata map (name → number).
|
||||
func BuildKindToNameMap(syntaxKinds map[string]int) map[uint32]string {
|
||||
result := make(map[uint32]string, len(syntaxKinds))
|
||||
for name, num := range syntaxKinds {
|
||||
key := uint32(num)
|
||||
// In case of collisions, prefer shorter/simpler names
|
||||
if existing, ok := result[key]; !ok || len(name) < len(existing) {
|
||||
result[key] = name
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// StripKindPrefix removes "Kind" prefix from names if present (for TS7 Go-style names).
|
||||
func StripKindPrefix(name string) string {
|
||||
if strings.HasPrefix(name, "Kind") {
|
||||
return name[4:]
|
||||
}
|
||||
return name
|
||||
}
|
||||
@@ -1,221 +0,0 @@
|
||||
// Package astconv decodes the binary AST format produced by the tsgo API
|
||||
// and converts it to the JSON format expected by the Java extractor.
|
||||
//
|
||||
// The binary format is documented in microsoft/typescript-go/internal/api/encoder/encoder.go.
|
||||
// Each source file is encoded as:
|
||||
//
|
||||
// Header (44 bytes) | String offsets | String data | Extended data | Structured data | Nodes (28 bytes each)
|
||||
//
|
||||
// Nodes are in a flat array with parent/next-sibling indices. The first node (index 0)
|
||||
// is a nil sentinel. The root node is at index 1.
|
||||
package astconv
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Binary format constants matching microsoft/typescript-go/internal/api/encoder.
|
||||
const (
|
||||
nodeSize = 28 // 7 × uint32
|
||||
|
||||
nodeOffsetKind = 0
|
||||
nodeOffsetPos = 4
|
||||
nodeOffsetEnd = 8
|
||||
nodeOffsetNext = 12
|
||||
nodeOffsetParent = 16
|
||||
nodeOffsetData = 20
|
||||
nodeOffsetFlags = 24
|
||||
|
||||
headerSize = 44
|
||||
headerOffsetMetadata = 0
|
||||
headerOffsetStringOff = 24
|
||||
headerOffsetStringData = 28
|
||||
headerOffsetExtData = 32
|
||||
headerOffsetStructData = 36
|
||||
headerOffsetNodes = 40
|
||||
|
||||
protocolVersion uint8 = 5
|
||||
|
||||
nodeDataTypeChildren uint32 = 0x00_00_00_00
|
||||
nodeDataTypeString uint32 = 0x40_00_00_00
|
||||
nodeDataTypeExtended uint32 = 0x80_00_00_00
|
||||
|
||||
nodeDataTypeMask uint32 = 0xC0_00_00_00
|
||||
nodeDataChildMask uint32 = 0x00_00_00_FF
|
||||
nodeDataStringMask uint32 = 0x00_FF_FF_FF
|
||||
|
||||
// SyntaxKindNodeList is the special kind value used for NodeList nodes.
|
||||
SyntaxKindNodeList uint32 = 0xFF_FF_FF_FF
|
||||
)
|
||||
|
||||
// BinaryAST provides random access to nodes in a binary-encoded TypeScript AST.
|
||||
type BinaryAST struct {
|
||||
raw []byte
|
||||
strOff uint32 // byte offset to string offset pairs
|
||||
strData uint32 // byte offset to string data
|
||||
extData uint32 // byte offset to extended node data
|
||||
structOff uint32 // byte offset to structured data
|
||||
nodeOff uint32 // byte offset to nodes section
|
||||
nodeCount int
|
||||
// Single Go string covering all data from strData onward.
|
||||
// String offsets index into this, so substrings are zero-alloc.
|
||||
allStrData string
|
||||
}
|
||||
|
||||
// DecodeBinaryAST parses the binary header and returns a BinaryAST for
|
||||
// random-access to nodes and strings.
|
||||
func DecodeBinaryAST(data []byte) (*BinaryAST, error) {
|
||||
if len(data) < headerSize {
|
||||
return nil, fmt.Errorf("data too short: %d bytes (need %d)", len(data), headerSize)
|
||||
}
|
||||
|
||||
version := data[headerOffsetMetadata+3]
|
||||
if version != protocolVersion {
|
||||
return nil, fmt.Errorf("unsupported protocol version %d (expected %d)", version, protocolVersion)
|
||||
}
|
||||
|
||||
b := &BinaryAST{
|
||||
raw: data,
|
||||
strOff: le32(data, headerOffsetStringOff),
|
||||
strData: le32(data, headerOffsetStringData),
|
||||
extData: le32(data, headerOffsetExtData),
|
||||
structOff: le32(data, headerOffsetStructData),
|
||||
nodeOff: le32(data, headerOffsetNodes),
|
||||
}
|
||||
|
||||
dataLen := uint32(len(data))
|
||||
if b.strOff > dataLen || b.strData > dataLen || b.extData > dataLen || b.nodeOff > dataLen {
|
||||
return nil, fmt.Errorf("invalid header offsets exceed data length %d", dataLen)
|
||||
}
|
||||
|
||||
b.nodeCount = (len(data) - int(b.nodeOff)) / nodeSize
|
||||
if b.nodeCount < 2 {
|
||||
return nil, fmt.Errorf("no nodes in AST (count=%d, need at least 2)", b.nodeCount)
|
||||
}
|
||||
|
||||
// The official decoder uses data[strData:] for zero-alloc substring slicing.
|
||||
b.allStrData = string(data[b.strData:])
|
||||
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// DecodeBinaryASTFromBase64 decodes a base64-encoded binary AST, as returned
|
||||
// by tsgo's getSourceFile API in JSON ({"data":"<base64>"}).
|
||||
func DecodeBinaryASTFromBase64(b64 string) (*BinaryAST, error) {
|
||||
data, err := base64.StdEncoding.DecodeString(b64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("base64 decode failed: %w", err)
|
||||
}
|
||||
return DecodeBinaryAST(data)
|
||||
}
|
||||
|
||||
// NodeCount returns the total number of nodes (including the nil sentinel at index 0).
|
||||
func (b *BinaryAST) NodeCount() int { return b.nodeCount }
|
||||
|
||||
// Node field accessors — all read uint32 from the nodes section.
|
||||
|
||||
func (b *BinaryAST) nf(i, offset int) uint32 {
|
||||
return le32(b.raw, int(b.nodeOff)+i*nodeSize+offset)
|
||||
}
|
||||
|
||||
// Kind returns the SyntaxKind of node i.
|
||||
func (b *BinaryAST) Kind(i int) uint32 { return b.nf(i, nodeOffsetKind) }
|
||||
|
||||
// Pos returns the start position (UTF-16 offset) of node i.
|
||||
func (b *BinaryAST) Pos(i int) uint32 { return b.nf(i, nodeOffsetPos) }
|
||||
|
||||
// End returns the end position (UTF-16 offset) of node i.
|
||||
func (b *BinaryAST) End(i int) uint32 { return b.nf(i, nodeOffsetEnd) }
|
||||
|
||||
// Next returns the index of the next sibling of node i, or 0 if none.
|
||||
func (b *BinaryAST) Next(i int) uint32 { return b.nf(i, nodeOffsetNext) }
|
||||
|
||||
// Parent returns the index of the parent of node i, or 0 if none.
|
||||
func (b *BinaryAST) Parent(i int) uint32 { return b.nf(i, nodeOffsetParent) }
|
||||
|
||||
// Data returns the raw 32-bit data field of node i.
|
||||
func (b *BinaryAST) Data(i int) uint32 { return b.nf(i, nodeOffsetData) }
|
||||
|
||||
// Flags returns the NodeFlags of node i.
|
||||
func (b *BinaryAST) Flags(i int) uint32 { return b.nf(i, nodeOffsetFlags) }
|
||||
|
||||
// DataType returns the top 2 bits of the data field (Children, String, or Extended).
|
||||
func (b *BinaryAST) DataType(i int) uint32 { return b.Data(i) & nodeDataTypeMask }
|
||||
|
||||
// DefinedBits returns bits 24-29 of the data field (6 bits of per-node-type flags).
|
||||
func (b *BinaryAST) DefinedBits(i int) uint8 { return uint8((b.Data(i) >> 24) & 0x3F) }
|
||||
|
||||
// ChildMask returns the low byte of the data field (child property bitmask).
|
||||
func (b *BinaryAST) ChildMask(i int) uint8 { return uint8(b.Data(i) & nodeDataChildMask) }
|
||||
|
||||
// StringIndex returns the 24-bit string table index from the data field.
|
||||
func (b *BinaryAST) StringIndex(i int) uint32 { return b.Data(i) & nodeDataStringMask }
|
||||
|
||||
// ExtOffset returns the 24-bit offset into the extended data section from the data field.
|
||||
func (b *BinaryAST) ExtOffset(i int) uint32 { return b.Data(i) & nodeDataStringMask }
|
||||
|
||||
// NodeListLen returns the number of children for a NodeList node (stored in data field).
|
||||
func (b *BinaryAST) NodeListLen(i int) uint32 { return b.Data(i) }
|
||||
|
||||
// IsNodeList returns true if node i is a NodeList.
|
||||
func (b *BinaryAST) IsNodeList(i int) bool { return b.Kind(i) == SyntaxKindNodeList }
|
||||
|
||||
// GetString reads a string from the string table at the given offset index.
|
||||
// The index comes from a String-type node's data field (24-bit value).
|
||||
func (b *BinaryAST) GetString(idx uint32) string {
|
||||
// Each string entry is two uint32 values (start, end) in the string offsets section.
|
||||
offBase := int(b.strOff) + int(idx)*4
|
||||
start := le32(b.raw, offBase)
|
||||
end := le32(b.raw, offBase+4)
|
||||
return b.allStrData[start:end]
|
||||
}
|
||||
|
||||
// ExtUint32 reads a uint32 from the extended data section at the given byte offset.
|
||||
func (b *BinaryAST) ExtUint32(off uint32) uint32 {
|
||||
return le32(b.raw, int(b.extData)+int(off))
|
||||
}
|
||||
|
||||
// Children returns the indices of all direct children of node i.
|
||||
// Children are identified by having parent == i. The first child is at i+1
|
||||
// (if its parent is i), and subsequent children are found via Next pointers.
|
||||
func (b *BinaryAST) Children(i int) []int {
|
||||
if i+1 >= b.nodeCount {
|
||||
return nil
|
||||
}
|
||||
firstChild := i + 1
|
||||
if b.Parent(firstChild) != uint32(i) {
|
||||
return nil
|
||||
}
|
||||
children := []int{firstChild}
|
||||
next := int(b.Next(firstChild))
|
||||
for next != 0 {
|
||||
children = append(children, next)
|
||||
next = int(b.Next(next))
|
||||
}
|
||||
return children
|
||||
}
|
||||
|
||||
// SourceText returns the source file text, extracted from the SourceFile's
|
||||
// extended data. Returns "" if the root node is not a SourceFile or if
|
||||
// the extended data is missing.
|
||||
func (b *BinaryAST) SourceText() string {
|
||||
if b.nodeCount < 2 {
|
||||
return ""
|
||||
}
|
||||
// Root is at index 1. Check if it has extended data type.
|
||||
if b.DataType(1)&nodeDataTypeMask != nodeDataTypeExtended {
|
||||
return ""
|
||||
}
|
||||
extOff := b.ExtOffset(1)
|
||||
textIdx := b.ExtUint32(extOff)
|
||||
return b.GetString(textIdx)
|
||||
}
|
||||
|
||||
func le32(data []byte, offset int) uint32 {
|
||||
if offset < 0 || offset+4 > len(data) {
|
||||
return 0
|
||||
}
|
||||
return binary.LittleEndian.Uint32(data[offset : offset+4])
|
||||
}
|
||||
@@ -1,876 +0,0 @@
|
||||
package astconv
|
||||
|
||||
import (
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// TS7 SyntaxKind values for tokens (from microsoft/typescript-go internal/ast/kind.go).
|
||||
const (
|
||||
KindUnknown = 0
|
||||
KindEndOfFile = 1
|
||||
KindSingleLineCommentTrivia = 2
|
||||
KindMultiLineCommentTrivia = 3
|
||||
KindNewLineTrivia = 4
|
||||
KindWhitespaceTrivia = 5
|
||||
KindShebangTrivia = 6
|
||||
KindConflictMarkerTrivia = 7
|
||||
KindNumericLiteral = 8
|
||||
KindBigIntLiteral = 9
|
||||
KindStringLiteral = 10
|
||||
KindRegularExpressionLiteral = 13
|
||||
KindNoSubstitutionTemplateLiteral = 14
|
||||
KindTemplateHead = 15
|
||||
KindTemplateMiddle = 16
|
||||
KindTemplateTail = 17
|
||||
KindOpenBraceToken = 18
|
||||
KindCloseBraceToken = 19
|
||||
KindOpenParenToken = 20
|
||||
KindCloseParenToken = 21
|
||||
KindOpenBracketToken = 22
|
||||
KindCloseBracketToken = 23
|
||||
KindDotToken = 24
|
||||
KindDotDotDotToken = 25
|
||||
KindSemicolonToken = 26
|
||||
KindCommaToken = 27
|
||||
KindQuestionDotToken = 28
|
||||
KindLessThanToken = 29
|
||||
KindLessThanSlashToken = 30
|
||||
KindGreaterThanToken = 31
|
||||
KindLessThanEqualsToken = 32
|
||||
KindGreaterThanEqualsToken = 33
|
||||
KindEqualsEqualsToken = 34
|
||||
KindExclamationEqualsToken = 35
|
||||
KindEqualsEqualsEqualsToken = 36
|
||||
KindExclamationEqualsEqualsToken = 37
|
||||
KindEqualsGreaterThanToken = 38
|
||||
KindPlusToken = 39
|
||||
KindMinusToken = 40
|
||||
KindAsteriskToken = 41
|
||||
KindAsteriskAsteriskToken = 42
|
||||
KindSlashToken = 43
|
||||
KindPercentToken = 44
|
||||
KindPlusPlusToken = 45
|
||||
KindMinusMinusToken = 46
|
||||
KindLessThanLessThanToken = 47
|
||||
KindGreaterThanGreaterThanToken = 48
|
||||
KindGreaterThanGreaterThanGreaterThanToken = 49
|
||||
KindAmpersandToken = 50
|
||||
KindBarToken = 51
|
||||
KindCaretToken = 52
|
||||
KindExclamationToken = 53
|
||||
KindTildeToken = 54
|
||||
KindAmpersandAmpersandToken = 55
|
||||
KindBarBarToken = 56
|
||||
KindQuestionToken = 57
|
||||
KindColonToken = 58
|
||||
KindAtToken = 59
|
||||
KindQuestionQuestionToken = 60
|
||||
KindHashToken = 62
|
||||
KindEqualsToken = 63
|
||||
KindPlusEqualsToken = 64
|
||||
KindMinusEqualsToken = 65
|
||||
KindAsteriskEqualsToken = 66
|
||||
KindAsteriskAsteriskEqualsToken = 67
|
||||
KindSlashEqualsToken = 68
|
||||
KindPercentEqualsToken = 69
|
||||
KindLessThanLessThanEqualsToken = 70
|
||||
KindGreaterThanGreaterThanEqualsToken = 71
|
||||
KindGreaterThanGreaterThanGreaterThanEqualsToken = 72
|
||||
KindAmpersandEqualsToken = 73
|
||||
KindBarEqualsToken = 74
|
||||
KindBarBarEqualsToken = 75
|
||||
KindAmpersandAmpersandEqualsToken = 76
|
||||
KindQuestionQuestionEqualsToken = 77
|
||||
KindCaretEqualsToken = 78
|
||||
KindIdentifier = 79
|
||||
KindPrivateIdentifier = 80
|
||||
)
|
||||
|
||||
// Token represents a single token from the scanner.
|
||||
type Token struct {
|
||||
Kind int `json:"kind"`
|
||||
TokenPos int `json:"tokenPos"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// RescanEvent tells the scanner to rescan at a given position.
|
||||
type RescanEvent struct {
|
||||
Pos int
|
||||
Kind string // "regex", "template", "greater"
|
||||
}
|
||||
|
||||
// Scanner tokenizes TypeScript source text.
|
||||
type Scanner struct {
|
||||
text string
|
||||
pos int
|
||||
events []RescanEvent
|
||||
evIdx int
|
||||
}
|
||||
|
||||
// NewScanner creates a scanner for the given source text.
|
||||
// rescanEvents should be sorted by position. They inform the scanner
|
||||
// about positions where regex literals, template tokens, or greater-than
|
||||
// rescanning is needed (matching the Node.js wrapper behavior).
|
||||
func NewScanner(text string, rescanEvents []RescanEvent) *Scanner {
|
||||
return &Scanner{
|
||||
text: text,
|
||||
pos: 0,
|
||||
events: rescanEvents,
|
||||
evIdx: 0,
|
||||
}
|
||||
}
|
||||
|
||||
// ScanAll produces all tokens from the source text, including trivia
|
||||
// (whitespace, newlines, comments), matching the Node.js wrapper behavior.
|
||||
func (s *Scanner) ScanAll() []Token {
|
||||
var tokens []Token
|
||||
for {
|
||||
tok := s.scan()
|
||||
tokens = append(tokens, tok)
|
||||
if tok.Kind == KindEndOfFile {
|
||||
break
|
||||
}
|
||||
}
|
||||
return tokens
|
||||
}
|
||||
|
||||
func (s *Scanner) peek() byte {
|
||||
if s.pos >= len(s.text) {
|
||||
return 0
|
||||
}
|
||||
return s.text[s.pos]
|
||||
}
|
||||
|
||||
func (s *Scanner) peekAt(offset int) byte {
|
||||
p := s.pos + offset
|
||||
if p >= len(s.text) {
|
||||
return 0
|
||||
}
|
||||
return s.text[p]
|
||||
}
|
||||
|
||||
func (s *Scanner) advance() {
|
||||
s.pos++
|
||||
}
|
||||
|
||||
func (s *Scanner) nextRescanPos() int {
|
||||
if s.evIdx < len(s.events) {
|
||||
return s.events[s.evIdx].Pos
|
||||
}
|
||||
return int(^uint(0) >> 1) // MaxInt
|
||||
}
|
||||
|
||||
func (s *Scanner) nextRescanKind() string {
|
||||
if s.evIdx < len(s.events) {
|
||||
return s.events[s.evIdx].Kind
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *Scanner) consumeRescan() {
|
||||
if s.evIdx < len(s.events) {
|
||||
s.evIdx++
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scanner) scan() Token {
|
||||
if s.pos >= len(s.text) {
|
||||
return Token{Kind: KindEndOfFile, TokenPos: s.pos, Text: ""}
|
||||
}
|
||||
|
||||
tokenPos := s.pos
|
||||
ch := s.peek()
|
||||
|
||||
// Whitespace (not newlines)
|
||||
if ch == ' ' || ch == '\t' || ch == '\f' || ch == '\v' {
|
||||
for s.pos < len(s.text) {
|
||||
c := s.text[s.pos]
|
||||
if c == ' ' || c == '\t' || c == '\f' || c == '\v' {
|
||||
s.pos++
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
return Token{Kind: KindWhitespaceTrivia, TokenPos: tokenPos, Text: s.text[tokenPos:s.pos]}
|
||||
}
|
||||
|
||||
// Newlines
|
||||
if ch == '\n' {
|
||||
s.advance()
|
||||
return Token{Kind: KindNewLineTrivia, TokenPos: tokenPos, Text: "\n"}
|
||||
}
|
||||
if ch == '\r' {
|
||||
s.advance()
|
||||
if s.peek() == '\n' {
|
||||
s.advance()
|
||||
}
|
||||
return Token{Kind: KindNewLineTrivia, TokenPos: tokenPos, Text: s.text[tokenPos:s.pos]}
|
||||
}
|
||||
|
||||
// Check for rescan event at this position.
|
||||
// TS5's scanner loop captures the token kind BEFORE the rescan event fires,
|
||||
// then uses the rescanned text. So regex tokens get kind=SlashToken with
|
||||
// text="/pattern/flags", and template continuation tokens get kind=CloseBraceToken
|
||||
// with the template text. We replicate this by scanning the full content but
|
||||
// using the pre-rescan kind.
|
||||
if tokenPos == s.nextRescanPos() {
|
||||
kind := s.nextRescanKind()
|
||||
s.consumeRescan()
|
||||
switch kind {
|
||||
case "regex":
|
||||
tok := s.scanRegExp(tokenPos)
|
||||
tok.Kind = KindSlashToken
|
||||
return tok
|
||||
case "template":
|
||||
tok := s.scanTemplatePart(tokenPos, true)
|
||||
tok.Kind = KindCloseBraceToken
|
||||
return tok
|
||||
case "greater":
|
||||
return s.scanGreater(tokenPos)
|
||||
}
|
||||
}
|
||||
|
||||
switch ch {
|
||||
case '/':
|
||||
next := s.peekAt(1)
|
||||
if next == '/' {
|
||||
return s.scanSingleLineComment(tokenPos)
|
||||
}
|
||||
if next == '*' {
|
||||
return s.scanMultiLineComment(tokenPos)
|
||||
}
|
||||
if next == '=' {
|
||||
s.pos += 2
|
||||
return Token{Kind: KindSlashEqualsToken, TokenPos: tokenPos, Text: "/="}
|
||||
}
|
||||
s.advance()
|
||||
return Token{Kind: KindSlashToken, TokenPos: tokenPos, Text: "/"}
|
||||
|
||||
case '\'', '"':
|
||||
return s.scanString(tokenPos, ch)
|
||||
|
||||
case '`':
|
||||
return s.scanTemplatePart(tokenPos, false)
|
||||
|
||||
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
|
||||
return s.scanNumber(tokenPos)
|
||||
|
||||
case '{':
|
||||
s.advance()
|
||||
return Token{Kind: KindOpenBraceToken, TokenPos: tokenPos, Text: "{"}
|
||||
case '}':
|
||||
s.advance()
|
||||
return Token{Kind: KindCloseBraceToken, TokenPos: tokenPos, Text: "}"}
|
||||
case '(':
|
||||
s.advance()
|
||||
return Token{Kind: KindOpenParenToken, TokenPos: tokenPos, Text: "("}
|
||||
case ')':
|
||||
s.advance()
|
||||
return Token{Kind: KindCloseParenToken, TokenPos: tokenPos, Text: ")"}
|
||||
case '[':
|
||||
s.advance()
|
||||
return Token{Kind: KindOpenBracketToken, TokenPos: tokenPos, Text: "["}
|
||||
case ']':
|
||||
s.advance()
|
||||
return Token{Kind: KindCloseBracketToken, TokenPos: tokenPos, Text: "]"}
|
||||
case ';':
|
||||
s.advance()
|
||||
return Token{Kind: KindSemicolonToken, TokenPos: tokenPos, Text: ";"}
|
||||
case ',':
|
||||
s.advance()
|
||||
return Token{Kind: KindCommaToken, TokenPos: tokenPos, Text: ","}
|
||||
case '~':
|
||||
s.advance()
|
||||
return Token{Kind: KindTildeToken, TokenPos: tokenPos, Text: "~"}
|
||||
case '@':
|
||||
s.advance()
|
||||
return Token{Kind: KindAtToken, TokenPos: tokenPos, Text: "@"}
|
||||
|
||||
case '.':
|
||||
if s.peekAt(1) == '.' && s.peekAt(2) == '.' {
|
||||
s.pos += 3
|
||||
return Token{Kind: KindDotDotDotToken, TokenPos: tokenPos, Text: "..."}
|
||||
}
|
||||
// .123 numeric literal
|
||||
if s.peekAt(1) >= '0' && s.peekAt(1) <= '9' {
|
||||
return s.scanNumber(tokenPos)
|
||||
}
|
||||
s.advance()
|
||||
return Token{Kind: KindDotToken, TokenPos: tokenPos, Text: "."}
|
||||
|
||||
case ':':
|
||||
s.advance()
|
||||
return Token{Kind: KindColonToken, TokenPos: tokenPos, Text: ":"}
|
||||
|
||||
case '?':
|
||||
if s.peekAt(1) == '.' && !(s.peekAt(2) >= '0' && s.peekAt(2) <= '9') {
|
||||
s.pos += 2
|
||||
return Token{Kind: KindQuestionDotToken, TokenPos: tokenPos, Text: "?."}
|
||||
}
|
||||
if s.peekAt(1) == '?' {
|
||||
if s.peekAt(2) == '=' {
|
||||
s.pos += 3
|
||||
return Token{Kind: KindQuestionQuestionEqualsToken, TokenPos: tokenPos, Text: "??="}
|
||||
}
|
||||
s.pos += 2
|
||||
return Token{Kind: KindQuestionQuestionToken, TokenPos: tokenPos, Text: "??"}
|
||||
}
|
||||
s.advance()
|
||||
return Token{Kind: KindQuestionToken, TokenPos: tokenPos, Text: "?"}
|
||||
|
||||
case '!':
|
||||
if s.peekAt(1) == '=' {
|
||||
if s.peekAt(2) == '=' {
|
||||
s.pos += 3
|
||||
return Token{Kind: KindExclamationEqualsEqualsToken, TokenPos: tokenPos, Text: "!=="}
|
||||
}
|
||||
s.pos += 2
|
||||
return Token{Kind: KindExclamationEqualsToken, TokenPos: tokenPos, Text: "!="}
|
||||
}
|
||||
s.advance()
|
||||
return Token{Kind: KindExclamationToken, TokenPos: tokenPos, Text: "!"}
|
||||
|
||||
case '=':
|
||||
if s.peekAt(1) == '=' {
|
||||
if s.peekAt(2) == '=' {
|
||||
s.pos += 3
|
||||
return Token{Kind: KindEqualsEqualsEqualsToken, TokenPos: tokenPos, Text: "==="}
|
||||
}
|
||||
s.pos += 2
|
||||
return Token{Kind: KindEqualsEqualsToken, TokenPos: tokenPos, Text: "=="}
|
||||
}
|
||||
if s.peekAt(1) == '>' {
|
||||
s.pos += 2
|
||||
return Token{Kind: KindEqualsGreaterThanToken, TokenPos: tokenPos, Text: "=>"}
|
||||
}
|
||||
s.advance()
|
||||
return Token{Kind: KindEqualsToken, TokenPos: tokenPos, Text: "="}
|
||||
|
||||
case '+':
|
||||
if s.peekAt(1) == '+' {
|
||||
s.pos += 2
|
||||
return Token{Kind: KindPlusPlusToken, TokenPos: tokenPos, Text: "++"}
|
||||
}
|
||||
if s.peekAt(1) == '=' {
|
||||
s.pos += 2
|
||||
return Token{Kind: KindPlusEqualsToken, TokenPos: tokenPos, Text: "+="}
|
||||
}
|
||||
s.advance()
|
||||
return Token{Kind: KindPlusToken, TokenPos: tokenPos, Text: "+"}
|
||||
|
||||
case '-':
|
||||
if s.peekAt(1) == '-' {
|
||||
s.pos += 2
|
||||
return Token{Kind: KindMinusMinusToken, TokenPos: tokenPos, Text: "--"}
|
||||
}
|
||||
if s.peekAt(1) == '=' {
|
||||
s.pos += 2
|
||||
return Token{Kind: KindMinusEqualsToken, TokenPos: tokenPos, Text: "-="}
|
||||
}
|
||||
s.advance()
|
||||
return Token{Kind: KindMinusToken, TokenPos: tokenPos, Text: "-"}
|
||||
|
||||
case '*':
|
||||
if s.peekAt(1) == '*' {
|
||||
if s.peekAt(2) == '=' {
|
||||
s.pos += 3
|
||||
return Token{Kind: KindAsteriskAsteriskEqualsToken, TokenPos: tokenPos, Text: "**="}
|
||||
}
|
||||
s.pos += 2
|
||||
return Token{Kind: KindAsteriskAsteriskToken, TokenPos: tokenPos, Text: "**"}
|
||||
}
|
||||
if s.peekAt(1) == '=' {
|
||||
s.pos += 2
|
||||
return Token{Kind: KindAsteriskEqualsToken, TokenPos: tokenPos, Text: "*="}
|
||||
}
|
||||
s.advance()
|
||||
return Token{Kind: KindAsteriskToken, TokenPos: tokenPos, Text: "*"}
|
||||
|
||||
case '%':
|
||||
if s.peekAt(1) == '=' {
|
||||
s.pos += 2
|
||||
return Token{Kind: KindPercentEqualsToken, TokenPos: tokenPos, Text: "%="}
|
||||
}
|
||||
s.advance()
|
||||
return Token{Kind: KindPercentToken, TokenPos: tokenPos, Text: "%"}
|
||||
|
||||
case '<':
|
||||
if s.peekAt(1) == '<' {
|
||||
if s.peekAt(2) == '=' {
|
||||
s.pos += 3
|
||||
return Token{Kind: KindLessThanLessThanEqualsToken, TokenPos: tokenPos, Text: "<<="}
|
||||
}
|
||||
s.pos += 2
|
||||
return Token{Kind: KindLessThanLessThanToken, TokenPos: tokenPos, Text: "<<"}
|
||||
}
|
||||
if s.peekAt(1) == '/' {
|
||||
s.pos += 2
|
||||
return Token{Kind: KindLessThanSlashToken, TokenPos: tokenPos, Text: "</"}
|
||||
}
|
||||
if s.peekAt(1) == '=' {
|
||||
s.pos += 2
|
||||
return Token{Kind: KindLessThanEqualsToken, TokenPos: tokenPos, Text: "<="}
|
||||
}
|
||||
s.advance()
|
||||
return Token{Kind: KindLessThanToken, TokenPos: tokenPos, Text: "<"}
|
||||
|
||||
case '>':
|
||||
// TypeScript scanner always produces single > tokens.
|
||||
// Multi-character operators (>>, >>>, >>=, etc.) are produced
|
||||
// only via reScanGreaterToken when the parser requests it.
|
||||
s.advance()
|
||||
return Token{Kind: KindGreaterThanToken, TokenPos: tokenPos, Text: ">"}
|
||||
|
||||
case '&':
|
||||
if s.peekAt(1) == '&' {
|
||||
if s.peekAt(2) == '=' {
|
||||
s.pos += 3
|
||||
return Token{Kind: KindAmpersandAmpersandEqualsToken, TokenPos: tokenPos, Text: "&&="}
|
||||
}
|
||||
s.pos += 2
|
||||
return Token{Kind: KindAmpersandAmpersandToken, TokenPos: tokenPos, Text: "&&"}
|
||||
}
|
||||
if s.peekAt(1) == '=' {
|
||||
s.pos += 2
|
||||
return Token{Kind: KindAmpersandEqualsToken, TokenPos: tokenPos, Text: "&="}
|
||||
}
|
||||
s.advance()
|
||||
return Token{Kind: KindAmpersandToken, TokenPos: tokenPos, Text: "&"}
|
||||
|
||||
case '|':
|
||||
if s.peekAt(1) == '|' {
|
||||
if s.peekAt(2) == '=' {
|
||||
s.pos += 3
|
||||
return Token{Kind: KindBarBarEqualsToken, TokenPos: tokenPos, Text: "||="}
|
||||
}
|
||||
s.pos += 2
|
||||
return Token{Kind: KindBarBarToken, TokenPos: tokenPos, Text: "||"}
|
||||
}
|
||||
if s.peekAt(1) == '=' {
|
||||
s.pos += 2
|
||||
return Token{Kind: KindBarEqualsToken, TokenPos: tokenPos, Text: "|="}
|
||||
}
|
||||
s.advance()
|
||||
return Token{Kind: KindBarToken, TokenPos: tokenPos, Text: "|"}
|
||||
|
||||
case '^':
|
||||
if s.peekAt(1) == '=' {
|
||||
s.pos += 2
|
||||
return Token{Kind: KindCaretEqualsToken, TokenPos: tokenPos, Text: "^="}
|
||||
}
|
||||
s.advance()
|
||||
return Token{Kind: KindCaretToken, TokenPos: tokenPos, Text: "^"}
|
||||
|
||||
case '#':
|
||||
// Could be private identifier
|
||||
if s.peekAt(1) == '!' && tokenPos == 0 {
|
||||
// Shebang — scan to end of line, emit as ShebangTrivia
|
||||
start := s.pos
|
||||
for s.pos < len(s.text) && s.text[s.pos] != '\n' && s.text[s.pos] != '\r' {
|
||||
s.pos++
|
||||
}
|
||||
text := s.text[start:s.pos]
|
||||
return Token{Kind: KindShebangTrivia, TokenPos: tokenPos, Text: text}
|
||||
}
|
||||
if isIdentStart(s.peekAt(1)) {
|
||||
return s.scanPrivateIdentifier(tokenPos)
|
||||
}
|
||||
s.advance()
|
||||
return Token{Kind: KindHashToken, TokenPos: tokenPos, Text: "#"}
|
||||
}
|
||||
|
||||
// Identifier or keyword
|
||||
if isIdentStartByte(ch) {
|
||||
return s.scanIdentifierOrKeyword(tokenPos)
|
||||
}
|
||||
|
||||
// Handle multi-byte Unicode identifier starts
|
||||
r, size := utf8.DecodeRuneInString(s.text[s.pos:])
|
||||
if r != utf8.RuneError && isIdentStartRune(r) {
|
||||
return s.scanIdentifierOrKeyword(tokenPos)
|
||||
}
|
||||
|
||||
// Unknown character
|
||||
s.pos += size
|
||||
return Token{Kind: KindUnknown, TokenPos: tokenPos, Text: s.text[tokenPos:s.pos]}
|
||||
}
|
||||
|
||||
func (s *Scanner) scanSingleLineComment(start int) Token {
|
||||
s.pos += 2 // skip //
|
||||
for s.pos < len(s.text) && s.text[s.pos] != '\n' && s.text[s.pos] != '\r' {
|
||||
s.pos++
|
||||
}
|
||||
return Token{Kind: KindSingleLineCommentTrivia, TokenPos: start, Text: s.text[start:s.pos]}
|
||||
}
|
||||
|
||||
func (s *Scanner) scanMultiLineComment(start int) Token {
|
||||
s.pos += 2 // skip /*
|
||||
for s.pos < len(s.text)-1 {
|
||||
if s.text[s.pos] == '*' && s.text[s.pos+1] == '/' {
|
||||
s.pos += 2
|
||||
return Token{Kind: KindMultiLineCommentTrivia, TokenPos: start, Text: s.text[start:s.pos]}
|
||||
}
|
||||
s.pos++
|
||||
}
|
||||
// Unterminated
|
||||
s.pos = len(s.text)
|
||||
return Token{Kind: KindMultiLineCommentTrivia, TokenPos: start, Text: s.text[start:s.pos]}
|
||||
}
|
||||
|
||||
func (s *Scanner) scanString(start int, quote byte) Token {
|
||||
s.advance() // skip opening quote
|
||||
for s.pos < len(s.text) {
|
||||
ch := s.text[s.pos]
|
||||
if ch == '\\' {
|
||||
s.pos += 2
|
||||
continue
|
||||
}
|
||||
if ch == quote {
|
||||
s.advance()
|
||||
return Token{Kind: KindStringLiteral, TokenPos: start, Text: s.text[start:s.pos]}
|
||||
}
|
||||
if ch == '\n' || ch == '\r' {
|
||||
// Unterminated string
|
||||
break
|
||||
}
|
||||
s.pos++
|
||||
}
|
||||
return Token{Kind: KindStringLiteral, TokenPos: start, Text: s.text[start:s.pos]}
|
||||
}
|
||||
|
||||
func (s *Scanner) scanTemplatePart(start int, isRescan bool) Token {
|
||||
if isRescan {
|
||||
// We're at a '}' that needs to be rescanned as TemplateMiddle or TemplateTail
|
||||
s.advance() // skip }
|
||||
} else {
|
||||
s.advance() // skip `
|
||||
}
|
||||
for s.pos < len(s.text) {
|
||||
ch := s.text[s.pos]
|
||||
if ch == '\\' {
|
||||
s.pos += 2
|
||||
continue
|
||||
}
|
||||
if ch == '`' {
|
||||
s.advance()
|
||||
if isRescan {
|
||||
return Token{Kind: KindTemplateTail, TokenPos: start, Text: s.text[start:s.pos]}
|
||||
}
|
||||
return Token{Kind: KindNoSubstitutionTemplateLiteral, TokenPos: start, Text: s.text[start:s.pos]}
|
||||
}
|
||||
if ch == '$' && s.peekAt(1) == '{' {
|
||||
s.pos += 2
|
||||
if isRescan {
|
||||
return Token{Kind: KindTemplateMiddle, TokenPos: start, Text: s.text[start:s.pos]}
|
||||
}
|
||||
return Token{Kind: KindTemplateHead, TokenPos: start, Text: s.text[start:s.pos]}
|
||||
}
|
||||
s.pos++
|
||||
}
|
||||
// Unterminated
|
||||
if isRescan {
|
||||
return Token{Kind: KindTemplateTail, TokenPos: start, Text: s.text[start:s.pos]}
|
||||
}
|
||||
return Token{Kind: KindNoSubstitutionTemplateLiteral, TokenPos: start, Text: s.text[start:s.pos]}
|
||||
}
|
||||
|
||||
func (s *Scanner) scanRegExp(start int) Token {
|
||||
s.advance() // skip /
|
||||
inCharClass := false
|
||||
for s.pos < len(s.text) {
|
||||
ch := s.text[s.pos]
|
||||
if ch == '\\' {
|
||||
s.pos += 2
|
||||
continue
|
||||
}
|
||||
if ch == '[' {
|
||||
inCharClass = true
|
||||
s.pos++
|
||||
continue
|
||||
}
|
||||
if ch == ']' {
|
||||
inCharClass = false
|
||||
s.pos++
|
||||
continue
|
||||
}
|
||||
if ch == '/' && !inCharClass {
|
||||
s.advance() // skip closing /
|
||||
// Scan flags
|
||||
for s.pos < len(s.text) && isIdentChar(s.text[s.pos]) {
|
||||
s.pos++
|
||||
}
|
||||
return Token{Kind: KindRegularExpressionLiteral, TokenPos: start, Text: s.text[start:s.pos]}
|
||||
}
|
||||
if ch == '\n' || ch == '\r' {
|
||||
break
|
||||
}
|
||||
s.pos++
|
||||
}
|
||||
return Token{Kind: KindRegularExpressionLiteral, TokenPos: start, Text: s.text[start:s.pos]}
|
||||
}
|
||||
|
||||
func (s *Scanner) scanGreater(start int) Token {
|
||||
s.advance() // skip >
|
||||
if s.peek() == '>' {
|
||||
s.advance()
|
||||
if s.peek() == '>' {
|
||||
s.advance()
|
||||
if s.peek() == '=' {
|
||||
s.advance()
|
||||
return Token{Kind: KindGreaterThanGreaterThanGreaterThanEqualsToken, TokenPos: start, Text: ">>>="}
|
||||
}
|
||||
return Token{Kind: KindGreaterThanGreaterThanGreaterThanToken, TokenPos: start, Text: ">>>"}
|
||||
}
|
||||
if s.peek() == '=' {
|
||||
s.advance()
|
||||
return Token{Kind: KindGreaterThanGreaterThanEqualsToken, TokenPos: start, Text: ">>="}
|
||||
}
|
||||
return Token{Kind: KindGreaterThanGreaterThanToken, TokenPos: start, Text: ">>"}
|
||||
}
|
||||
if s.peek() == '=' {
|
||||
s.advance()
|
||||
return Token{Kind: KindGreaterThanEqualsToken, TokenPos: start, Text: ">="}
|
||||
}
|
||||
return Token{Kind: KindGreaterThanToken, TokenPos: start, Text: ">"}
|
||||
}
|
||||
|
||||
func (s *Scanner) scanNumber(start int) Token {
|
||||
if s.peek() == '0' {
|
||||
next := s.peekAt(1)
|
||||
if next == 'x' || next == 'X' {
|
||||
s.pos += 2
|
||||
s.scanHexDigits()
|
||||
return s.finishBigIntOrNumber(start)
|
||||
}
|
||||
if next == 'b' || next == 'B' {
|
||||
s.pos += 2
|
||||
s.scanBinaryDigits()
|
||||
return s.finishBigIntOrNumber(start)
|
||||
}
|
||||
if next == 'o' || next == 'O' {
|
||||
s.pos += 2
|
||||
s.scanOctalDigits()
|
||||
return s.finishBigIntOrNumber(start)
|
||||
}
|
||||
}
|
||||
|
||||
s.scanDecimalDigits()
|
||||
if s.peek() == '.' {
|
||||
s.advance()
|
||||
s.scanDecimalDigits()
|
||||
}
|
||||
if s.peek() == 'e' || s.peek() == 'E' {
|
||||
s.advance()
|
||||
if s.peek() == '+' || s.peek() == '-' {
|
||||
s.advance()
|
||||
}
|
||||
s.scanDecimalDigits()
|
||||
}
|
||||
return s.finishBigIntOrNumber(start)
|
||||
}
|
||||
|
||||
func (s *Scanner) finishBigIntOrNumber(start int) Token {
|
||||
if s.peek() == 'n' {
|
||||
s.advance()
|
||||
return Token{Kind: KindBigIntLiteral, TokenPos: start, Text: s.text[start:s.pos]}
|
||||
}
|
||||
return Token{Kind: KindNumericLiteral, TokenPos: start, Text: s.text[start:s.pos]}
|
||||
}
|
||||
|
||||
func (s *Scanner) scanDecimalDigits() {
|
||||
for s.pos < len(s.text) {
|
||||
ch := s.text[s.pos]
|
||||
if (ch >= '0' && ch <= '9') || ch == '_' {
|
||||
s.pos++
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scanner) scanHexDigits() {
|
||||
for s.pos < len(s.text) {
|
||||
ch := s.text[s.pos]
|
||||
if (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F') || ch == '_' {
|
||||
s.pos++
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scanner) scanBinaryDigits() {
|
||||
for s.pos < len(s.text) {
|
||||
ch := s.text[s.pos]
|
||||
if ch == '0' || ch == '1' || ch == '_' {
|
||||
s.pos++
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scanner) scanOctalDigits() {
|
||||
for s.pos < len(s.text) {
|
||||
ch := s.text[s.pos]
|
||||
if (ch >= '0' && ch <= '7') || ch == '_' {
|
||||
s.pos++
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scanner) scanIdentifierOrKeyword(start int) Token {
|
||||
for s.pos < len(s.text) {
|
||||
ch := s.text[s.pos]
|
||||
if isIdentChar(ch) {
|
||||
s.pos++
|
||||
} else if ch >= 0x80 {
|
||||
r, size := utf8.DecodeRuneInString(s.text[s.pos:])
|
||||
if r != utf8.RuneError && isIdentContinueRune(r) {
|
||||
s.pos += size
|
||||
} else {
|
||||
break
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
text := s.text[start:s.pos]
|
||||
if kind, ok := keywordKinds[text]; ok {
|
||||
return Token{Kind: kind, TokenPos: start, Text: text}
|
||||
}
|
||||
return Token{Kind: KindIdentifier, TokenPos: start, Text: text}
|
||||
}
|
||||
|
||||
func (s *Scanner) scanPrivateIdentifier(start int) Token {
|
||||
s.advance() // skip #
|
||||
for s.pos < len(s.text) && isIdentChar(s.text[s.pos]) {
|
||||
s.pos++
|
||||
}
|
||||
return Token{Kind: KindPrivateIdentifier, TokenPos: start, Text: s.text[start:s.pos]}
|
||||
}
|
||||
|
||||
func isIdentStartByte(ch byte) bool {
|
||||
return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_' || ch == '$'
|
||||
}
|
||||
|
||||
func isIdentStart(ch byte) bool {
|
||||
return isIdentStartByte(ch)
|
||||
}
|
||||
|
||||
func isIdentStartRune(r rune) bool {
|
||||
// JavaScript ID_Start: Lu, Ll, Lt, Lm, Lo, Nl, plus _ and $
|
||||
return unicode.IsLetter(r) || unicode.Is(unicode.Nl, r) || r == '_' || r == '$'
|
||||
}
|
||||
|
||||
// isIdentContinueRune returns true if the rune is valid in a JS identifier (not first position).
|
||||
// JavaScript ID_Continue: ID_Start + Mn, Mc, Nd, Pc, plus ZWNJ/ZWJ.
|
||||
func isIdentContinueRune(r rune) bool {
|
||||
return unicode.IsLetter(r) ||
|
||||
unicode.IsDigit(r) ||
|
||||
unicode.Is(unicode.Nl, r) ||
|
||||
unicode.Is(unicode.Mn, r) ||
|
||||
unicode.Is(unicode.Mc, r) ||
|
||||
unicode.Is(unicode.Pc, r) ||
|
||||
r == '_' || r == '$' ||
|
||||
r == '\u200C' || r == '\u200D'
|
||||
}
|
||||
|
||||
func isIdentChar(ch byte) bool {
|
||||
return isIdentStartByte(ch) || (ch >= '0' && ch <= '9')
|
||||
}
|
||||
|
||||
// keywordKinds maps keyword text to TS7 SyntaxKind values.
|
||||
// These start at KindBreakKeyword = 82.
|
||||
var keywordKinds = map[string]int{
|
||||
"break": 82,
|
||||
"case": 83,
|
||||
"catch": 84,
|
||||
"class": 85,
|
||||
"const": 86,
|
||||
"continue": 87,
|
||||
"debugger": 88,
|
||||
"default": 89,
|
||||
"delete": 90,
|
||||
"do": 91,
|
||||
"else": 92,
|
||||
"enum": 93,
|
||||
"export": 94,
|
||||
"extends": 95,
|
||||
"false": 96,
|
||||
"finally": 97,
|
||||
"for": 98,
|
||||
"function": 99,
|
||||
"if": 100,
|
||||
"import": 101,
|
||||
"in": 102,
|
||||
"instanceof": 103,
|
||||
"new": 104,
|
||||
"null": 105,
|
||||
"return": 106,
|
||||
"super": 107,
|
||||
"switch": 108,
|
||||
"this": 109,
|
||||
"throw": 110,
|
||||
"true": 111,
|
||||
"try": 112,
|
||||
"typeof": 113,
|
||||
"var": 114,
|
||||
"void": 115,
|
||||
"while": 116,
|
||||
"with": 117,
|
||||
// Strict mode reserved words
|
||||
"implements": 118,
|
||||
"interface": 119,
|
||||
"let": 120,
|
||||
"package": 121,
|
||||
"private": 122,
|
||||
"protected": 123,
|
||||
"public": 124,
|
||||
"static": 125,
|
||||
"yield": 126,
|
||||
// Contextual keywords
|
||||
"abstract": 127,
|
||||
"accessor": 128,
|
||||
"as": 129,
|
||||
"asserts": 130,
|
||||
"assert": 131,
|
||||
"any": 132,
|
||||
"async": 133,
|
||||
"await": 134,
|
||||
"boolean": 135,
|
||||
"constructor": 136,
|
||||
"declare": 137,
|
||||
"get": 138,
|
||||
"immediate": 139,
|
||||
"infer": 140,
|
||||
"intrinsic": 141,
|
||||
"is": 142,
|
||||
"keyof": 143,
|
||||
"module": 144,
|
||||
"namespace": 145,
|
||||
"never": 146,
|
||||
"out": 147,
|
||||
"readonly": 148,
|
||||
"require": 149,
|
||||
"number": 150,
|
||||
"object": 151,
|
||||
"satisfies": 152,
|
||||
"set": 153,
|
||||
"string": 154,
|
||||
"symbol": 155,
|
||||
"type": 156,
|
||||
"undefined": 157,
|
||||
"unique": 158,
|
||||
"unknown": 159,
|
||||
"using": 160,
|
||||
"from": 161,
|
||||
"global": 162,
|
||||
"bigint": 163,
|
||||
"override": 164,
|
||||
"of": 165,
|
||||
"defer": 166,
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
// Package astconv handles conversion between TypeScript 7's AST representation
|
||||
// and the JSON format expected by the Java extractor.
|
||||
//
|
||||
// The Java extractor expects AST nodes to have:
|
||||
// - "kind" as a symbolic string name (e.g., "SourceFile"), not a numeric value
|
||||
// - "$pos" and "$end" as character offsets
|
||||
// - "$lineStarts" on the root SourceFile node
|
||||
// - "$tokens" array on the root node
|
||||
// - "$declarationKind" on VariableDeclarationList nodes ("var", "let", "const")
|
||||
// - Only whitelisted property names (see propertyWhitelist)
|
||||
package astconv
|
||||
|
||||
// PropertyWhitelist is the set of property names that should be included
|
||||
// in the serialized AST JSON. This must match the whitelist in the Node.js
|
||||
// wrapper (main.ts).
|
||||
var PropertyWhitelist = map[string]bool{
|
||||
"$declarationKind": true,
|
||||
"$end": true,
|
||||
"$lineStarts": true,
|
||||
"$overloadIndex": true,
|
||||
"$pos": true,
|
||||
"$tokens": true,
|
||||
"argument": true,
|
||||
"argumentExpression": true,
|
||||
"arguments": true,
|
||||
"assertsModifier": true,
|
||||
"asteriskToken": true,
|
||||
"attributes": true,
|
||||
"block": true,
|
||||
"body": true,
|
||||
"caseBlock": true,
|
||||
"catchClause": true,
|
||||
"checkType": true,
|
||||
"children": true,
|
||||
"clauses": true,
|
||||
"closingElement": true,
|
||||
"closingFragment": true,
|
||||
"condition": true,
|
||||
"constraint": true,
|
||||
"constructor": true,
|
||||
"declarationList": true,
|
||||
"declarations": true,
|
||||
"default": true,
|
||||
"delete": true,
|
||||
"dotDotDotToken": true,
|
||||
"elements": true,
|
||||
"elementType": true,
|
||||
"elementTypes": true,
|
||||
"elseStatement": true,
|
||||
"escapedText": true,
|
||||
"exclamationToken": true,
|
||||
"exportClause": true,
|
||||
"expression": true,
|
||||
"exprName": true,
|
||||
"extendsType": true,
|
||||
"falseType": true,
|
||||
"finallyBlock": true,
|
||||
"flags": true,
|
||||
"head": true,
|
||||
"heritageClauses": true,
|
||||
"importClause": true,
|
||||
"incrementor": true,
|
||||
"indexType": true,
|
||||
"init": true,
|
||||
"initializer": true,
|
||||
"isExportEquals": true,
|
||||
"isTypeOf": true,
|
||||
"isTypeOnly": true,
|
||||
"keywordToken": true,
|
||||
"kind": true,
|
||||
"label": true,
|
||||
"left": true,
|
||||
"literal": true,
|
||||
"members": true,
|
||||
"messageText": true,
|
||||
"modifiers": true,
|
||||
"moduleReference": true,
|
||||
"moduleSpecifier": true,
|
||||
"name": true,
|
||||
"namedBindings": true,
|
||||
"objectType": true,
|
||||
"openingElement": true,
|
||||
"openingFragment": true,
|
||||
"operand": true,
|
||||
"operator": true,
|
||||
"operatorToken": true,
|
||||
"parameterName": true,
|
||||
"parameters": true,
|
||||
"parseDiagnostics": true,
|
||||
"phaseModifier": true,
|
||||
"properties": true,
|
||||
"propertyName": true,
|
||||
"qualifier": true,
|
||||
"questionDotToken": true,
|
||||
"questionToken": true,
|
||||
"right": true,
|
||||
"selfClosing": true,
|
||||
"statement": true,
|
||||
"statements": true,
|
||||
"tag": true,
|
||||
"tagName": true,
|
||||
"template": true,
|
||||
"templateSpans": true,
|
||||
"text": true,
|
||||
"thenStatement": true,
|
||||
"token": true,
|
||||
"tokenPos": true,
|
||||
"trueType": true,
|
||||
"tryBlock": true,
|
||||
"type": true,
|
||||
"typeArguments": true,
|
||||
"typeName": true,
|
||||
"typeParameter": true,
|
||||
"typeParameters": true,
|
||||
"types": true,
|
||||
"variableDeclaration": true,
|
||||
"whenFalse": true,
|
||||
"whenTrue": true,
|
||||
}
|
||||
|
||||
// MetaProperties are property names used in the parse response wrapper
|
||||
// (not part of the AST itself but part of the response envelope).
|
||||
var MetaProperties = map[string]bool{
|
||||
"ast": true,
|
||||
"type": true,
|
||||
}
|
||||
|
||||
// IsAllowedProperty returns true if the property name should be included
|
||||
// in the serialized AST JSON.
|
||||
func IsAllowedProperty(name string) bool {
|
||||
if PropertyWhitelist[name] {
|
||||
return true
|
||||
}
|
||||
if MetaProperties[name] {
|
||||
return true
|
||||
}
|
||||
// Numeric keys (array indices) are always allowed
|
||||
if len(name) > 0 && name[0] >= '0' && name[0] <= '9' {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
package astconv
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestAllowedProperties(t *testing.T) {
|
||||
allowed := []string{"kind", "$pos", "$end", "statements", "body", "name", "type"}
|
||||
for _, p := range allowed {
|
||||
if !IsAllowedProperty(p) {
|
||||
t.Errorf("expected %q to be allowed", p)
|
||||
}
|
||||
}
|
||||
|
||||
disallowed := []string{"parent", "symbol", "localSymbol", "nextContainer", "flowNode"}
|
||||
for _, p := range disallowed {
|
||||
if IsAllowedProperty(p) {
|
||||
t.Errorf("expected %q to be disallowed", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNumericKeys(t *testing.T) {
|
||||
for _, k := range []string{"0", "1", "42", "999"} {
|
||||
if !IsAllowedProperty(k) {
|
||||
t.Errorf("expected numeric key %q to be allowed", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetaProperties(t *testing.T) {
|
||||
if !IsAllowedProperty("ast") {
|
||||
t.Error("expected 'ast' to be allowed (meta property)")
|
||||
}
|
||||
if !IsAllowedProperty("type") {
|
||||
t.Error("expected 'type' to be allowed (meta property)")
|
||||
}
|
||||
}
|
||||
@@ -1,191 +0,0 @@
|
||||
// Package protocol implements the line-delimited JSON protocol used to
|
||||
// communicate between the Java extractor and the TypeScript parser wrapper.
|
||||
//
|
||||
// The protocol matches the one implemented by the Node.js wrapper in
|
||||
// lib/typescript/src/main.ts. Commands are read from stdin as one JSON
|
||||
// object per line, and responses are written to stdout in the same format.
|
||||
package protocol
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Command represents a parsed command from the Java extractor.
|
||||
type Command struct {
|
||||
Command string `json:"command"`
|
||||
Filename string `json:"filename,omitempty"`
|
||||
Filenames []string `json:"filenames,omitempty"`
|
||||
}
|
||||
|
||||
// Response is the interface for all protocol responses.
|
||||
type Response interface {
|
||||
ResponseType() string
|
||||
}
|
||||
|
||||
// MetadataResponse is sent in reply to a "get-metadata" command.
|
||||
type MetadataResponse struct {
|
||||
Type string `json:"type"`
|
||||
SyntaxKinds map[string]int `json:"syntaxKinds"`
|
||||
NodeFlags map[string]int `json:"nodeFlags"`
|
||||
}
|
||||
|
||||
func (r *MetadataResponse) ResponseType() string { return "metadata" }
|
||||
|
||||
// OKResponse is sent in reply to a "prepare-files" command.
|
||||
type OKResponse struct {
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
func (r *OKResponse) ResponseType() string { return "ok" }
|
||||
|
||||
// ASTResponse is sent in reply to a "parse" command.
|
||||
type ASTResponse struct {
|
||||
Type string `json:"type"`
|
||||
AST interface{} `json:"ast"`
|
||||
}
|
||||
|
||||
func (r *ASTResponse) ResponseType() string { return "ast" }
|
||||
|
||||
// ResetDoneResponse is sent in reply to a "reset" command.
|
||||
type ResetDoneResponse struct {
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
func (r *ResetDoneResponse) ResponseType() string { return "reset-done" }
|
||||
|
||||
// ErrorResponse is sent when an error occurs during processing.
|
||||
type ErrorResponse struct {
|
||||
Type string `json:"type"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
func (r *ErrorResponse) ResponseType() string { return "error" }
|
||||
|
||||
// Handler defines the interface for handling protocol commands.
|
||||
type Handler interface {
|
||||
// HandleParse parses a TypeScript file and returns the AST.
|
||||
HandleParse(filename string) (interface{}, error)
|
||||
|
||||
// HandlePrepareFiles informs the handler that the given files will be
|
||||
// requested in order, allowing pre-parsing.
|
||||
HandlePrepareFiles(filenames []string) error
|
||||
|
||||
// HandleReset resets the handler to a fresh state.
|
||||
HandleReset() error
|
||||
|
||||
// HandleGetMetadata returns the syntax kind and node flag mappings.
|
||||
HandleGetMetadata() (*MetadataResponse, error)
|
||||
}
|
||||
|
||||
// Server reads commands from stdin and dispatches them to a Handler.
|
||||
type Server struct {
|
||||
handler Handler
|
||||
in io.Reader
|
||||
out io.Writer
|
||||
}
|
||||
|
||||
// NewServer creates a new protocol server.
|
||||
func NewServer(handler Handler) *Server {
|
||||
return &Server{
|
||||
handler: handler,
|
||||
in: os.Stdin,
|
||||
out: os.Stdout,
|
||||
}
|
||||
}
|
||||
|
||||
// NewServerWithIO creates a server with custom I/O streams (for testing).
|
||||
func NewServerWithIO(handler Handler, in io.Reader, out io.Writer) *Server {
|
||||
return &Server{
|
||||
handler: handler,
|
||||
in: in,
|
||||
out: out,
|
||||
}
|
||||
}
|
||||
|
||||
// Run reads commands from stdin and processes them until a "quit" command
|
||||
// is received or stdin is closed.
|
||||
func (s *Server) Run() error {
|
||||
scanner := bufio.NewScanner(s.in)
|
||||
// Allow for very large JSON payloads.
|
||||
scanner.Buffer(make([]byte, 1024*1024), 100*1024*1024)
|
||||
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
var cmd Command
|
||||
if err := json.Unmarshal([]byte(line), &cmd); err != nil {
|
||||
s.writeResponse(&ErrorResponse{
|
||||
Type: "error",
|
||||
Message: fmt.Sprintf("failed to parse command: %v", err),
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
quit, err := s.dispatch(cmd)
|
||||
if err != nil {
|
||||
s.writeResponse(&ErrorResponse{
|
||||
Type: "error",
|
||||
Message: err.Error(),
|
||||
})
|
||||
continue
|
||||
}
|
||||
if quit {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return scanner.Err()
|
||||
}
|
||||
|
||||
func (s *Server) dispatch(cmd Command) (quit bool, err error) {
|
||||
switch cmd.Command {
|
||||
case "parse":
|
||||
ast, err := s.handler.HandleParse(cmd.Filename)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
s.writeResponse(&ASTResponse{
|
||||
Type: "ast",
|
||||
AST: ast,
|
||||
})
|
||||
case "prepare-files":
|
||||
if err := s.handler.HandlePrepareFiles(cmd.Filenames); err != nil {
|
||||
return false, err
|
||||
}
|
||||
s.writeResponse(&OKResponse{Type: "ok"})
|
||||
case "reset":
|
||||
if err := s.handler.HandleReset(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
s.writeResponse(&ResetDoneResponse{Type: "reset-done"})
|
||||
case "get-metadata":
|
||||
resp, err := s.handler.HandleGetMetadata()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
s.writeResponse(resp)
|
||||
case "quit":
|
||||
return true, nil
|
||||
default:
|
||||
return false, fmt.Errorf("unknown command: %s", cmd.Command)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (s *Server) writeResponse(resp Response) {
|
||||
data, err := json.Marshal(resp)
|
||||
if err != nil {
|
||||
// If we can't marshal the response, write an error.
|
||||
fmt.Fprintf(s.out, `{"type":"error","message":"marshal error: %s"}`+"\n", err.Error())
|
||||
return
|
||||
}
|
||||
s.out.Write(data)
|
||||
s.out.Write([]byte("\n"))
|
||||
}
|
||||
@@ -1,163 +0,0 @@
|
||||
package protocol
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// mockHandler implements Handler for testing.
|
||||
type mockHandler struct {
|
||||
parseFunc func(string) (interface{}, error)
|
||||
prepareFilesFunc func([]string) error
|
||||
resetFunc func() error
|
||||
getMetadataFunc func() (*MetadataResponse, error)
|
||||
}
|
||||
|
||||
func (h *mockHandler) HandleParse(filename string) (interface{}, error) {
|
||||
if h.parseFunc != nil {
|
||||
return h.parseFunc(filename)
|
||||
}
|
||||
return map[string]interface{}{"kind": "SourceFile"}, nil
|
||||
}
|
||||
|
||||
func (h *mockHandler) HandlePrepareFiles(filenames []string) error {
|
||||
if h.prepareFilesFunc != nil {
|
||||
return h.prepareFilesFunc(filenames)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *mockHandler) HandleReset() error {
|
||||
if h.resetFunc != nil {
|
||||
return h.resetFunc()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *mockHandler) HandleGetMetadata() (*MetadataResponse, error) {
|
||||
if h.getMetadataFunc != nil {
|
||||
return h.getMetadataFunc()
|
||||
}
|
||||
return &MetadataResponse{
|
||||
Type: "metadata",
|
||||
SyntaxKinds: map[string]int{"SourceFile": 316},
|
||||
NodeFlags: map[string]int{"None": 0},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func TestServerGetMetadata(t *testing.T) {
|
||||
input := `{"command":"get-metadata"}` + "\n" + `{"command":"quit"}` + "\n"
|
||||
var output bytes.Buffer
|
||||
|
||||
handler := &mockHandler{}
|
||||
server := NewServerWithIO(handler, strings.NewReader(input), &output)
|
||||
|
||||
if err := server.Run(); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
var resp MetadataResponse
|
||||
if err := json.Unmarshal(output.Bytes()[:bytes.IndexByte(output.Bytes(), '\n')], &resp); err != nil {
|
||||
t.Fatalf("failed to parse response: %v", err)
|
||||
}
|
||||
|
||||
if resp.Type != "metadata" {
|
||||
t.Errorf("expected type 'metadata', got %q", resp.Type)
|
||||
}
|
||||
if _, ok := resp.SyntaxKinds["SourceFile"]; !ok {
|
||||
t.Error("expected syntaxKinds to contain SourceFile")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerParse(t *testing.T) {
|
||||
input := `{"command":"parse","filename":"test.ts"}` + "\n" + `{"command":"quit"}` + "\n"
|
||||
var output bytes.Buffer
|
||||
|
||||
handler := &mockHandler{}
|
||||
server := NewServerWithIO(handler, strings.NewReader(input), &output)
|
||||
|
||||
if err := server.Run(); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
var resp ASTResponse
|
||||
if err := json.Unmarshal(output.Bytes()[:bytes.IndexByte(output.Bytes(), '\n')], &resp); err != nil {
|
||||
t.Fatalf("failed to parse response: %v", err)
|
||||
}
|
||||
|
||||
if resp.Type != "ast" {
|
||||
t.Errorf("expected type 'ast', got %q", resp.Type)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerPrepareFiles(t *testing.T) {
|
||||
input := `{"command":"prepare-files","filenames":["a.ts","b.ts"]}` + "\n" + `{"command":"quit"}` + "\n"
|
||||
var output bytes.Buffer
|
||||
|
||||
var receivedFiles []string
|
||||
handler := &mockHandler{
|
||||
prepareFilesFunc: func(files []string) error {
|
||||
receivedFiles = files
|
||||
return nil
|
||||
},
|
||||
}
|
||||
server := NewServerWithIO(handler, strings.NewReader(input), &output)
|
||||
|
||||
if err := server.Run(); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if len(receivedFiles) != 2 || receivedFiles[0] != "a.ts" || receivedFiles[1] != "b.ts" {
|
||||
t.Errorf("expected [a.ts b.ts], got %v", receivedFiles)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerReset(t *testing.T) {
|
||||
input := `{"command":"reset"}` + "\n" + `{"command":"quit"}` + "\n"
|
||||
var output bytes.Buffer
|
||||
|
||||
resetCalled := false
|
||||
handler := &mockHandler{
|
||||
resetFunc: func() error {
|
||||
resetCalled = true
|
||||
return nil
|
||||
},
|
||||
}
|
||||
server := NewServerWithIO(handler, strings.NewReader(input), &output)
|
||||
|
||||
if err := server.Run(); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if !resetCalled {
|
||||
t.Error("expected reset to be called")
|
||||
}
|
||||
|
||||
var resp ResetDoneResponse
|
||||
if err := json.Unmarshal(output.Bytes()[:bytes.IndexByte(output.Bytes(), '\n')], &resp); err != nil {
|
||||
t.Fatalf("failed to parse response: %v", err)
|
||||
}
|
||||
|
||||
if resp.Type != "reset-done" {
|
||||
t.Errorf("expected type 'reset-done', got %q", resp.Type)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerQuit(t *testing.T) {
|
||||
input := `{"command":"quit"}` + "\n"
|
||||
var output bytes.Buffer
|
||||
|
||||
handler := &mockHandler{}
|
||||
server := NewServerWithIO(handler, strings.NewReader(input), &output)
|
||||
|
||||
if err := server.Run(); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
// No output expected for quit
|
||||
if output.Len() != 0 {
|
||||
t.Errorf("expected no output, got %q", output.String())
|
||||
}
|
||||
}
|
||||
@@ -1,411 +0,0 @@
|
||||
package tsparser
|
||||
|
||||
// GetStaticTS7Metadata returns hardcoded metadata for TypeScript 7.
|
||||
// This must be kept in sync with the TypeScript compiler's SyntaxKind and
|
||||
// NodeFlags enums.
|
||||
//
|
||||
// The SyntaxKind values here correspond to the TypeScript 7 (Go port)
|
||||
// compiler. The Java extractor uses the string names (not numeric IDs)
|
||||
// to identify node kinds, so the exact numeric values only matter for
|
||||
// the metadata response.
|
||||
func GetStaticTS7Metadata() *Metadata {
|
||||
return &Metadata{
|
||||
SyntaxKinds: syntaxKinds,
|
||||
NodeFlags: nodeFlags,
|
||||
}
|
||||
}
|
||||
|
||||
// GetSyntaxKinds returns the raw SyntaxKind name→number map.
|
||||
func GetSyntaxKinds() map[string]int {
|
||||
return syntaxKinds
|
||||
}
|
||||
|
||||
// BuildKindToNameMap returns a number→name reverse map for SyntaxKinds.
|
||||
func BuildKindToNameMap() map[uint32]string {
|
||||
m := make(map[uint32]string, len(syntaxKinds))
|
||||
for name, num := range syntaxKinds {
|
||||
key := uint32(num)
|
||||
if existing, ok := m[key]; !ok || len(name) < len(existing) {
|
||||
m[key] = name
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// syntaxKinds maps SyntaxKind names to their numeric values in TypeScript 7.
|
||||
// Generated from microsoft/typescript-go/internal/ast/kind.go (iota enum).
|
||||
var syntaxKinds = map[string]int{
|
||||
"Unknown": 0,
|
||||
"EndOfFileToken": 1,
|
||||
"SingleLineCommentTrivia": 2,
|
||||
"MultiLineCommentTrivia": 3,
|
||||
"NewLineTrivia": 4,
|
||||
"WhitespaceTrivia": 5,
|
||||
"ShebangTrivia": 6,
|
||||
"ConflictMarkerTrivia": 7,
|
||||
"NumericLiteral": 8,
|
||||
"BigIntLiteral": 9,
|
||||
"StringLiteral": 10,
|
||||
"JsxText": 11,
|
||||
"JsxTextAllWhiteSpaces": 12,
|
||||
"RegularExpressionLiteral": 13,
|
||||
"NoSubstitutionTemplateLiteral": 14,
|
||||
"TemplateHead": 15,
|
||||
"TemplateMiddle": 16,
|
||||
"TemplateTail": 17,
|
||||
"OpenBraceToken": 18,
|
||||
"CloseBraceToken": 19,
|
||||
"OpenParenToken": 20,
|
||||
"CloseParenToken": 21,
|
||||
"OpenBracketToken": 22,
|
||||
"CloseBracketToken": 23,
|
||||
"DotToken": 24,
|
||||
"DotDotDotToken": 25,
|
||||
"SemicolonToken": 26,
|
||||
"CommaToken": 27,
|
||||
"QuestionDotToken": 28,
|
||||
"LessThanToken": 29,
|
||||
"LessThanSlashToken": 30,
|
||||
"GreaterThanToken": 31,
|
||||
"LessThanEqualsToken": 32,
|
||||
"GreaterThanEqualsToken": 33,
|
||||
"EqualsEqualsToken": 34,
|
||||
"ExclamationEqualsToken": 35,
|
||||
"EqualsEqualsEqualsToken": 36,
|
||||
"ExclamationEqualsEqualsToken": 37,
|
||||
"EqualsGreaterThanToken": 38,
|
||||
"PlusToken": 39,
|
||||
"MinusToken": 40,
|
||||
"AsteriskToken": 41,
|
||||
"AsteriskAsteriskToken": 42,
|
||||
"SlashToken": 43,
|
||||
"PercentToken": 44,
|
||||
"PlusPlusToken": 45,
|
||||
"MinusMinusToken": 46,
|
||||
"LessThanLessThanToken": 47,
|
||||
"GreaterThanGreaterThanToken": 48,
|
||||
"GreaterThanGreaterThanGreaterThanToken": 49,
|
||||
"AmpersandToken": 50,
|
||||
"BarToken": 51,
|
||||
"CaretToken": 52,
|
||||
"ExclamationToken": 53,
|
||||
"TildeToken": 54,
|
||||
"AmpersandAmpersandToken": 55,
|
||||
"BarBarToken": 56,
|
||||
"QuestionToken": 57,
|
||||
"ColonToken": 58,
|
||||
"AtToken": 59,
|
||||
"QuestionQuestionToken": 60,
|
||||
"HashToken": 62,
|
||||
"EqualsToken": 63,
|
||||
"PlusEqualsToken": 64,
|
||||
"MinusEqualsToken": 65,
|
||||
"AsteriskEqualsToken": 66,
|
||||
"AsteriskAsteriskEqualsToken": 67,
|
||||
"SlashEqualsToken": 68,
|
||||
"PercentEqualsToken": 69,
|
||||
"LessThanLessThanEqualsToken": 70,
|
||||
"GreaterThanGreaterThanEqualsToken": 71,
|
||||
"GreaterThanGreaterThanGreaterThanEqualsToken": 72,
|
||||
"AmpersandEqualsToken": 73,
|
||||
"BarEqualsToken": 74,
|
||||
"BarBarEqualsToken": 75,
|
||||
"AmpersandAmpersandEqualsToken": 76,
|
||||
"QuestionQuestionEqualsToken": 77,
|
||||
"CaretEqualsToken": 78,
|
||||
"Identifier": 79,
|
||||
"PrivateIdentifier": 80,
|
||||
"BreakKeyword": 82,
|
||||
"CaseKeyword": 83,
|
||||
"CatchKeyword": 84,
|
||||
"ClassKeyword": 85,
|
||||
"ConstKeyword": 86,
|
||||
"ContinueKeyword": 87,
|
||||
"DebuggerKeyword": 88,
|
||||
"DefaultKeyword": 89,
|
||||
"DeleteKeyword": 90,
|
||||
"DoKeyword": 91,
|
||||
"ElseKeyword": 92,
|
||||
"EnumKeyword": 93,
|
||||
"ExportKeyword": 94,
|
||||
"ExtendsKeyword": 95,
|
||||
"FalseKeyword": 96,
|
||||
"FinallyKeyword": 97,
|
||||
"ForKeyword": 98,
|
||||
"FunctionKeyword": 99,
|
||||
"IfKeyword": 100,
|
||||
"ImportKeyword": 101,
|
||||
"InKeyword": 102,
|
||||
"InstanceOfKeyword": 103,
|
||||
"NewKeyword": 104,
|
||||
"NullKeyword": 105,
|
||||
"ReturnKeyword": 106,
|
||||
"SuperKeyword": 107,
|
||||
"SwitchKeyword": 108,
|
||||
"ThisKeyword": 109,
|
||||
"ThrowKeyword": 110,
|
||||
"TrueKeyword": 111,
|
||||
"TryKeyword": 112,
|
||||
"TypeOfKeyword": 113,
|
||||
"VarKeyword": 114,
|
||||
"VoidKeyword": 115,
|
||||
"WhileKeyword": 116,
|
||||
"WithKeyword": 117,
|
||||
"ImplementsKeyword": 118,
|
||||
"InterfaceKeyword": 119,
|
||||
"LetKeyword": 120,
|
||||
"PackageKeyword": 121,
|
||||
"PrivateKeyword": 122,
|
||||
"ProtectedKeyword": 123,
|
||||
"PublicKeyword": 124,
|
||||
"StaticKeyword": 125,
|
||||
"YieldKeyword": 126,
|
||||
"AbstractKeyword": 127,
|
||||
"AccessorKeyword": 128,
|
||||
"AsKeyword": 129,
|
||||
"AssertsKeyword": 130,
|
||||
"AssertKeyword": 131,
|
||||
"AnyKeyword": 132,
|
||||
"AsyncKeyword": 133,
|
||||
"AwaitKeyword": 134,
|
||||
"BooleanKeyword": 135,
|
||||
"ConstructorKeyword": 136,
|
||||
"DeclareKeyword": 137,
|
||||
"GetKeyword": 138,
|
||||
"InferKeyword": 140,
|
||||
"IntrinsicKeyword": 141,
|
||||
"IsKeyword": 142,
|
||||
"KeyOfKeyword": 143,
|
||||
"ModuleKeyword": 144,
|
||||
"NamespaceKeyword": 145,
|
||||
"NeverKeyword": 146,
|
||||
"ReadonlyKeyword": 148,
|
||||
"RequireKeyword": 149,
|
||||
"NumberKeyword": 150,
|
||||
"ObjectKeyword": 151,
|
||||
"SetKeyword": 153,
|
||||
"StringKeyword": 154,
|
||||
"SymbolKeyword": 155,
|
||||
"TypeKeyword": 156,
|
||||
"UndefinedKeyword": 157,
|
||||
"UniqueKeyword": 158,
|
||||
"UnknownKeyword": 159,
|
||||
"FromKeyword": 161,
|
||||
"BigIntKeyword": 163,
|
||||
"OverrideKeyword": 164,
|
||||
"OfKeyword": 165,
|
||||
"DeferKeyword": 166,
|
||||
"QualifiedName": 167,
|
||||
"ComputedPropertyName": 168,
|
||||
"TypeParameter": 169,
|
||||
"Parameter": 170,
|
||||
"Decorator": 171,
|
||||
"PropertySignature": 172,
|
||||
"PropertyDeclaration": 173,
|
||||
"MethodSignature": 174,
|
||||
"MethodDeclaration": 175,
|
||||
"ClassStaticBlockDeclaration": 176,
|
||||
"Constructor": 177,
|
||||
"GetAccessor": 178,
|
||||
"SetAccessor": 179,
|
||||
"CallSignature": 180,
|
||||
"ConstructSignature": 181,
|
||||
"IndexSignature": 182,
|
||||
"TypePredicate": 183,
|
||||
"TypeReference": 184,
|
||||
"FunctionType": 185,
|
||||
"ConstructorType": 186,
|
||||
"TypeQuery": 187,
|
||||
"TypeLiteral": 188,
|
||||
"ArrayType": 189,
|
||||
"TupleType": 190,
|
||||
"OptionalType": 191,
|
||||
"RestType": 192,
|
||||
"UnionType": 193,
|
||||
"IntersectionType": 194,
|
||||
"ConditionalType": 195,
|
||||
"InferType": 196,
|
||||
"ParenthesizedType": 197,
|
||||
"ThisType": 198,
|
||||
"TypeOperator": 199,
|
||||
"IndexedAccessType": 200,
|
||||
"MappedType": 201,
|
||||
"LiteralType": 202,
|
||||
"NamedTupleMember": 203,
|
||||
"TemplateLiteralType": 204,
|
||||
"TemplateLiteralTypeSpan": 205,
|
||||
"ImportType": 206,
|
||||
"ObjectBindingPattern": 207,
|
||||
"ArrayBindingPattern": 208,
|
||||
"BindingElement": 209,
|
||||
"ArrayLiteralExpression": 210,
|
||||
"ObjectLiteralExpression": 211,
|
||||
"PropertyAccessExpression": 212,
|
||||
"ElementAccessExpression": 213,
|
||||
"CallExpression": 214,
|
||||
"NewExpression": 215,
|
||||
"TaggedTemplateExpression": 216,
|
||||
"TypeAssertionExpression": 217,
|
||||
"ParenthesizedExpression": 218,
|
||||
"FunctionExpression": 219,
|
||||
"ArrowFunction": 220,
|
||||
"DeleteExpression": 221,
|
||||
"TypeOfExpression": 222,
|
||||
"VoidExpression": 223,
|
||||
"AwaitExpression": 224,
|
||||
"PrefixUnaryExpression": 225,
|
||||
"PostfixUnaryExpression": 226,
|
||||
"BinaryExpression": 227,
|
||||
"ConditionalExpression": 228,
|
||||
"TemplateExpression": 229,
|
||||
"YieldExpression": 230,
|
||||
"SpreadElement": 231,
|
||||
"ClassExpression": 232,
|
||||
"OmittedExpression": 233,
|
||||
"ExpressionWithTypeArguments": 234,
|
||||
"AsExpression": 235,
|
||||
"NonNullExpression": 236,
|
||||
"MetaProperty": 237,
|
||||
"SatisfiesExpression": 239,
|
||||
"TemplateSpan": 240,
|
||||
"SemicolonClassElement": 241,
|
||||
"Block": 242,
|
||||
"EmptyStatement": 243,
|
||||
"VariableStatement": 244,
|
||||
"ExpressionStatement": 245,
|
||||
"IfStatement": 246,
|
||||
"DoStatement": 247,
|
||||
"WhileStatement": 248,
|
||||
"ForStatement": 249,
|
||||
"ForInStatement": 250,
|
||||
"ForOfStatement": 251,
|
||||
"ContinueStatement": 252,
|
||||
"BreakStatement": 253,
|
||||
"ReturnStatement": 254,
|
||||
"WithStatement": 255,
|
||||
"SwitchStatement": 256,
|
||||
"LabeledStatement": 257,
|
||||
"ThrowStatement": 258,
|
||||
"TryStatement": 259,
|
||||
"DebuggerStatement": 260,
|
||||
"VariableDeclaration": 261,
|
||||
"VariableDeclarationList": 262,
|
||||
"FunctionDeclaration": 263,
|
||||
"ClassDeclaration": 264,
|
||||
"InterfaceDeclaration": 265,
|
||||
"TypeAliasDeclaration": 266,
|
||||
"EnumDeclaration": 267,
|
||||
"ModuleDeclaration": 268,
|
||||
"ModuleBlock": 269,
|
||||
"CaseBlock": 270,
|
||||
"NamespaceExportDeclaration": 271,
|
||||
"ImportEqualsDeclaration": 272,
|
||||
"ImportDeclaration": 273,
|
||||
"ImportClause": 274,
|
||||
"NamespaceImport": 275,
|
||||
"NamedImports": 276,
|
||||
"ImportSpecifier": 277,
|
||||
"ExportAssignment": 278,
|
||||
"ExportDeclaration": 279,
|
||||
"NamedExports": 280,
|
||||
"NamespaceExport": 281,
|
||||
"ExportSpecifier": 282,
|
||||
"MissingDeclaration": 283,
|
||||
"ExternalModuleReference": 284,
|
||||
"JsxElement": 285,
|
||||
"JsxSelfClosingElement": 286,
|
||||
"JsxOpeningElement": 287,
|
||||
"JsxClosingElement": 288,
|
||||
"JsxFragment": 289,
|
||||
"JsxOpeningFragment": 290,
|
||||
"JsxClosingFragment": 291,
|
||||
"JsxAttribute": 292,
|
||||
"JsxAttributes": 293,
|
||||
"JsxSpreadAttribute": 294,
|
||||
"JsxExpression": 295,
|
||||
"JsxNamespacedName": 296,
|
||||
"CaseClause": 297,
|
||||
"DefaultClause": 298,
|
||||
"HeritageClause": 299,
|
||||
"CatchClause": 300,
|
||||
"ImportAttributes": 301,
|
||||
"ImportAttribute": 302,
|
||||
"PropertyAssignment": 303,
|
||||
"ShorthandPropertyAssignment": 304,
|
||||
"SpreadAssignment": 305,
|
||||
"EnumMember": 306,
|
||||
"SourceFile": 307,
|
||||
"JSDocTypeExpression": 308,
|
||||
"JSDocNameReference": 309,
|
||||
"JSDocNullableType": 312,
|
||||
"JSDocNonNullableType": 313,
|
||||
"JSDocOptionalType": 314,
|
||||
"JSDocVariadicType": 315,
|
||||
"JSDoc": 316,
|
||||
"JSDocText": 317,
|
||||
"JSDocTypeLiteral": 318,
|
||||
"JSDocSignature": 319,
|
||||
"JSDocLink": 320,
|
||||
"JSDocLinkCode": 321,
|
||||
"JSDocLinkPlain": 322,
|
||||
"JSDocTag": 323,
|
||||
"JSDocAugmentsTag": 324,
|
||||
"JSDocImplementsTag": 325,
|
||||
"JSDocDeprecatedTag": 326,
|
||||
"JSDocPublicTag": 327,
|
||||
"JSDocPrivateTag": 328,
|
||||
"JSDocProtectedTag": 329,
|
||||
"JSDocReadonlyTag": 330,
|
||||
"JSDocOverrideTag": 331,
|
||||
"JSDocCallbackTag": 332,
|
||||
"JSDocOverloadTag": 333,
|
||||
"JSDocParameterTag": 334,
|
||||
"JSDocReturnTag": 335,
|
||||
"JSDocThisTag": 336,
|
||||
"JSDocTypeTag": 337,
|
||||
"JSDocTemplateTag": 338,
|
||||
"JSDocTypedefTag": 339,
|
||||
"JSDocSeeTag": 340,
|
||||
"JSDocPropertyTag": 341,
|
||||
"JSDocThrowsTag": 342,
|
||||
"JSDocSatisfiesTag": 343,
|
||||
"JSDocImportTag": 344,
|
||||
}
|
||||
|
||||
// nodeFlags maps NodeFlags names to their numeric values sent to the Java extractor.
|
||||
// The Java extractor only checks Using, NestedNamespace, and GlobalAugmentation.
|
||||
//
|
||||
// TS7 binary AST flag layout (differs from TS5):
|
||||
// bit 0: Let, bit 1: Const, bit 2: Using, bit 3: NestedNamespace (not set in binary),
|
||||
// bit 4: Namespace, bit 5: OptionalChain, bit 6: ExportContext (was GlobalAugmentation
|
||||
// in TS5 at bit 11), bit 7: ContainsThis, ...
|
||||
//
|
||||
// GlobalAugmentation is NOT a flag in the TS7 binary format. We use a synthetic bit (30)
|
||||
// that the converter sets on `declare global {}` nodes so the Java extractor can detect them.
|
||||
var nodeFlags = map[string]int{
|
||||
"None": 0,
|
||||
"Let": 1,
|
||||
"Const": 2,
|
||||
"Using": 4, // Let | Const
|
||||
"AwaitUsing": 6, // Using | Const
|
||||
"NestedNamespace": 8, // bit 3 — synthetic, set by converter
|
||||
"Namespace": 16, // bit 4
|
||||
"OptionalChain": 32, // bit 5
|
||||
"ExportContext": 64, // bit 6
|
||||
"GlobalAugmentation": 1 << 30, // synthetic — set by converter for `declare global {}`
|
||||
"ContainsThis": 128, // bit 7
|
||||
"HasImplicitReturn": 256, // bit 8
|
||||
"HasExplicitReturn": 512, // bit 9
|
||||
"HasAsyncFunctions": 1024, // bit 10
|
||||
"DisallowInContext": 2048, // bit 11
|
||||
"YieldContext": 4096, // bit 12
|
||||
"DecoratorContext": 8192, // bit 13
|
||||
"AwaitContext": 16384, // bit 14
|
||||
"DisallowConditionalTypesContext": 32768, // bit 15
|
||||
"ThisNodeHasError": 65536, // bit 16
|
||||
"JavaScriptFile": 131072, // bit 17
|
||||
"ThisNodeOrAnySubNodesHasError": 262144, // bit 18
|
||||
"HasAggregatedChildData": 524288, // bit 19
|
||||
"JSDoc": 8388608, // bit 23
|
||||
"JsonFile": 67108864, // bit 26
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
// Package tsparser provides an interface for parsing TypeScript files and
|
||||
// implementations backed by different TypeScript compiler versions.
|
||||
//
|
||||
// The primary implementation uses the tsgo binary (TypeScript 7's Go-based
|
||||
// compiler) as a subprocess via its --api mode.
|
||||
package tsparser
|
||||
|
||||
import "io"
|
||||
|
||||
// ParseResult holds the parsed AST for a single file.
|
||||
type ParseResult struct {
|
||||
// AST is the parsed AST tree, ready for JSON serialization.
|
||||
AST interface{}
|
||||
|
||||
// RawData holds the raw binary-encoded source file data from tsgo.
|
||||
// This is present when using the tsgo API backend and needs to be
|
||||
// decoded into the AST format expected by the Java extractor.
|
||||
RawData []byte
|
||||
}
|
||||
|
||||
// Metadata holds the compiler metadata (syntax kind and node flag mappings).
|
||||
type Metadata struct {
|
||||
SyntaxKinds map[string]int
|
||||
NodeFlags map[string]int
|
||||
}
|
||||
|
||||
// Parser is the interface for TypeScript parsing backends.
|
||||
type Parser interface {
|
||||
// Parse parses the given file and returns the AST.
|
||||
Parse(filename string) (*ParseResult, error)
|
||||
|
||||
// GetMetadata returns the syntax kind and node flag mappings for
|
||||
// the underlying TypeScript compiler.
|
||||
GetMetadata() (*Metadata, error)
|
||||
|
||||
// Reset discards any cached state and returns the parser to a fresh state.
|
||||
Reset() error
|
||||
|
||||
// Close shuts down the parser, releasing any resources.
|
||||
Close() error
|
||||
}
|
||||
|
||||
// TsgoBinaryFinder locates the tsgo binary. This is separated to allow
|
||||
// different search strategies (PATH, npm package, env var, etc.).
|
||||
type TsgoBinaryFinder interface {
|
||||
// FindBinary returns the path to the tsgo binary.
|
||||
FindBinary() (string, error)
|
||||
}
|
||||
|
||||
// Config configures the parser backend.
|
||||
type Config struct {
|
||||
// TsgoBinary is the explicit path to the tsgo binary.
|
||||
// If empty, the binary is found via TsgoBinaryFinder or PATH.
|
||||
TsgoBinary string
|
||||
|
||||
// Stderr is where to redirect the tsgo process's stderr.
|
||||
// If nil, stderr is discarded.
|
||||
Stderr io.Writer
|
||||
}
|
||||
@@ -1,110 +0,0 @@
|
||||
package tsparser
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// StandaloneParser implements the Parser interface by invoking the tsgo
|
||||
// binary once per file (non-persistent). This is simpler but slower than
|
||||
// the TsgoParser which keeps a persistent subprocess.
|
||||
//
|
||||
// This is intended as a fallback and for testing. For production use,
|
||||
// prefer TsgoParser.
|
||||
type StandaloneParser struct {
|
||||
config Config
|
||||
}
|
||||
|
||||
// NewStandaloneParser creates a parser that invokes tsgo once per file.
|
||||
func NewStandaloneParser(config Config) *StandaloneParser {
|
||||
return &StandaloneParser{config: config}
|
||||
}
|
||||
|
||||
func (p *StandaloneParser) findBinary() (string, error) {
|
||||
if p.config.TsgoBinary != "" {
|
||||
return p.config.TsgoBinary, nil
|
||||
}
|
||||
path, err := exec.LookPath("tsgo")
|
||||
if err == nil {
|
||||
return path, nil
|
||||
}
|
||||
return "", fmt.Errorf("tsgo binary not found on PATH")
|
||||
}
|
||||
|
||||
// Parse parses a single TypeScript file by running tsgo.
|
||||
// Since tsgo doesn't have a direct "dump AST" mode, this uses a
|
||||
// minimal tsconfig.json to parse the file and extract diagnostics.
|
||||
//
|
||||
// TODO: Replace with direct API call when the tsgo Go API is public.
|
||||
func (p *StandaloneParser) Parse(filename string) (*ParseResult, error) {
|
||||
binary, err := p.findBinary()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
absPath, err := filepath.Abs(filename)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to resolve path: %w", err)
|
||||
}
|
||||
|
||||
// Create a temporary tsconfig to parse just this one file.
|
||||
tmpDir, err := os.MkdirTemp("", "tsparser-*")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create temp dir: %w", err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
tsconfig := map[string]interface{}{
|
||||
"compilerOptions": map[string]interface{}{
|
||||
"target": "ESNext",
|
||||
"module": "ESNext",
|
||||
"jsx": "preserve",
|
||||
"experimentalDecorators": true,
|
||||
"noResolve": true,
|
||||
"noEmit": true,
|
||||
},
|
||||
"files": []string{absPath},
|
||||
}
|
||||
tsconfigData, _ := json.Marshal(tsconfig)
|
||||
tsconfigPath := filepath.Join(tmpDir, "tsconfig.json")
|
||||
if err := os.WriteFile(tsconfigPath, tsconfigData, 0644); err != nil {
|
||||
return nil, fmt.Errorf("failed to write tsconfig: %w", err)
|
||||
}
|
||||
|
||||
cmd := exec.Command(binary, "--project", tsconfigPath, "--noEmit")
|
||||
var stderr strings.Builder
|
||||
cmd.Stderr = &stderr
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
// tsgo reports type errors via exit code, but the parse may still succeed.
|
||||
// We only care about parse errors, not type errors.
|
||||
_ = output
|
||||
}
|
||||
|
||||
// tsgo doesn't dump the AST directly. For now, return a placeholder
|
||||
// indicating the file was processed. The actual AST extraction will
|
||||
// need the Go API or a custom tsgo build.
|
||||
return &ParseResult{
|
||||
AST: map[string]interface{}{
|
||||
"kind": "SourceFile",
|
||||
"_note": "placeholder: tsgo CLI does not support AST dump; awaiting Go API",
|
||||
"_file": absPath,
|
||||
"_error": stderr.String(),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetMetadata returns static TS7 metadata.
|
||||
func (p *StandaloneParser) GetMetadata() (*Metadata, error) {
|
||||
return GetStaticTS7Metadata(), nil
|
||||
}
|
||||
|
||||
// Reset is a no-op for the standalone parser.
|
||||
func (p *StandaloneParser) Reset() error { return nil }
|
||||
|
||||
// Close is a no-op for the standalone parser.
|
||||
func (p *StandaloneParser) Close() error { return nil }
|
||||
@@ -1,483 +0,0 @@
|
||||
package tsparser
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/textproto"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"github.com/github/codeql/javascript/extractor/lib/typescript-go/internal/astconv"
|
||||
)
|
||||
|
||||
// TsgoParser implements the Parser interface by running the tsgo binary
|
||||
// as a subprocess using its --api --async (JSON-RPC) mode.
|
||||
//
|
||||
// The tsgo API uses LSP-style Content-Length framing with JSON-RPC 2.0.
|
||||
// The API is project-based: you initialize, create a snapshot (optionally
|
||||
// opening a project/tsconfig), then query source files from that snapshot.
|
||||
// Source files are returned as a custom binary encoding (not JSON).
|
||||
//
|
||||
// This is a transitional implementation. When the typescript-go project
|
||||
// exposes a public Go API, this should be replaced with direct in-process
|
||||
// calls for better performance.
|
||||
type TsgoParser struct {
|
||||
config Config
|
||||
mu sync.Mutex
|
||||
cmd *exec.Cmd
|
||||
stdin io.WriteCloser
|
||||
stdout *bufio.Reader
|
||||
started bool
|
||||
nextID int
|
||||
|
||||
// Cached handles from the API session
|
||||
snapshotHandle string
|
||||
projectHandle string
|
||||
}
|
||||
|
||||
// NewTsgoParser creates a parser backed by the tsgo binary.
|
||||
func NewTsgoParser(config Config) *TsgoParser {
|
||||
return &TsgoParser{
|
||||
config: config,
|
||||
nextID: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *TsgoParser) findBinary() (string, error) {
|
||||
if p.config.TsgoBinary != "" {
|
||||
return p.config.TsgoBinary, nil
|
||||
}
|
||||
// Look for tsgo on PATH (installed via: npm install -g @typescript/native-preview)
|
||||
path, err := exec.LookPath("tsgo")
|
||||
if err == nil {
|
||||
// The npm-installed tsgo is a Node.js wrapper script that invokes the native binary.
|
||||
// Try to resolve the native binary directly so we don't need Node.js at runtime.
|
||||
if native := resolveNativeTsgo(path); native != "" {
|
||||
return native, nil
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
return "", fmt.Errorf("tsgo binary not found on PATH; install with: npm install -g @typescript/native-preview")
|
||||
}
|
||||
|
||||
// resolveNativeTsgo attempts to find the native tsgo binary inside an npm installation.
|
||||
// The npm package @typescript/native-preview installs a Node.js wrapper at bin/tsgo
|
||||
// which delegates to a platform-specific native binary at:
|
||||
// node_modules/@typescript/native-preview-<platform>-<arch>/lib/tsgo
|
||||
func resolveNativeTsgo(wrapperPath string) string {
|
||||
// Follow symlinks to find the real wrapper location
|
||||
resolved, err := filepath.EvalSymlinks(wrapperPath)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
// The wrapper is at <prefix>/bin/tsgo.js or <prefix>/bin/tsgo
|
||||
// The native binary is at <prefix>/node_modules/@typescript/native-preview-<os>-<arch>/lib/tsgo
|
||||
pkgDir := filepath.Dir(filepath.Dir(resolved))
|
||||
platformPkg := fmt.Sprintf("@typescript/native-preview-%s-%s", runtime.GOOS, runtime.GOARCH)
|
||||
native := filepath.Join(pkgDir, "node_modules", platformPkg, "lib", "tsgo")
|
||||
if info, err := os.Stat(native); err == nil && !info.IsDir() {
|
||||
return native
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// startProcess starts the tsgo subprocess without sending any API requests.
|
||||
func (p *TsgoParser) startProcess() error {
|
||||
if p.started {
|
||||
return nil
|
||||
}
|
||||
|
||||
binary, err := p.findBinary()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
p.cmd = exec.Command(binary, "--api", "--async")
|
||||
p.cmd.Stderr = p.config.Stderr
|
||||
if p.cmd.Stderr == nil {
|
||||
p.cmd.Stderr = os.Stderr
|
||||
}
|
||||
|
||||
stdin, err := p.cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create stdin pipe: %w", err)
|
||||
}
|
||||
p.stdin = stdin
|
||||
|
||||
stdout, err := p.cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create stdout pipe: %w", err)
|
||||
}
|
||||
p.stdout = bufio.NewReaderSize(stdout, 10*1024*1024)
|
||||
|
||||
if err := p.cmd.Start(); err != nil {
|
||||
return fmt.Errorf("failed to start tsgo: %w", err)
|
||||
}
|
||||
|
||||
p.started = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureInitialized starts the process and sends the initialize request.
|
||||
func (p *TsgoParser) ensureInitialized() error {
|
||||
if err := p.startProcess(); err != nil {
|
||||
return err
|
||||
}
|
||||
if p.snapshotHandle != "" {
|
||||
return nil // Already initialized
|
||||
}
|
||||
|
||||
// Send initialize request
|
||||
_, err := p.sendRequest("initialize", map[string]interface{}{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize tsgo API: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// jsonRPCRequest is a JSON-RPC 2.0 request.
|
||||
type jsonRPCRequest struct {
|
||||
JSONRPC string `json:"jsonrpc"`
|
||||
ID int `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params interface{} `json:"params,omitempty"`
|
||||
}
|
||||
|
||||
// jsonRPCResponse is a JSON-RPC 2.0 response.
|
||||
type jsonRPCResponse struct {
|
||||
JSONRPC string `json:"jsonrpc"`
|
||||
ID int `json:"id"`
|
||||
Result json.RawMessage `json:"result,omitempty"`
|
||||
Error *jsonRPCError `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type jsonRPCError struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// writeMessage writes an LSP-framed message (Content-Length header + body).
|
||||
func (p *TsgoParser) writeMessage(data []byte) error {
|
||||
header := fmt.Sprintf("Content-Length: %d\r\n\r\n", len(data))
|
||||
if _, err := io.WriteString(p.stdin, header); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := p.stdin.Write(data)
|
||||
return err
|
||||
}
|
||||
|
||||
// readMessage reads an LSP-framed message (Content-Length header + body).
|
||||
func (p *TsgoParser) readMessage() ([]byte, error) {
|
||||
tp := textproto.NewReader(p.stdout)
|
||||
header, err := tp.ReadMIMEHeader()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read message header: %w", err)
|
||||
}
|
||||
|
||||
lengthStr := header.Get("Content-Length")
|
||||
if lengthStr == "" {
|
||||
return nil, fmt.Errorf("missing Content-Length header")
|
||||
}
|
||||
length, err := strconv.Atoi(lengthStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid Content-Length: %w", err)
|
||||
}
|
||||
|
||||
body := make([]byte, length)
|
||||
if _, err := io.ReadFull(p.stdout, body); err != nil {
|
||||
return nil, fmt.Errorf("failed to read message body: %w", err)
|
||||
}
|
||||
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// sendRequest sends a JSON-RPC request and returns the response. Not locked.
|
||||
func (p *TsgoParser) sendRequest(method string, params interface{}) (json.RawMessage, error) {
|
||||
id := p.nextID
|
||||
p.nextID++
|
||||
|
||||
req := jsonRPCRequest{
|
||||
JSONRPC: "2.0",
|
||||
ID: id,
|
||||
Method: method,
|
||||
Params: params,
|
||||
}
|
||||
|
||||
data, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "[tsgo] >>> %s id=%d\n", method, id)
|
||||
|
||||
if err := p.writeMessage(data); err != nil {
|
||||
return nil, fmt.Errorf("failed to write request: %w", err)
|
||||
}
|
||||
|
||||
// Read responses, skipping notifications (messages without a matching id).
|
||||
// In --async mode, tsgo may send diagnostic notifications between responses.
|
||||
for {
|
||||
respData, err := p.readMessage()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
var resp jsonRPCResponse
|
||||
if err := json.Unmarshal(respData, &resp); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse response: %w", err)
|
||||
}
|
||||
|
||||
// Skip notifications (id=0 means no id field was present in JSON)
|
||||
if resp.ID != id {
|
||||
continue
|
||||
}
|
||||
|
||||
if resp.Error != nil {
|
||||
return nil, fmt.Errorf("tsgo API error %d: %s", resp.Error.Code, resp.Error.Message)
|
||||
}
|
||||
|
||||
return resp.Result, nil
|
||||
}
|
||||
}
|
||||
|
||||
// call sends a request with proper locking and initialization.
|
||||
func (p *TsgoParser) call(method string, params interface{}) (json.RawMessage, error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
if err := p.ensureInitialized(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return p.sendRequest(method, params)
|
||||
}
|
||||
|
||||
// updateSnapshotResponse is the response from the updateSnapshot API call.
|
||||
type updateSnapshotResponse struct {
|
||||
Snapshot string `json:"snapshot"`
|
||||
Projects []struct {
|
||||
ID string `json:"id"`
|
||||
ConfigFileName string `json:"configFileName"`
|
||||
} `json:"projects"`
|
||||
}
|
||||
|
||||
// ensureProjectOpen opens a project for the given file.
|
||||
// The tsgo API requires a tsconfig for project opening, so if none exists
|
||||
// in the file's directory, we create a temporary one.
|
||||
func (p *TsgoParser) ensureProjectOpen(filename string) error {
|
||||
if p.snapshotHandle != "" && p.projectHandle != "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
dir := filepath.Dir(filename)
|
||||
base := filepath.Base(filename)
|
||||
tsconfigPath := filepath.Join(dir, "tsconfig.json")
|
||||
|
||||
// If no tsconfig exists, create a temporary one
|
||||
createdTsconfig := false
|
||||
if _, err := os.Stat(tsconfigPath); os.IsNotExist(err) {
|
||||
tsconfig := fmt.Sprintf(`{
|
||||
"compilerOptions": {
|
||||
"target": "esnext",
|
||||
"module": "esnext",
|
||||
"noEmit": true,
|
||||
"strict": false,
|
||||
"allowJs": true
|
||||
},
|
||||
"files": [%q]
|
||||
}`, base)
|
||||
if err := os.WriteFile(tsconfigPath, []byte(tsconfig), 0644); err != nil {
|
||||
return fmt.Errorf("failed to create temporary tsconfig: %w", err)
|
||||
}
|
||||
createdTsconfig = true
|
||||
}
|
||||
|
||||
result, err := p.sendRequest("updateSnapshot", map[string]interface{}{
|
||||
"openProject": tsconfigPath,
|
||||
})
|
||||
|
||||
// Clean up temporary tsconfig
|
||||
if createdTsconfig {
|
||||
os.Remove(tsconfigPath)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open project: %w", err)
|
||||
}
|
||||
|
||||
var resp updateSnapshotResponse
|
||||
if err := json.Unmarshal(result, &resp); err != nil {
|
||||
return fmt.Errorf("failed to parse updateSnapshot response: %w", err)
|
||||
}
|
||||
|
||||
p.snapshotHandle = resp.Snapshot
|
||||
if len(resp.Projects) > 0 {
|
||||
p.projectHandle = resp.Projects[0].ID
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Parse parses the given file using the tsgo API.
|
||||
//
|
||||
// The tsgo API is project-based, so for each parse request we ensure
|
||||
// a project is open, then call getSourceFile. The response is a custom
|
||||
// binary encoding of the AST (not JSON).
|
||||
//
|
||||
// When the public Go API becomes available, this should be replaced
|
||||
// with direct parser.ParseSourceFile() calls.
|
||||
func (p *TsgoParser) Parse(filename string) (*ParseResult, error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
if err := p.ensureInitialized(); err != nil {
|
||||
return nil, fmt.Errorf("parse %s: %w", filename, err)
|
||||
}
|
||||
|
||||
if err := p.ensureProjectOpen(filename); err != nil {
|
||||
return nil, fmt.Errorf("parse %s: %w", filename, err)
|
||||
}
|
||||
|
||||
params := map[string]interface{}{
|
||||
"file": filename,
|
||||
}
|
||||
if p.snapshotHandle != "" {
|
||||
params["snapshot"] = p.snapshotHandle
|
||||
}
|
||||
if p.projectHandle != "" {
|
||||
params["project"] = p.projectHandle
|
||||
}
|
||||
|
||||
result, err := p.sendRequest("getSourceFile", params)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse %s: %w", filename, err)
|
||||
}
|
||||
|
||||
// The result is {"data":"<base64>"} containing a binary-encoded AST.
|
||||
var dataResp struct {
|
||||
Data string `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(result, &dataResp); err != nil {
|
||||
return nil, fmt.Errorf("parse %s: failed to parse getSourceFile response: %w", filename, err)
|
||||
}
|
||||
|
||||
binaryAST, err := astconv.DecodeBinaryASTFromBase64(dataResp.Data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse %s: failed to decode binary AST: %w", filename, err)
|
||||
}
|
||||
|
||||
// Fetch syntactic diagnostics (parse errors) from the compiler.
|
||||
diags := p.getSyntacticDiagnostics(filename)
|
||||
|
||||
kindToName := BuildKindToNameMap()
|
||||
converter := astconv.NewConverter(binaryAST, kindToName)
|
||||
converter.SetParseDiagnostics(diags)
|
||||
astObj, err := converter.Convert()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse %s: failed to convert AST: %w", filename, err)
|
||||
}
|
||||
|
||||
filtered := astconv.FilterWhitelist(astObj)
|
||||
|
||||
return &ParseResult{
|
||||
AST: filtered,
|
||||
RawData: []byte(dataResp.Data),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// getSyntacticDiagnostics fetches parse errors from the tsgo API.
|
||||
// Only includes true parse errors (diagnostic code < 2000), not semantic-level
|
||||
// diagnostics like deprecation warnings that TS7 added (e.g., code 2880 for
|
||||
// import assertions). Returns an empty slice on error (best-effort).
|
||||
func (p *TsgoParser) getSyntacticDiagnostics(filename string) []astconv.ParseDiagnostic {
|
||||
params := map[string]interface{}{
|
||||
"file": filename,
|
||||
}
|
||||
if p.snapshotHandle != "" {
|
||||
params["snapshot"] = p.snapshotHandle
|
||||
}
|
||||
if p.projectHandle != "" {
|
||||
params["project"] = p.projectHandle
|
||||
}
|
||||
|
||||
result, err := p.sendRequest("getSyntacticDiagnostics", params)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var rawDiags []struct {
|
||||
Pos int `json:"pos"`
|
||||
End int `json:"end"`
|
||||
Code int `json:"code"`
|
||||
Category int `json:"category"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
if err := json.Unmarshal(result, &rawDiags); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
diags := make([]astconv.ParseDiagnostic, 0, len(rawDiags))
|
||||
for _, d := range rawDiags {
|
||||
// Only include genuine parse errors (codes 1000-1999).
|
||||
// Higher codes are semantic diagnostics that TS7 reports as "syntactic"
|
||||
// but which don't indicate actual parse failures.
|
||||
if d.Code < 1000 || d.Code >= 2000 {
|
||||
continue
|
||||
}
|
||||
diags = append(diags, astconv.ParseDiagnostic{
|
||||
Pos: d.Pos,
|
||||
End: d.End,
|
||||
MessageText: d.Text,
|
||||
})
|
||||
}
|
||||
return diags
|
||||
}
|
||||
|
||||
// GetMetadata returns the syntax kinds and node flags.
|
||||
func (p *TsgoParser) GetMetadata() (*Metadata, error) {
|
||||
return GetStaticTS7Metadata(), nil
|
||||
}
|
||||
|
||||
// Reset resets the parser state, killing and restarting the subprocess.
|
||||
func (p *TsgoParser) Reset() error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
p.killProcess()
|
||||
p.started = false
|
||||
p.nextID = 1
|
||||
p.snapshotHandle = ""
|
||||
p.projectHandle = ""
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close shuts down the tsgo subprocess.
|
||||
func (p *TsgoParser) Close() error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
p.killProcess()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *TsgoParser) killProcess() {
|
||||
if !p.started {
|
||||
return
|
||||
}
|
||||
|
||||
if p.stdin != nil {
|
||||
p.stdin.Close()
|
||||
}
|
||||
if p.cmd != nil && p.cmd.Process != nil {
|
||||
p.cmd.Process.Kill()
|
||||
p.cmd.Wait() //nolint:errcheck
|
||||
}
|
||||
p.started = false
|
||||
}
|
||||
@@ -1,302 +0,0 @@
|
||||
package tsparser
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTsgoInitialize(t *testing.T) {
|
||||
if _, err := exec.LookPath("tsgo"); err != nil {
|
||||
t.Skip("tsgo not found on PATH; install with: npm install -g @typescript/native-preview")
|
||||
}
|
||||
|
||||
parser := NewTsgoParser(Config{Stderr: os.Stderr})
|
||||
defer parser.Close()
|
||||
|
||||
// Test that we can start the process and send the initialize request
|
||||
parser.mu.Lock()
|
||||
err := parser.startProcess()
|
||||
if err != nil {
|
||||
parser.mu.Unlock()
|
||||
t.Fatalf("Failed to start tsgo process: %v", err)
|
||||
}
|
||||
|
||||
result, err := parser.sendRequest("initialize", map[string]interface{}{})
|
||||
parser.mu.Unlock()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to send initialize request: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Initialize response: %s", string(result))
|
||||
|
||||
// Parse the response
|
||||
var initResp struct {
|
||||
UseCaseSensitiveFileNames bool `json:"useCaseSensitiveFileNames"`
|
||||
CurrentDirectory string `json:"currentDirectory"`
|
||||
}
|
||||
if err := json.Unmarshal(result, &initResp); err != nil {
|
||||
t.Fatalf("Failed to parse initialize response: %v", err)
|
||||
}
|
||||
|
||||
if initResp.CurrentDirectory == "" {
|
||||
t.Error("Expected non-empty CurrentDirectory in initialize response")
|
||||
}
|
||||
t.Logf("Initialized: caseSensitive=%v, cwd=%s",
|
||||
initResp.UseCaseSensitiveFileNames, initResp.CurrentDirectory)
|
||||
}
|
||||
|
||||
func TestTsgoPing(t *testing.T) {
|
||||
if _, err := exec.LookPath("tsgo"); err != nil {
|
||||
t.Skip("tsgo not found on PATH")
|
||||
}
|
||||
|
||||
parser := NewTsgoParser(Config{Stderr: os.Stderr})
|
||||
defer parser.Close()
|
||||
|
||||
parser.mu.Lock()
|
||||
if err := parser.startProcess(); err != nil {
|
||||
parser.mu.Unlock()
|
||||
t.Fatalf("Failed to start tsgo: %v", err)
|
||||
}
|
||||
|
||||
result, err := parser.sendRequest("ping", nil)
|
||||
parser.mu.Unlock()
|
||||
if err != nil {
|
||||
t.Fatalf("Ping failed: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Ping response: %s", string(result))
|
||||
}
|
||||
|
||||
func TestTsgoUpdateSnapshotAndGetSourceFile(t *testing.T) {
|
||||
if _, err := exec.LookPath("tsgo"); err != nil {
|
||||
t.Skip("tsgo not found on PATH")
|
||||
}
|
||||
|
||||
// Find the sample test file
|
||||
testFile := findTestFile(t)
|
||||
testDir := filepath.Dir(testFile)
|
||||
|
||||
// Create a minimal tsconfig.json for the test file
|
||||
tsconfigPath := filepath.Join(testDir, "tsconfig.json")
|
||||
tsconfig := []byte(`{
|
||||
"compilerOptions": {
|
||||
"target": "esnext",
|
||||
"module": "esnext",
|
||||
"noEmit": true,
|
||||
"strict": false
|
||||
},
|
||||
"files": ["sample.ts"]
|
||||
}`)
|
||||
if err := os.WriteFile(tsconfigPath, tsconfig, 0644); err != nil {
|
||||
t.Fatalf("Failed to create tsconfig.json: %v", err)
|
||||
}
|
||||
defer os.Remove(tsconfigPath)
|
||||
|
||||
var stderr bytes.Buffer
|
||||
parser := NewTsgoParser(Config{Stderr: &stderr})
|
||||
defer parser.Close()
|
||||
|
||||
parser.mu.Lock()
|
||||
defer parser.mu.Unlock()
|
||||
|
||||
// Step 1: Start and initialize
|
||||
if err := parser.startProcess(); err != nil {
|
||||
t.Fatalf("Failed to start tsgo: %v", err)
|
||||
}
|
||||
|
||||
initResult, err := parser.sendRequest("initialize", map[string]interface{}{})
|
||||
if err != nil {
|
||||
t.Fatalf("Initialize failed: %v", err)
|
||||
}
|
||||
t.Logf("Initialize: %s", string(initResult))
|
||||
|
||||
// Step 2: Update snapshot with project
|
||||
snapResult, err := parser.sendRequest("updateSnapshot", map[string]interface{}{
|
||||
"openProject": tsconfigPath,
|
||||
})
|
||||
if err != nil {
|
||||
t.Logf("Stderr output: %s", stderr.String())
|
||||
t.Fatalf("updateSnapshot failed: %v", err)
|
||||
}
|
||||
t.Logf("UpdateSnapshot: %s", string(snapResult))
|
||||
|
||||
var snapResp updateSnapshotResponse
|
||||
if err := json.Unmarshal(snapResult, &snapResp); err != nil {
|
||||
t.Fatalf("Failed to parse updateSnapshot response: %v", err)
|
||||
}
|
||||
|
||||
if snapResp.Snapshot == "" {
|
||||
t.Fatal("Expected non-empty snapshot handle")
|
||||
}
|
||||
t.Logf("Got snapshot: %s, %d projects", snapResp.Snapshot, len(snapResp.Projects))
|
||||
for i, p := range snapResp.Projects {
|
||||
t.Logf(" Project %d: id=%s config=%s", i, p.ID, p.ConfigFileName)
|
||||
}
|
||||
|
||||
if len(snapResp.Projects) == 0 {
|
||||
t.Fatal("Expected at least one project in snapshot")
|
||||
}
|
||||
|
||||
// Step 3: Get source file
|
||||
sfResult, err := parser.sendRequest("getSourceFile", map[string]interface{}{
|
||||
"snapshot": snapResp.Snapshot,
|
||||
"project": snapResp.Projects[0].ID,
|
||||
"file": testFile,
|
||||
})
|
||||
if err != nil {
|
||||
t.Logf("Stderr output: %s", stderr.String())
|
||||
t.Fatalf("getSourceFile failed: %v", err)
|
||||
}
|
||||
|
||||
// The response should contain base64-encoded binary data
|
||||
t.Logf("getSourceFile response length: %d bytes", len(sfResult))
|
||||
if len(sfResult) < 10 {
|
||||
t.Logf("getSourceFile response: %s", string(sfResult))
|
||||
} else {
|
||||
t.Logf("getSourceFile response (first 200 chars): %s", string(sfResult[:min(200, len(sfResult))]))
|
||||
}
|
||||
|
||||
if len(sfResult) == 0 || string(sfResult) == "null" {
|
||||
t.Error("Expected non-empty source file response")
|
||||
} else {
|
||||
t.Logf("Successfully retrieved source file data from tsgo API!")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTsgoGetMetadata(t *testing.T) {
|
||||
if _, err := exec.LookPath("tsgo"); err != nil {
|
||||
t.Skip("tsgo not found on PATH")
|
||||
}
|
||||
|
||||
parser := NewTsgoParser(Config{Stderr: os.Stderr})
|
||||
defer parser.Close()
|
||||
|
||||
meta, err := parser.GetMetadata()
|
||||
if err != nil {
|
||||
t.Fatalf("GetMetadata failed: %v", err)
|
||||
}
|
||||
|
||||
if len(meta.SyntaxKinds) == 0 {
|
||||
t.Error("Expected non-empty SyntaxKinds")
|
||||
}
|
||||
if _, ok := meta.SyntaxKinds["SourceFile"]; !ok {
|
||||
t.Error("Expected SyntaxKinds to contain 'SourceFile'")
|
||||
}
|
||||
if len(meta.NodeFlags) == 0 {
|
||||
t.Error("Expected non-empty NodeFlags")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStaticMetadata(t *testing.T) {
|
||||
meta := GetStaticTS7Metadata()
|
||||
|
||||
required := []string{"SourceFile", "Identifier", "Block", "VariableStatement",
|
||||
"FunctionDeclaration", "ClassDeclaration", "InterfaceDeclaration"}
|
||||
for _, kind := range required {
|
||||
if _, ok := meta.SyntaxKinds[kind]; !ok {
|
||||
t.Errorf("Missing required SyntaxKind: %s", kind)
|
||||
}
|
||||
}
|
||||
|
||||
requiredFlags := []string{"None", "Let", "Const", "Namespace"}
|
||||
for _, flag := range requiredFlags {
|
||||
if _, ok := meta.NodeFlags[flag]; !ok {
|
||||
t.Errorf("Missing required NodeFlag: %s", flag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func findTestFile(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir, _ := os.Getwd()
|
||||
for {
|
||||
candidate := filepath.Join(dir, "testdata", "sample.ts")
|
||||
if _, err := os.Stat(candidate); err == nil {
|
||||
return candidate
|
||||
}
|
||||
parent := filepath.Dir(dir)
|
||||
if parent == dir {
|
||||
break
|
||||
}
|
||||
dir = parent
|
||||
}
|
||||
t.Fatal("Could not find testdata/sample.ts")
|
||||
return ""
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func TestTsgoParse(t *testing.T) {
|
||||
if _, err := exec.LookPath("tsgo"); err != nil {
|
||||
t.Skip("tsgo not found on PATH")
|
||||
}
|
||||
|
||||
sampleFile := findTestFile(t)
|
||||
parser := NewTsgoParser(Config{Stderr: os.Stderr})
|
||||
defer parser.Close()
|
||||
|
||||
result, err := parser.Parse(sampleFile)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse failed: %v", err)
|
||||
}
|
||||
|
||||
ast, ok := result.AST.(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("Expected AST to be map[string]interface{}, got %T", result.AST)
|
||||
}
|
||||
|
||||
// Verify the root is a SourceFile
|
||||
kindVal, ok := ast["kind"]
|
||||
if !ok {
|
||||
t.Fatal("Missing 'kind' property on root node")
|
||||
}
|
||||
kindNum, ok := kindVal.(int)
|
||||
if !ok {
|
||||
t.Fatalf("Expected 'kind' to be int, got %T", kindVal)
|
||||
}
|
||||
if kindNum != 307 { // SourceFile = 307 in TS7
|
||||
t.Errorf("Expected root kind=307 (SourceFile), got %d", kindNum)
|
||||
}
|
||||
|
||||
// Verify $pos and $end
|
||||
if _, ok := ast["$pos"]; !ok {
|
||||
t.Error("Missing '$pos' property")
|
||||
}
|
||||
if _, ok := ast["$end"]; !ok {
|
||||
t.Error("Missing '$end' property")
|
||||
}
|
||||
|
||||
// Verify statements array
|
||||
stmts, ok := ast["statements"]
|
||||
if !ok {
|
||||
t.Fatal("Missing 'statements' property")
|
||||
}
|
||||
stmtsArr, ok := stmts.([]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("Expected statements to be array, got %T", stmts)
|
||||
}
|
||||
if len(stmtsArr) == 0 {
|
||||
t.Error("Expected non-empty statements array")
|
||||
}
|
||||
|
||||
// Print a nicely indented snippet for debug
|
||||
jsonBytes, err := json.MarshalIndent(ast, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to marshal AST: %v", err)
|
||||
}
|
||||
snippet := string(jsonBytes)
|
||||
if len(snippet) > 2000 {
|
||||
snippet = snippet[:2000] + "\n... (truncated)"
|
||||
}
|
||||
t.Logf("Parsed AST (first 2000 chars):\n%s", snippet)
|
||||
}
|
||||
@@ -1,409 +0,0 @@
|
||||
// Package validation provides a Go-based test framework for comparing
|
||||
// JSON output between the Node.js and Go TypeScript parser wrappers.
|
||||
//
|
||||
// Run with: go test ./internal/validation/ -v
|
||||
//
|
||||
// This requires both the Go wrapper binary and Node.js with the
|
||||
// TypeScript wrapper available.
|
||||
package validation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// normalizeJSON parses JSON and re-serializes it with sorted keys for
|
||||
// stable comparison.
|
||||
func normalizeJSON(data []byte) ([]byte, error) {
|
||||
var obj interface{}
|
||||
if err := json.Unmarshal(data, &obj); err != nil {
|
||||
return nil, fmt.Errorf("invalid JSON: %w", err)
|
||||
}
|
||||
return json.MarshalIndent(sortKeys(obj), "", " ")
|
||||
}
|
||||
|
||||
// sortKeys recursively sorts map keys in a JSON-like structure.
|
||||
func sortKeys(v interface{}) interface{} {
|
||||
switch val := v.(type) {
|
||||
case map[string]interface{}:
|
||||
sorted := make(map[string]interface{}, len(val))
|
||||
for k, v := range val {
|
||||
sorted[k] = sortKeys(v)
|
||||
}
|
||||
return sorted
|
||||
case []interface{}:
|
||||
result := make([]interface{}, len(val))
|
||||
for i, v := range val {
|
||||
result[i] = sortKeys(v)
|
||||
}
|
||||
return result
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}
|
||||
|
||||
// findProjectRoot finds the typescript-go project root by walking up from
|
||||
// the current test file.
|
||||
func findProjectRoot(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for {
|
||||
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
|
||||
return dir
|
||||
}
|
||||
parent := filepath.Dir(dir)
|
||||
if parent == dir {
|
||||
t.Fatal("could not find project root (no go.mod found)")
|
||||
}
|
||||
dir = parent
|
||||
}
|
||||
}
|
||||
|
||||
// findNodeJSWrapper finds the compiled Node.js TypeScript wrapper.
|
||||
func findNodeJSWrapper(projectRoot string) (string, error) {
|
||||
tsDir := filepath.Join(projectRoot, "..", "typescript")
|
||||
|
||||
jsPath := filepath.Join(tsDir, "build", "main.js")
|
||||
if _, err := os.Stat(jsPath); err == nil {
|
||||
return jsPath, nil
|
||||
}
|
||||
|
||||
tsPath := filepath.Join(tsDir, "src", "main.ts")
|
||||
if _, err := os.Stat(tsPath); err != nil {
|
||||
return "", fmt.Errorf("Node.js wrapper not found at %s", tsPath)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(filepath.Join(tsDir, "node_modules")); err != nil {
|
||||
cmd := exec.Command("npm", "install", "--no-audit", "--no-fund")
|
||||
cmd.Dir = tsDir
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return "", fmt.Errorf("npm install failed in %s: %v\n%s", tsDir, err, output)
|
||||
}
|
||||
}
|
||||
|
||||
cmd := exec.Command("npm", "run", "build")
|
||||
cmd.Dir = tsDir
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return "", fmt.Errorf("npm run build failed in %s: %v\n%s", tsDir, err, output)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(jsPath); err == nil {
|
||||
return jsPath, nil
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("Node.js wrapper not found after build; expected at %s", jsPath)
|
||||
}
|
||||
|
||||
// parseWithNodeJSProtocol parses a file using the Node.js wrapper's protocol.
|
||||
// It starts the wrapper, sends the protocol commands, and extracts the AST response.
|
||||
func parseWithNodeJSProtocol(wrapperPath, filename string) ([]byte, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(ctx, "node", "--no-warnings", wrapperPath)
|
||||
var stderr, stdout bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
cmd.Stdout = &stdout
|
||||
|
||||
stdinPipe, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("failed to start node.js wrapper: %v", err)
|
||||
}
|
||||
|
||||
commands := []string{
|
||||
fmt.Sprintf(`{"command":"parse","filename":"%s"}`, escapeJSON(filename)),
|
||||
`{"command":"quit"}`,
|
||||
}
|
||||
|
||||
for _, c := range commands {
|
||||
if _, err := io.WriteString(stdinPipe, c+"\n"); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
stdinPipe.Close()
|
||||
|
||||
err = cmd.Wait()
|
||||
if ctx.Err() != nil {
|
||||
return nil, fmt.Errorf("Node.js wrapper timed out; stderr: %s", stderr.String())
|
||||
}
|
||||
|
||||
lines := strings.Split(strings.TrimSpace(stdout.String()), "\n")
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var resp map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(line), &resp); err != nil {
|
||||
continue
|
||||
}
|
||||
if resp["type"] == "ast" {
|
||||
return []byte(line), nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("no AST response found in output; stderr: %s; stdout lines: %d",
|
||||
stderr.String(), len(lines))
|
||||
}
|
||||
|
||||
// parseWithGoProtocol parses a file using the Go wrapper's protocol.
|
||||
func parseWithGoProtocol(binaryPath, filename string) ([]byte, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(ctx, binaryPath)
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
stdinPipe, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("failed to start Go wrapper: %v", err)
|
||||
}
|
||||
|
||||
commands := []string{
|
||||
fmt.Sprintf(`{"command":"parse","filename":"%s"}`, escapeJSON(filename)),
|
||||
`{"command":"quit"}`,
|
||||
}
|
||||
|
||||
for _, c := range commands {
|
||||
if _, err := io.WriteString(stdinPipe, c+"\n"); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
stdinPipe.Close()
|
||||
|
||||
err = cmd.Wait()
|
||||
if ctx.Err() != nil {
|
||||
return nil, fmt.Errorf("Go wrapper timed out; stderr: %s", stderr.String())
|
||||
}
|
||||
if err != nil {
|
||||
// Non-zero exit is ok if we got output (error responses are valid)
|
||||
}
|
||||
|
||||
lines := strings.Split(strings.TrimSpace(stdout.String()), "\n")
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var resp map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(line), &resp); err != nil {
|
||||
continue
|
||||
}
|
||||
if resp["type"] == "ast" {
|
||||
return []byte(line), nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("no AST response in output; stderr: %s; stdout: %q",
|
||||
stderr.String(), stdout.String())
|
||||
}
|
||||
|
||||
func escapeJSON(s string) string {
|
||||
b, _ := json.Marshal(s)
|
||||
// Strip the surrounding quotes since we embed in a template
|
||||
return string(b[1 : len(b)-1])
|
||||
}
|
||||
|
||||
// TestCompareOutputs compares the JSON output of both wrappers for test files.
|
||||
func TestCompareOutputs(t *testing.T) {
|
||||
projectRoot := findProjectRoot(t)
|
||||
|
||||
// Build the Go wrapper
|
||||
binaryPath := filepath.Join(projectRoot, "bin", "typescript-parser-wrapper")
|
||||
buildCmd := exec.Command("go", "build", "-o", binaryPath, "./cmd/typescript-parser-wrapper/")
|
||||
buildCmd.Dir = projectRoot
|
||||
if output, err := buildCmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("failed to build Go wrapper: %v\n%s", err, output)
|
||||
}
|
||||
|
||||
// Find the Node.js wrapper
|
||||
nodejsWrapper, err := findNodeJSWrapper(projectRoot)
|
||||
if err != nil {
|
||||
t.Skipf("Skipping comparison test: %v", err)
|
||||
}
|
||||
|
||||
if _, err := exec.LookPath("node"); err != nil {
|
||||
t.Skip("Skipping comparison test: node not found on PATH")
|
||||
}
|
||||
|
||||
// Gather test files
|
||||
testFiles, err := filepath.Glob(filepath.Join(projectRoot, "testdata", "*.ts"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Extractor test inputs can be included via VALIDATION_EXTRACTOR_TESTS=1
|
||||
if os.Getenv("VALIDATION_EXTRACTOR_TESTS") == "1" {
|
||||
extractorTestDir := filepath.Join(projectRoot, "..", "..", "tests", "ts", "input")
|
||||
if extractorFiles, err := filepath.Glob(filepath.Join(extractorTestDir, "*.ts")); err == nil {
|
||||
testFiles = append(testFiles, extractorFiles...)
|
||||
}
|
||||
}
|
||||
|
||||
if len(testFiles) == 0 {
|
||||
t.Skip("No test files found")
|
||||
}
|
||||
|
||||
for _, file := range testFiles {
|
||||
basename := filepath.Base(file)
|
||||
t.Run(basename, func(t *testing.T) {
|
||||
nodejsOut, err := parseWithNodeJSProtocol(nodejsWrapper, file)
|
||||
if err != nil {
|
||||
t.Skipf("Node.js wrapper failed: %v", err)
|
||||
}
|
||||
|
||||
goOut, err := parseWithGoProtocol(binaryPath, file)
|
||||
if err != nil {
|
||||
t.Skipf("Go wrapper failed: %v", err)
|
||||
}
|
||||
|
||||
nodejsNorm, err := normalizeJSON(bytes.TrimSpace(nodejsOut))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to normalize Node.js output: %v", err)
|
||||
}
|
||||
|
||||
goNorm, err := normalizeJSON(bytes.TrimSpace(goOut))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to normalize Go output: %v", err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(nodejsNorm, goNorm) {
|
||||
outDir := filepath.Join(projectRoot, "validation-output")
|
||||
os.MkdirAll(outDir, 0755)
|
||||
os.WriteFile(filepath.Join(outDir, basename+".nodejs.json"), nodejsNorm, 0644)
|
||||
os.WriteFile(filepath.Join(outDir, basename+".go.json"), goNorm, 0644)
|
||||
|
||||
// Parse both outputs and check for structural diffs (ignoring expected kind/flags differences)
|
||||
var nodejsObj, goObj map[string]interface{}
|
||||
json.Unmarshal(nodejsNorm, &nodejsObj)
|
||||
json.Unmarshal(goNorm, &goObj)
|
||||
|
||||
structural := countStructuralDiffs(nodejsObj["ast"], goObj["ast"], "root")
|
||||
if structural > 0 {
|
||||
t.Errorf("Output has %d structural diff(s) for %s (beyond expected kind/flags diffs)\n"+
|
||||
" Node.js output saved to: validation-output/%s.nodejs.json\n"+
|
||||
" Go output saved to: validation-output/%s.go.json",
|
||||
structural, basename, basename, basename)
|
||||
} else {
|
||||
t.Logf("Output for %s differs only in expected kind/flags/token numeric values (TS5 vs TS7)", basename)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeJSON(t *testing.T) {
|
||||
input := `{"b":2,"a":1,"c":{"z":26,"y":25}}`
|
||||
expected := `{
|
||||
"a": 1,
|
||||
"b": 2,
|
||||
"c": {
|
||||
"y": 25,
|
||||
"z": 26
|
||||
}
|
||||
}`
|
||||
result, err := normalizeJSON([]byte(input))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(result) != expected {
|
||||
t.Errorf("got:\n%s\nexpected:\n%s", string(result), expected)
|
||||
}
|
||||
}
|
||||
|
||||
// numericValueKeys are JSON object keys whose numeric values are expected to differ
|
||||
// between TS5 and TS7 (SyntaxKind/NodeFlags numeric values).
|
||||
var numericValueKeys = map[string]bool{
|
||||
"kind": true,
|
||||
"flags": true,
|
||||
"token": true,
|
||||
"operator": true,
|
||||
}
|
||||
|
||||
// countStructuralDiffs recursively compares two JSON values and returns the
|
||||
// number of differences that are NOT expected TS5↔TS7 numeric kind/flags diffs.
|
||||
func countStructuralDiffs(a, b interface{}, path string) int {
|
||||
count := 0
|
||||
switch av := a.(type) {
|
||||
case map[string]interface{}:
|
||||
bv, ok := b.(map[string]interface{})
|
||||
if !ok {
|
||||
return 1
|
||||
}
|
||||
allKeys := map[string]bool{}
|
||||
for k := range av {
|
||||
allKeys[k] = true
|
||||
}
|
||||
for k := range bv {
|
||||
allKeys[k] = true
|
||||
}
|
||||
for k := range allKeys {
|
||||
aVal, aOk := av[k]
|
||||
bVal, bOk := bv[k]
|
||||
if !aOk || !bOk {
|
||||
count++
|
||||
continue
|
||||
}
|
||||
count += countStructuralDiffs(aVal, bVal, path+"."+k)
|
||||
}
|
||||
case []interface{}:
|
||||
bv, ok := b.([]interface{})
|
||||
if !ok {
|
||||
return 1
|
||||
}
|
||||
if len(av) != len(bv) {
|
||||
return 1
|
||||
}
|
||||
for i := range av {
|
||||
count += countStructuralDiffs(av[i], bv[i], fmt.Sprintf("%s[%d]", path, i))
|
||||
}
|
||||
default:
|
||||
if a != b {
|
||||
// Check if this is an expected numeric diff for kind/flags/token/operator
|
||||
key := lastPathComponent(path)
|
||||
if numericValueKeys[key] {
|
||||
// Both must be numbers for this to be an expected diff
|
||||
_, aNum := a.(float64)
|
||||
_, bNum := b.(float64)
|
||||
if aNum && bNum {
|
||||
return 0 // Expected TS5↔TS7 numeric diff
|
||||
}
|
||||
}
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func lastPathComponent(path string) string {
|
||||
for i := len(path) - 1; i >= 0; i-- {
|
||||
if path[i] == '.' {
|
||||
return path[i+1:]
|
||||
}
|
||||
}
|
||||
return path
|
||||
}
|
||||
@@ -1,254 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# validate-output.sh — Compare JSON output between the Node.js and Go
|
||||
# TypeScript parser wrappers.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/validate-output.sh [<ts-file> ...]
|
||||
#
|
||||
# Without arguments, it validates all .ts files from the test input directory.
|
||||
#
|
||||
# Environment variables:
|
||||
# NODEJS_WRAPPER — Path to Node.js wrapper main.js (default: auto-detect)
|
||||
# GO_WRAPPER — Path to Go wrapper binary (default: builds from source)
|
||||
# TIMEOUT — Seconds to wait for each parse (default: 10)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
EXTRACTOR_LIB="$(cd "$PROJECT_DIR/.." && pwd)"
|
||||
TYPESCRIPT_DIR="$EXTRACTOR_LIB/typescript"
|
||||
TIMEOUT="${TIMEOUT:-10}"
|
||||
|
||||
# Locate the Node.js wrapper (prefer compiled .js)
|
||||
find_nodejs_wrapper() {
|
||||
local js_path="$TYPESCRIPT_DIR/build/main.js"
|
||||
if [ -f "$js_path" ]; then
|
||||
echo "$js_path"
|
||||
return
|
||||
fi
|
||||
echo ""
|
||||
}
|
||||
|
||||
NODEJS_WRAPPER="${NODEJS_WRAPPER:-$(find_nodejs_wrapper)}"
|
||||
|
||||
# Build and locate the Go wrapper
|
||||
GO_WRAPPER="${GO_WRAPPER:-$PROJECT_DIR/bin/typescript-parser-wrapper}"
|
||||
if [ ! -f "$GO_WRAPPER" ]; then
|
||||
echo "Building Go wrapper..."
|
||||
mkdir -p "$PROJECT_DIR/bin"
|
||||
(cd "$PROJECT_DIR" && go build -o bin/typescript-parser-wrapper ./cmd/typescript-parser-wrapper/) || {
|
||||
echo "Failed to build Go wrapper."
|
||||
GO_WRAPPER=""
|
||||
}
|
||||
fi
|
||||
|
||||
# Colors (disabled if not a terminal)
|
||||
if [ -t 1 ]; then
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
else
|
||||
RED='' GREEN='' YELLOW='' NC=''
|
||||
fi
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
SKIP=0
|
||||
|
||||
# Normalize JSON: sort keys, stable indentation
|
||||
normalize_json() {
|
||||
python3 -c "
|
||||
import json, sys
|
||||
try:
|
||||
obj = json.load(sys.stdin)
|
||||
print(json.dumps(obj, sort_keys=True, indent=2))
|
||||
except:
|
||||
sys.exit(1)
|
||||
" 2>/dev/null
|
||||
}
|
||||
|
||||
# Parse a file using the wrapper's stdin protocol.
|
||||
# Usage: parse_with_protocol <cmd> <file>
|
||||
# cmd: the shell command to start the wrapper (e.g., "node main.js" or "./wrapper")
|
||||
# file: absolute path to the .ts file
|
||||
#
|
||||
# Sends parse + quit commands on stdin and extracts the AST response line.
|
||||
parse_with_protocol() {
|
||||
local cmd="$1"
|
||||
local file="$2"
|
||||
|
||||
local output
|
||||
output=$(printf '{"command":"parse","filename":"%s"}\n{"command":"quit"}\n' "$file" \
|
||||
| timeout "$TIMEOUT" $cmd 2>/dev/null) || true
|
||||
|
||||
# Extract the line containing the AST response
|
||||
echo "$output" | while IFS= read -r line; do
|
||||
if echo "$line" | python3 -c "import json,sys; d=json.load(sys.stdin); sys.exit(0 if d.get('type')=='ast' else 1)" 2>/dev/null; then
|
||||
echo "$line"
|
||||
break
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Parse a file with the Node.js wrapper
|
||||
parse_nodejs() {
|
||||
local file="$1"
|
||||
if [ -z "$NODEJS_WRAPPER" ]; then
|
||||
echo ""
|
||||
return
|
||||
fi
|
||||
parse_with_protocol "node --no-warnings $NODEJS_WRAPPER" "$file"
|
||||
}
|
||||
|
||||
# Parse a file with the Go wrapper
|
||||
parse_go() {
|
||||
local file="$1"
|
||||
if [ -z "$GO_WRAPPER" ]; then
|
||||
echo ""
|
||||
return
|
||||
fi
|
||||
parse_with_protocol "$GO_WRAPPER" "$file"
|
||||
}
|
||||
|
||||
compare_output() {
|
||||
local file="$1"
|
||||
local basename
|
||||
basename="$(basename "$file")"
|
||||
|
||||
local nodejs_out go_out
|
||||
nodejs_out=$(parse_nodejs "$file")
|
||||
go_out=$(parse_go "$file")
|
||||
|
||||
if [ -z "$nodejs_out" ] && [ -z "$go_out" ]; then
|
||||
echo -e " ${YELLOW}SKIP${NC} $basename (both outputs empty)"
|
||||
SKIP=$((SKIP + 1))
|
||||
return
|
||||
fi
|
||||
|
||||
if [ -z "$nodejs_out" ]; then
|
||||
echo -e " ${YELLOW}SKIP${NC} $basename (Node.js output empty)"
|
||||
SKIP=$((SKIP + 1))
|
||||
return
|
||||
fi
|
||||
|
||||
if [ -z "$go_out" ]; then
|
||||
echo -e " ${YELLOW}SKIP${NC} $basename (Go output empty)"
|
||||
SKIP=$((SKIP + 1))
|
||||
return
|
||||
fi
|
||||
|
||||
local nodejs_norm go_norm
|
||||
nodejs_norm=$(echo "$nodejs_out" | normalize_json) || {
|
||||
echo -e " ${YELLOW}SKIP${NC} $basename (Node.js output not valid JSON)"
|
||||
SKIP=$((SKIP + 1))
|
||||
return
|
||||
}
|
||||
go_norm=$(echo "$go_out" | normalize_json) || {
|
||||
echo -e " ${YELLOW}SKIP${NC} $basename (Go output not valid JSON)"
|
||||
SKIP=$((SKIP + 1))
|
||||
return
|
||||
}
|
||||
|
||||
if [ "$nodejs_norm" = "$go_norm" ]; then
|
||||
echo -e " ${GREEN}PASS${NC} $basename (exact match)"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
# Check if differences are only expected TS5↔TS7 numeric kind/flags/token/operator values
|
||||
local structural_diffs
|
||||
structural_diffs=$(python3 -c "
|
||||
import json, sys
|
||||
|
||||
NUMERIC_VALUE_KEYS = {'kind', 'flags', 'token', 'operator'}
|
||||
|
||||
def count_structural(a, b, path='root'):
|
||||
count = 0
|
||||
if isinstance(a, dict) and isinstance(b, dict):
|
||||
keys = set(a) | set(b)
|
||||
for k in keys:
|
||||
# parseDiagnostics: Go always returns [], Node.js may have actual diagnostics
|
||||
if k == 'parseDiagnostics':
|
||||
continue
|
||||
if k not in a or k not in b:
|
||||
count += 1
|
||||
else:
|
||||
count += count_structural(a[k], b[k], path + '.' + k)
|
||||
elif isinstance(a, list) and isinstance(b, list):
|
||||
if len(a) != len(b):
|
||||
return 1
|
||||
for i in range(len(a)):
|
||||
count += count_structural(a[i], b[i], f'{path}[{i}]')
|
||||
elif a != b:
|
||||
key = path.rsplit('.', 1)[-1] if '.' in path else path
|
||||
if key in NUMERIC_VALUE_KEYS and isinstance(a, (int, float)) and isinstance(b, (int, float)):
|
||||
return 0
|
||||
count = 1
|
||||
return count
|
||||
|
||||
a = json.loads(sys.argv[1])
|
||||
b = json.loads(sys.argv[2])
|
||||
print(count_structural(a, b))
|
||||
" "$nodejs_norm" "$go_norm" 2>/dev/null) || structural_diffs="?"
|
||||
|
||||
if [ "$structural_diffs" = "0" ]; then
|
||||
echo -e " ${GREEN}PASS${NC} $basename (only expected TS5↔TS7 kind/flags diffs)"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo -e " ${RED}FAIL${NC} $basename ($structural_diffs structural diff(s))"
|
||||
FAIL=$((FAIL + 1))
|
||||
|
||||
# Save outputs for inspection
|
||||
local outdir="$PROJECT_DIR/validation-output"
|
||||
mkdir -p "$outdir"
|
||||
echo "$nodejs_norm" > "$outdir/${basename}.nodejs.json"
|
||||
echo "$go_norm" > "$outdir/${basename}.go.json"
|
||||
|
||||
# Show first few lines of diff
|
||||
diff <(echo "$nodejs_norm") <(echo "$go_norm") | head -30 || true
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Gather files
|
||||
files=()
|
||||
if [ $# -gt 0 ]; then
|
||||
files=("$@")
|
||||
else
|
||||
# Use extractor test inputs
|
||||
TEST_DIR="$EXTRACTOR_LIB/../tests/ts/input"
|
||||
if [ -d "$TEST_DIR" ]; then
|
||||
for f in "$TEST_DIR"/*.ts; do
|
||||
[ -f "$f" ] && files+=("$f")
|
||||
done
|
||||
fi
|
||||
|
||||
# Also use our own test data
|
||||
for f in "$PROJECT_DIR/testdata"/*.ts; do
|
||||
[ -f "$f" ] && files+=("$f")
|
||||
done
|
||||
fi
|
||||
|
||||
if [ ${#files[@]} -eq 0 ]; then
|
||||
echo "No TypeScript files to validate."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "=== TypeScript Parser Wrapper Validation ==="
|
||||
echo " Node.js wrapper: ${NODEJS_WRAPPER:-not found}"
|
||||
echo " Go wrapper: ${GO_WRAPPER:-not built}"
|
||||
echo " Files: ${#files[@]}"
|
||||
echo " Timeout: ${TIMEOUT}s per file"
|
||||
echo ""
|
||||
|
||||
for file in "${files[@]}"; do
|
||||
compare_output "$(realpath "$file")"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Results ==="
|
||||
echo -e " ${GREEN}PASS: $PASS${NC} ${RED}FAIL: $FAIL${NC} ${YELLOW}SKIP: $SKIP${NC}"
|
||||
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,77 +0,0 @@
|
||||
// Test file for validating the TypeScript parser wrapper.
|
||||
// This file exercises various TypeScript features to ensure the AST
|
||||
// serialization produces the correct output.
|
||||
|
||||
interface Greeter {
|
||||
greet(name: string): string;
|
||||
}
|
||||
|
||||
class HelloGreeter implements Greeter {
|
||||
private prefix: string;
|
||||
|
||||
constructor(prefix: string = "Hello") {
|
||||
this.prefix = prefix;
|
||||
}
|
||||
|
||||
greet(name: string): string {
|
||||
return `${this.prefix}, ${name}!`;
|
||||
}
|
||||
}
|
||||
|
||||
// Generics
|
||||
function identity<T>(arg: T): T {
|
||||
return arg;
|
||||
}
|
||||
|
||||
// Conditional types
|
||||
type IsString<T> = T extends string ? "yes" : "no";
|
||||
|
||||
// Async/await
|
||||
async function fetchData(url: string): Promise<string> {
|
||||
const response = await fetch(url);
|
||||
return response.text();
|
||||
}
|
||||
|
||||
// Destructuring
|
||||
const { a, b, ...rest } = { a: 1, b: 2, c: 3, d: 4 };
|
||||
const [first, second, ...remaining] = [1, 2, 3, 4, 5];
|
||||
|
||||
// Enums
|
||||
enum Direction {
|
||||
Up = "UP",
|
||||
Down = "DOWN",
|
||||
Left = "LEFT",
|
||||
Right = "RIGHT",
|
||||
}
|
||||
|
||||
// Type assertions
|
||||
const value = "hello" as unknown as number;
|
||||
|
||||
// Optional chaining
|
||||
const len = value?.toString()?.length;
|
||||
|
||||
// Nullish coalescing
|
||||
const result = len ?? 0;
|
||||
|
||||
// Decorators
|
||||
function log(target: any, key: string, descriptor: PropertyDescriptor) {
|
||||
return descriptor;
|
||||
}
|
||||
|
||||
// Namespace
|
||||
namespace Validation {
|
||||
export interface StringValidator {
|
||||
isAcceptable(s: string): boolean;
|
||||
}
|
||||
}
|
||||
|
||||
// Mapped types
|
||||
type Readonly<T> = {
|
||||
readonly [P in keyof T]: T[P];
|
||||
};
|
||||
|
||||
// Template literal types
|
||||
type EventName = `on${string}`;
|
||||
|
||||
// Export
|
||||
export { HelloGreeter, Direction, fetchData };
|
||||
@@ -56,11 +56,6 @@ import ch.qos.logback.classic.Level;
|
||||
* $SEMMLE_DIST/tools/typescript-parser-wrapper/main.js}; non-standard locations can be configured
|
||||
* using the property {@value #PARSER_WRAPPER_PATH_ENV_VAR}.
|
||||
*
|
||||
* <p>Alternatively, a Go-based parser wrapper can be used by setting the environment variable
|
||||
* {@value #USE_GO_PARSER_VAR} to {@code true}. This uses the TypeScript 7 (Go-based) compiler
|
||||
* and does not require Node.js. The Go binary location can be configured using {@value
|
||||
* #GO_PARSER_WRAPPER_PATH_ENV_VAR}.
|
||||
*
|
||||
* <p>The script launches the Node.js wrapper in the Node.js runtime, looking for {@code node} on
|
||||
* the {@code PATH} by default. Non-standard locations can be configured using the property {@value
|
||||
* #TYPESCRIPT_NODE_RUNTIME_VAR}, and additional arguments can be configured using the property
|
||||
@@ -129,23 +124,6 @@ public class TypeScriptParser {
|
||||
*/
|
||||
public static final String TYPESCRIPT_NODE_FLAGS = "SEMMLE_TYPESCRIPT_NODE_FLAGS";
|
||||
|
||||
/**
|
||||
* An environment variable that, when set to {@code true}, causes the extractor to use a Go-based
|
||||
* TypeScript parser wrapper (using TypeScript 7) instead of the Node.js wrapper.
|
||||
*
|
||||
* <p>This is experimental and does not require Node.js to be installed.
|
||||
*/
|
||||
public static final String USE_GO_PARSER_VAR = "SEMMLE_TYPESCRIPT_USE_GO_PARSER";
|
||||
|
||||
/**
|
||||
* An environment variable that can be set to indicate the location of the Go TypeScript parser
|
||||
* wrapper binary.
|
||||
*
|
||||
* <p>Only used when {@value #USE_GO_PARSER_VAR} is set to {@code true}.
|
||||
* Defaults to {@code $SEMMLE_DIST/tools/typescript-parser-wrapper-go} if not set.
|
||||
*/
|
||||
public static final String GO_PARSER_WRAPPER_PATH_ENV_VAR = "SEMMLE_TYPESCRIPT_GO_PARSER_WRAPPER";
|
||||
|
||||
/**
|
||||
* Exit code for Node.js in case of a fatal error from V8. This exit code sometimes occurs
|
||||
* when the process runs out of memory.
|
||||
@@ -163,9 +141,6 @@ public class TypeScriptParser {
|
||||
|
||||
private String parserWrapperCommand;
|
||||
|
||||
/** Whether we are using the Go-based TypeScript parser instead of Node.js. */
|
||||
private boolean useGoParser = "true".equalsIgnoreCase(Env.systemEnv().get(USE_GO_PARSER_VAR));
|
||||
|
||||
/** Streams for communicating with the Node.js parser wrapper process. */
|
||||
private BufferedWriter toParserWrapper;
|
||||
|
||||
@@ -196,19 +171,10 @@ public class TypeScriptParser {
|
||||
/**
|
||||
* Verifies that Node.js and TypeScript are installed and throws an exception otherwise.
|
||||
*
|
||||
* <p>When the Go parser is enabled, this only verifies the Go binary exists.
|
||||
*
|
||||
* @param verbose if true, log the Node.js executable path, version strings, and any additional
|
||||
* arguments.
|
||||
*/
|
||||
public void verifyInstallation(boolean verbose) {
|
||||
if (useGoParser) {
|
||||
File goWrapper = getGoParserWrapper();
|
||||
if (verbose) {
|
||||
System.out.println("Using Go TypeScript parser wrapper: " + goWrapper.getAbsolutePath());
|
||||
}
|
||||
return;
|
||||
}
|
||||
verifyNodeInstallation();
|
||||
if (verbose) {
|
||||
System.out.println("Found Node.js at: " + nodeJsRuntime);
|
||||
@@ -307,69 +273,8 @@ public class TypeScriptParser {
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Start the parser wrapper process (Node.js or Go). */
|
||||
private void setupParserWrapper() {
|
||||
if (useGoParser) {
|
||||
setupGoParserWrapper();
|
||||
} else {
|
||||
setupNodeParserWrapper();
|
||||
}
|
||||
}
|
||||
|
||||
/** Start the Go-based parser wrapper process. */
|
||||
private void setupGoParserWrapper() {
|
||||
File goWrapper = getGoParserWrapper();
|
||||
|
||||
List<String> cmd = new ArrayList<>();
|
||||
cmd.add(goWrapper.getAbsolutePath());
|
||||
|
||||
ProcessBuilder pb = new ProcessBuilder(cmd);
|
||||
parserWrapperCommand = StringUtil.glue(" ", cmd);
|
||||
|
||||
// Pass the tsgo binary location if configured
|
||||
String tsgoBinary = Env.systemEnv().get("SEMMLE_TYPESCRIPT_TSGO_BINARY");
|
||||
if (tsgoBinary != null) {
|
||||
pb.environment().put("SEMMLE_TYPESCRIPT_TSGO_BINARY", tsgoBinary);
|
||||
}
|
||||
|
||||
try {
|
||||
pb.redirectError(Redirect.INHERIT);
|
||||
parserWrapperProcess = pb.start();
|
||||
OutputStream os = parserWrapperProcess.getOutputStream();
|
||||
OutputStreamWriter osw = new OutputStreamWriter(os, "UTF-8");
|
||||
toParserWrapper = new BufferedWriter(osw);
|
||||
InputStream is = parserWrapperProcess.getInputStream();
|
||||
InputStreamReader isr = new InputStreamReader(is, "UTF-8");
|
||||
fromParserWrapper = new BufferedReader(isr);
|
||||
this.loadMetadata();
|
||||
} catch (IOException e) {
|
||||
throw new CatastrophicError(
|
||||
"Could not start Go TypeScript parser wrapper "
|
||||
+ "(command: " + parserWrapperCommand + ")",
|
||||
e);
|
||||
}
|
||||
}
|
||||
|
||||
/** Get the location of the Go parser wrapper binary. */
|
||||
private File getGoParserWrapper() {
|
||||
String explicitPath = Env.systemEnv().get(GO_PARSER_WRAPPER_PATH_ENV_VAR);
|
||||
File goWrapper;
|
||||
if (explicitPath != null) {
|
||||
goWrapper = new File(explicitPath);
|
||||
} else {
|
||||
goWrapper =
|
||||
new File(EnvironmentVariables.getExtractorRoot(), "tools/typescript-parser-wrapper-go");
|
||||
}
|
||||
if (!goWrapper.isFile()) {
|
||||
throw new ResourceError(
|
||||
"Could not find Go TypeScript parser wrapper: " + goWrapper + " does not exist.\n"
|
||||
+ "Set " + GO_PARSER_WRAPPER_PATH_ENV_VAR + " to the path of the Go wrapper binary.");
|
||||
}
|
||||
return goWrapper;
|
||||
}
|
||||
|
||||
/** Start the Node.js parser wrapper process. */
|
||||
private void setupNodeParserWrapper() {
|
||||
private void setupParserWrapper() {
|
||||
verifyNodeInstallation();
|
||||
|
||||
int mainMemoryMb =
|
||||
@@ -439,7 +344,7 @@ public class TypeScriptParser {
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a {@code request} to the parser wrapper process, and return the response it
|
||||
* Send a {@code request} to the Node.js parser wrapper process, and return the response it
|
||||
* replies with.
|
||||
*/
|
||||
private JsonObject talkToParserWrapper(JsonObject request) {
|
||||
@@ -607,7 +512,7 @@ public class TypeScriptParser {
|
||||
}
|
||||
|
||||
/**
|
||||
* Forcibly closes the parser wrapper process (Node.js or Go).
|
||||
* Forcibly closes the Node.js process.
|
||||
*
|
||||
* <p>A new process will be started the next time a request is made.
|
||||
*/
|
||||
|
||||
@@ -1 +1 @@
|
||||
**/javascript/extractor/tests/*/input//
|
||||
**/*ql*/javascript/extractor/tests/*/input//
|
||||
|
||||
@@ -12,19 +12,27 @@
|
||||
*/
|
||||
|
||||
import python
|
||||
private import LegacyPointsTo
|
||||
private import semmle.python.dataflow.new.internal.DataFlowDispatch
|
||||
|
||||
ClassObject left_base(ClassObject type, ClassObject base) {
|
||||
exists(int i | i > 0 and type.getBaseType(i) = base and result = type.getBaseType(i - 1))
|
||||
/**
|
||||
* Gets the `i`th base class of `cls`, if it can be resolved to a user-defined class.
|
||||
*/
|
||||
Class getBaseType(Class cls, int i) {
|
||||
cls.getBase(i) = classTracker(result).asExpr() and
|
||||
result != cls
|
||||
}
|
||||
|
||||
predicate invalid_mro(ClassObject t, ClassObject left, ClassObject right) {
|
||||
t.isNewStyle() and
|
||||
Class left_base(Class type, Class base) {
|
||||
exists(int i | i > 0 and getBaseType(type, i) = base and result = getBaseType(type, i - 1))
|
||||
}
|
||||
|
||||
predicate invalid_mro(Class t, Class left, Class right) {
|
||||
DuckTyping::isNewStyle(t) and
|
||||
left = left_base(t, right) and
|
||||
left = right.getAnImproperSuperType()
|
||||
left = getADirectSuperclass*(right)
|
||||
}
|
||||
|
||||
from ClassObject t, ClassObject left, ClassObject right
|
||||
from Class t, Class left, Class right
|
||||
where invalid_mro(t, left, right)
|
||||
select t,
|
||||
"Construction of class " + t.getName() +
|
||||
|
||||
@@ -1 +1 @@
|
||||
| inconsistent_mro.py:9:1:9:14 | class Z | Construction of class Z can fail due to invalid method resolution order(MRO) for bases $@ and $@. | inconsistent_mro.py:3:1:3:16 | class X | X | inconsistent_mro.py:6:1:6:11 | class Y | Y |
|
||||
| inconsistent_mro.py:9:1:9:14 | Class Z | Construction of class Z can fail due to invalid method resolution order(MRO) for bases $@ and $@. | inconsistent_mro.py:3:1:3:16 | Class X | X | inconsistent_mro.py:6:1:6:11 | Class Y | Y |
|
||||
|
||||
@@ -1,2 +1 @@
|
||||
| inconsistent_mro.py:9:1:9:14 | class Z | Construction of class Z can fail due to invalid method resolution order(MRO) for bases $@ and $@. | inconsistent_mro.py:3:1:3:16 | class X | X | inconsistent_mro.py:6:1:6:11 | class Y | Y |
|
||||
| inconsistent_mro.py:16:1:16:19 | class N | Construction of class N can fail due to invalid method resolution order(MRO) for bases $@ and $@. | file://:Compiled Code:0:0:0:0 | builtin-class object | object | inconsistent_mro.py:12:1:12:8 | class O | O |
|
||||
| inconsistent_mro.py:9:1:9:14 | Class Z | Construction of class Z can fail due to invalid method resolution order(MRO) for bases $@ and $@. | inconsistent_mro.py:3:1:3:16 | Class X | X | inconsistent_mro.py:6:1:6:11 | Class Y | Y |
|
||||
|
||||
@@ -63,35 +63,6 @@ signature module InputSig<LocationSig Location> {
|
||||
|
||||
DataFlowType getNodeType(Node node);
|
||||
|
||||
/**
|
||||
* Gets a special type to use for parameter node `p` belonging to callables with a
|
||||
* source node where a source call context `FlowFeature` is used, if any.
|
||||
*
|
||||
* This can be used to prevent lambdas from being resolved, when a concrete call
|
||||
* context is needed. Example:
|
||||
*
|
||||
* ```csharp
|
||||
* void Foo(Action<string> a)
|
||||
* {
|
||||
* var x = Source();
|
||||
* a(x); // (1)
|
||||
* a = s => Sink(s); // (2)
|
||||
* a(x); // (3)
|
||||
* }
|
||||
*
|
||||
* void Bar()
|
||||
* {
|
||||
* Foo(s => Sink(s)); // (4)
|
||||
* }
|
||||
* ```
|
||||
*
|
||||
* If a source call context flow feature is used, `a` can be assigned a special
|
||||
* type that is incompatible with the type of _any_ lambda expression, which will
|
||||
* prevent the call edge from (1) to (4). Note that the call edge from (3) to (2)
|
||||
* will still be valid.
|
||||
*/
|
||||
default DataFlowType getSourceContextParameterNodeType(Node p) { none() }
|
||||
|
||||
predicate nodeIsHidden(Node node);
|
||||
|
||||
class DataFlowExpr;
|
||||
|
||||
@@ -1103,16 +1103,6 @@ module MakeImpl<LocationSig Location, InputSig<Location> Lang> {
|
||||
private module FwdTypeFlowInput implements TypeFlowInput {
|
||||
predicate enableTypeFlow = Param::enableTypeFlow/0;
|
||||
|
||||
pragma[nomagic]
|
||||
predicate isParameterNodeInSourceCallContext(ParamNode p) {
|
||||
hasSourceCallCtx() and
|
||||
exists(Node source, DataFlowCallable c |
|
||||
Config::isSource(pragma[only_bind_into](source), _) and
|
||||
nodeEnclosingCallable(source, c) and
|
||||
nodeEnclosingCallable(p, c)
|
||||
)
|
||||
}
|
||||
|
||||
predicate relevantCallEdgeIn = PrevStage::relevantCallEdgeIn/2;
|
||||
|
||||
predicate relevantCallEdgeOut = PrevStage::relevantCallEdgeOut/2;
|
||||
@@ -1420,8 +1410,6 @@ module MakeImpl<LocationSig Location, InputSig<Location> Lang> {
|
||||
private module RevTypeFlowInput implements TypeFlowInput {
|
||||
predicate enableTypeFlow = Param::enableTypeFlow/0;
|
||||
|
||||
predicate isParameterNodeInSourceCallContext(ParamNode p) { none() }
|
||||
|
||||
predicate relevantCallEdgeIn(Call call, Callable c) {
|
||||
flowOutOfCallAp(call, c, _, _, _, _, _)
|
||||
}
|
||||
|
||||
@@ -1893,9 +1893,6 @@ module MakeImplCommon<LocationSig Location, InputSig<Location> Lang> {
|
||||
signature module TypeFlowInput {
|
||||
predicate enableTypeFlow();
|
||||
|
||||
/** Holds if `p` is a parameter of a callable with a source node that has a call context. */
|
||||
predicate isParameterNodeInSourceCallContext(ParamNode p);
|
||||
|
||||
/** Holds if the edge is possibly needed in the direction `call` to `c`. */
|
||||
predicate relevantCallEdgeIn(Call call, Callable c);
|
||||
|
||||
@@ -1956,9 +1953,6 @@ module MakeImplCommon<LocationSig Location, InputSig<Location> Lang> {
|
||||
/**
|
||||
* Holds if a sequence of calls may propagate the value of `arg` to some
|
||||
* argument-to-parameter call edge that strengthens the static type.
|
||||
*
|
||||
* This predicate is a reverse flow computation, starting at calls that
|
||||
* strengthen the type and then following relevant call edges backwards.
|
||||
*/
|
||||
pragma[nomagic]
|
||||
private predicate trackedArgTypeCand(ArgNode arg) {
|
||||
@@ -1993,9 +1987,6 @@ module MakeImplCommon<LocationSig Location, InputSig<Location> Lang> {
|
||||
* Holds if `p` is part of a value-propagating call path where the
|
||||
* end-points have stronger types than the intermediate parameter and
|
||||
* argument nodes.
|
||||
*
|
||||
* This predicate is a forward flow computation, intersecting with the
|
||||
* reverse flow computation done in `trackedArgTypeCand`.
|
||||
*/
|
||||
private predicate trackedParamType(ParamNode p) {
|
||||
exists(Call call1, Callable c1, ArgNode argOut, Call call2, Callable c2, ArgNode argIn |
|
||||
@@ -2022,8 +2013,6 @@ module MakeImplCommon<LocationSig Location, InputSig<Location> Lang> {
|
||||
typeStrongerThanFilter(at, pt)
|
||||
)
|
||||
or
|
||||
Input::isParameterNodeInSourceCallContext(p)
|
||||
or
|
||||
exists(ArgNode arg |
|
||||
trackedArgType(arg) and
|
||||
relevantCallEdge(_, _, arg, p) and
|
||||
@@ -2115,12 +2104,8 @@ module MakeImplCommon<LocationSig Location, InputSig<Location> Lang> {
|
||||
* context.
|
||||
*/
|
||||
private predicate typeFlowParamType(ParamNode p, Type t, boolean cc) {
|
||||
exists(Callable c | Input::dataFlowNonCallEntry(c, cc) |
|
||||
cc = true and
|
||||
nodeEnclosingCallable(p, c) and
|
||||
t = getSourceContextParameterNodeType(p)
|
||||
or
|
||||
(cc = false or not exists(getSourceContextParameterNodeType(p))) and
|
||||
exists(Callable c |
|
||||
Input::dataFlowNonCallEntry(c, cc) and
|
||||
trackedParamWithType(p, t, c)
|
||||
)
|
||||
or
|
||||
|
||||
Reference in New Issue
Block a user