diff --git a/MODULE.bazel b/MODULE.bazel index 24b1c757e37..b6d7319a13c 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -19,7 +19,7 @@ bazel_dep(name = "rules_cc", version = "0.2.17") bazel_dep(name = "rules_go", version = "0.60.0") bazel_dep(name = "rules_java", version = "9.6.1") bazel_dep(name = "rules_pkg", version = "1.2.0") -bazel_dep(name = "rules_nodejs", version = "6.7.3") +bazel_dep(name = "rules_nodejs", version = "6.7.5") bazel_dep(name = "rules_python", version = "1.9.0") bazel_dep(name = "rules_shell", version = "0.7.1") bazel_dep(name = "bazel_skylib", version = "1.9.0") @@ -322,7 +322,7 @@ go_sdk.download(version = "1.26.5") go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps") go_deps.from_file(go_mod = "//go/extractor:go.mod") -use_repo(go_deps, "com_github_stretchr_testify", "org_golang_x_mod", "org_golang_x_tools") +use_repo(go_deps, "com_github_stretchr_testify", "org_golang_x_mod", "org_golang_x_sys", "org_golang_x_tools") ripunzip_archive = use_repo_rule("//misc/ripunzip:ripunzip.bzl", "ripunzip_archive") diff --git a/cpp/ql/lib/change-notes/2026-07-28-winreg-sources.md b/cpp/ql/lib/change-notes/2026-07-28-winreg-sources.md new file mode 100644 index 00000000000..9a70926b998 --- /dev/null +++ b/cpp/ql/lib/change-notes/2026-07-28-winreg-sources.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* Added flow source models for `RegQueryValue` and related functions from the `winreg.h` Windows header. \ No newline at end of file diff --git a/cpp/ql/lib/ext/Windows.model.yml b/cpp/ql/lib/ext/Windows.model.yml index 6794e9a8664..e9f6dfd7fbf 100644 --- a/cpp/ql/lib/ext/Windows.model.yml +++ b/cpp/ql/lib/ext/Windows.model.yml @@ -34,6 +34,17 @@ extensions: - ["", "", False, "HttpReceiveHttpRequest", "", "", "Argument[*3]", "remote", "manual"] - ["", "", False, "HttpReceiveRequestEntityBody", "", "", "Argument[*3]", "remote", "manual"] - ["", "", False, "HttpReceiveClientCertificate", "", "", "Argument[*3]", "remote", "manual"] + # winreg.h + - ["", "", False, "RegQueryValueA", "", "", "Argument[*2]", "local", "manual"] + - ["", "", False, "RegQueryValueExA", "", "", "Argument[*4]", "local", "manual"] + - ["", "", False, "RegQueryValueW", "", "", "Argument[*2]", "local", "manual"] + - ["", "", False, "RegQueryValueExW", "", "", "Argument[*4]", "local", "manual"] + - ["", "", False, "RegGetValueA", "", "", "Argument[*5]", "local", "manual"] + - ["", "", False, "RegGetValueW", "", "", "Argument[*5]", "local", "manual"] + # TODO: Once we support access paths at sources we should also mark Argument[*1].Field[*ve_valueptr] + - ["", "", False, "RegQueryMultipleValuesA", "", "", "Argument[*3]", "local", "manual"] + # TODO: Once we support access paths at sources we should also mark Argument[*1].Field[*ve_valueptr] + - ["", "", False, "RegQueryMultipleValuesW", "", "", "Argument[*3]", "local", "manual"] - addsTo: pack: codeql/cpp-all extensible: summaryModel diff --git a/cpp/ql/test/library-tests/dataflow/external-models/flow.expected b/cpp/ql/test/library-tests/dataflow/external-models/flow.expected index 64180436450..df453de8e1d 100644 --- a/cpp/ql/test/library-tests/dataflow/external-models/flow.expected +++ b/cpp/ql/test/library-tests/dataflow/external-models/flow.expected @@ -17,104 +17,111 @@ models | 16 | Source: ; ; false; NtReadFile; ; ; Argument[*5]; local; manual | | 17 | Source: ; ; false; ReadFile; ; ; Argument[*1]; local; manual | | 18 | Source: ; ; false; ReadFileEx; ; ; Argument[*1]; local; manual | -| 19 | Source: ; ; false; WinHttpQueryHeaders; ; ; Argument[*3]; remote; manual | -| 20 | Source: ; ; false; WinHttpQueryHeadersEx; ; ; Argument[**8]; remote; manual | -| 21 | Source: ; ; false; WinHttpQueryHeadersEx; ; ; Argument[*5]; remote; manual | -| 22 | Source: ; ; false; WinHttpQueryHeadersEx; ; ; Argument[*6]; remote; manual | -| 23 | Source: ; ; false; WinHttpReadData; ; ; Argument[*1]; remote; manual | -| 24 | Source: ; ; false; WinHttpReadDataEx; ; ; Argument[*1]; remote; manual | -| 25 | Source: ; ; false; ymlSource; ; ; ReturnValue; local; manual | -| 26 | Source: Azure::Core::Http; RawResponse; true; ExtractBodyStream; ; ; ReturnValue[*]; remote; manual | -| 27 | Source: Azure::Core::Http; RawResponse; true; GetBody; ; ; ReturnValue[*]; remote; manual | -| 28 | Source: Azure::Core::Http; RawResponse; true; GetHeaders; ; ; ReturnValue[*]; remote; manual | -| 29 | Source: Azure::Core::Http; Request; true; GetBodyStream; ; ; ReturnValue[*]; remote; manual | -| 30 | Source: Azure::Core::Http; Request; true; GetHeader; ; ; ReturnValue; remote; manual | -| 31 | Source: Azure::Core::Http; Request; true; GetHeaders; ; ; ReturnValue; remote; manual | -| 32 | Source: boost::asio; ; false; read_until; ; ; Argument[*1]; remote; manual | -| 33 | Summary: ; ; false; CommandLineToArgvA; ; ; Argument[*0]; ReturnValue[**]; taint; manual | -| 34 | Summary: ; ; false; CreateRemoteThread; ; ; Argument[@4]; Argument[3].Parameter[@0]; value; manual | -| 35 | Summary: ; ; false; CreateRemoteThreadEx; ; ; Argument[@4]; Argument[3].Parameter[@0]; value; manual | -| 36 | Summary: ; ; false; CreateThread; ; ; Argument[@3]; Argument[2].Parameter[@0]; value; manual | -| 37 | Summary: ; ; false; ReadFileEx; ; ; Argument[*3].Field[@hEvent]; Argument[4].Parameter[*2].Field[@hEvent]; value; manual | -| 38 | Summary: ; ; false; RtlCopyDeviceMemory; ; ; Argument[*@1]; Argument[*@0]; value; manual | -| 39 | Summary: ; ; false; RtlCopyMemory; ; ; Argument[*@1]; Argument[*@0]; value; manual | -| 40 | Summary: ; ; false; RtlCopyMemoryNonTemporal; ; ; Argument[*@1]; Argument[*@0]; value; manual | -| 41 | Summary: ; ; false; RtlCopyUnicodeString; ; ; Argument[*1].Field[*Buffer]; Argument[*0].Field[*Buffer]; value; manual | -| 42 | Summary: ; ; false; RtlCopyVolatileMemory; ; ; Argument[*@1]; Argument[*@0]; value; manual | -| 43 | Summary: ; ; false; RtlInitUnicodeString; ; ; Argument[*1]; Argument[*0].Field[*Buffer]; value; manual | -| 44 | Summary: ; ; false; RtlMoveMemory; ; ; Argument[*@1]; Argument[*@0]; value; manual | -| 45 | Summary: ; ; false; RtlMoveVolatileMemory; ; ; Argument[*@1]; Argument[*@0]; value; manual | -| 46 | Summary: ; ; false; WinHttpCrackUrl; ; ; Argument[*0]; Argument[*3]; taint; manual | -| 47 | Summary: ; ; false; callWithArgument; ; ; Argument[1]; Argument[0].Parameter[0]; value; manual | -| 48 | Summary: ; ; false; callWithNonTypeTemplate; (const T &); ; Argument[*0]; ReturnValue; value; manual | -| 49 | Summary: ; ; false; pthread_create; ; ; Argument[@3]; Argument[2].Parameter[@0]; value; manual | -| 50 | Summary: ; ; false; read_field_from_struct; ; ; Argument[*0].Field[MyNamespace::MyStructInNamespace::myField]; ReturnValue; value; manual | -| 51 | Summary: ; ; false; read_field_from_struct_2; ; ; Argument[*0].Field[MyGlobalStruct::myField]; ReturnValue; value; manual | -| 52 | Summary: ; ; false; ymlStepGenerated; ; ; Argument[0]; ReturnValue; taint; df-generated | -| 53 | Summary: ; ; false; ymlStepManual; ; ; Argument[0]; ReturnValue; taint; manual | -| 54 | Summary: ; ; false; ymlStepManual_with_body; ; ; Argument[0]; ReturnValue; taint; manual | -| 55 | Summary: ; MyString; true; operator[]; ; ; Argument[-1]; ReturnValue[*]; taint; manual | -| 56 | Summary: ; MyString; true; operator[]; ; ; ReturnValue[*]; Argument[-1]; taint; manual | -| 57 | Summary: ; ReverseFlow; true; get_ptr; ; ; ReturnValue[*]; Argument[-1].Field[ReverseFlow::value]; value; manual | -| 58 | Summary: ; TemplateClass1; true; templateFunction2; (U,V); ; Argument[1]; ReturnValue; value; manual | -| 59 | Summary: ; TemplateClass1; false; templateFunction; (T,U); ; Argument[0]; ReturnValue; value; manual | -| 60 | Summary: ; TemplateClass2; true; function; (U,T); ; Argument[1]; ReturnValue; value; manual | -| 61 | Summary: Azure::Core::IO; BodyStream; true; Read; ; ; Argument[-1]; Argument[*0]; taint; manual | -| 62 | Summary: Azure::Core::IO; BodyStream; true; ReadToCount; ; ; Argument[-1]; Argument[*0]; taint; manual | -| 63 | Summary: Azure::Core::IO; BodyStream; true; ReadToEnd; ; ; Argument[-1]; ReturnValue.Element; taint; manual | -| 64 | Summary: Azure; Nullable; true; Value; ; ; Argument[-1]; ReturnValue[*]; taint; manual | -| 65 | Summary: boost::asio; ; false; buffer; ; ; Argument[*0]; ReturnValue; taint; manual | +| 19 | Source: ; ; false; RegGetValueA; ; ; Argument[*5]; local; manual | +| 20 | Source: ; ; false; RegQueryMultipleValuesA; ; ; Argument[*3]; local; manual | +| 21 | Source: ; ; false; RegQueryMultipleValuesW; ; ; Argument[*3]; local; manual | +| 22 | Source: ; ; false; RegQueryValueA; ; ; Argument[*2]; local; manual | +| 23 | Source: ; ; false; RegQueryValueExA; ; ; Argument[*4]; local; manual | +| 24 | Source: ; ; false; RegQueryValueExW; ; ; Argument[*4]; local; manual | +| 25 | Source: ; ; false; RegQueryValueW; ; ; Argument[*2]; local; manual | +| 26 | Source: ; ; false; WinHttpQueryHeaders; ; ; Argument[*3]; remote; manual | +| 27 | Source: ; ; false; WinHttpQueryHeadersEx; ; ; Argument[**8]; remote; manual | +| 28 | Source: ; ; false; WinHttpQueryHeadersEx; ; ; Argument[*5]; remote; manual | +| 29 | Source: ; ; false; WinHttpQueryHeadersEx; ; ; Argument[*6]; remote; manual | +| 30 | Source: ; ; false; WinHttpReadData; ; ; Argument[*1]; remote; manual | +| 31 | Source: ; ; false; WinHttpReadDataEx; ; ; Argument[*1]; remote; manual | +| 32 | Source: ; ; false; ymlSource; ; ; ReturnValue; local; manual | +| 33 | Source: Azure::Core::Http; RawResponse; true; ExtractBodyStream; ; ; ReturnValue[*]; remote; manual | +| 34 | Source: Azure::Core::Http; RawResponse; true; GetBody; ; ; ReturnValue[*]; remote; manual | +| 35 | Source: Azure::Core::Http; RawResponse; true; GetHeaders; ; ; ReturnValue[*]; remote; manual | +| 36 | Source: Azure::Core::Http; Request; true; GetBodyStream; ; ; ReturnValue[*]; remote; manual | +| 37 | Source: Azure::Core::Http; Request; true; GetHeader; ; ; ReturnValue; remote; manual | +| 38 | Source: Azure::Core::Http; Request; true; GetHeaders; ; ; ReturnValue; remote; manual | +| 39 | Source: boost::asio; ; false; read_until; ; ; Argument[*1]; remote; manual | +| 40 | Summary: ; ; false; CommandLineToArgvA; ; ; Argument[*0]; ReturnValue[**]; taint; manual | +| 41 | Summary: ; ; false; CreateRemoteThread; ; ; Argument[@4]; Argument[3].Parameter[@0]; value; manual | +| 42 | Summary: ; ; false; CreateRemoteThreadEx; ; ; Argument[@4]; Argument[3].Parameter[@0]; value; manual | +| 43 | Summary: ; ; false; CreateThread; ; ; Argument[@3]; Argument[2].Parameter[@0]; value; manual | +| 44 | Summary: ; ; false; ReadFileEx; ; ; Argument[*3].Field[@hEvent]; Argument[4].Parameter[*2].Field[@hEvent]; value; manual | +| 45 | Summary: ; ; false; RtlCopyDeviceMemory; ; ; Argument[*@1]; Argument[*@0]; value; manual | +| 46 | Summary: ; ; false; RtlCopyMemory; ; ; Argument[*@1]; Argument[*@0]; value; manual | +| 47 | Summary: ; ; false; RtlCopyMemoryNonTemporal; ; ; Argument[*@1]; Argument[*@0]; value; manual | +| 48 | Summary: ; ; false; RtlCopyUnicodeString; ; ; Argument[*1].Field[*Buffer]; Argument[*0].Field[*Buffer]; value; manual | +| 49 | Summary: ; ; false; RtlCopyVolatileMemory; ; ; Argument[*@1]; Argument[*@0]; value; manual | +| 50 | Summary: ; ; false; RtlInitUnicodeString; ; ; Argument[*1]; Argument[*0].Field[*Buffer]; value; manual | +| 51 | Summary: ; ; false; RtlMoveMemory; ; ; Argument[*@1]; Argument[*@0]; value; manual | +| 52 | Summary: ; ; false; RtlMoveVolatileMemory; ; ; Argument[*@1]; Argument[*@0]; value; manual | +| 53 | Summary: ; ; false; WinHttpCrackUrl; ; ; Argument[*0]; Argument[*3]; taint; manual | +| 54 | Summary: ; ; false; callWithArgument; ; ; Argument[1]; Argument[0].Parameter[0]; value; manual | +| 55 | Summary: ; ; false; callWithNonTypeTemplate; (const T &); ; Argument[*0]; ReturnValue; value; manual | +| 56 | Summary: ; ; false; pthread_create; ; ; Argument[@3]; Argument[2].Parameter[@0]; value; manual | +| 57 | Summary: ; ; false; read_field_from_struct; ; ; Argument[*0].Field[MyNamespace::MyStructInNamespace::myField]; ReturnValue; value; manual | +| 58 | Summary: ; ; false; read_field_from_struct_2; ; ; Argument[*0].Field[MyGlobalStruct::myField]; ReturnValue; value; manual | +| 59 | Summary: ; ; false; ymlStepGenerated; ; ; Argument[0]; ReturnValue; taint; df-generated | +| 60 | Summary: ; ; false; ymlStepManual; ; ; Argument[0]; ReturnValue; taint; manual | +| 61 | Summary: ; ; false; ymlStepManual_with_body; ; ; Argument[0]; ReturnValue; taint; manual | +| 62 | Summary: ; MyString; true; operator[]; ; ; Argument[-1]; ReturnValue[*]; taint; manual | +| 63 | Summary: ; MyString; true; operator[]; ; ; ReturnValue[*]; Argument[-1]; taint; manual | +| 64 | Summary: ; ReverseFlow; true; get_ptr; ; ; ReturnValue[*]; Argument[-1].Field[ReverseFlow::value]; value; manual | +| 65 | Summary: ; TemplateClass1; true; templateFunction2; (U,V); ; Argument[1]; ReturnValue; value; manual | +| 66 | Summary: ; TemplateClass1; false; templateFunction; (T,U); ; Argument[0]; ReturnValue; value; manual | +| 67 | Summary: ; TemplateClass2; true; function; (U,T); ; Argument[1]; ReturnValue; value; manual | +| 68 | Summary: Azure::Core::IO; BodyStream; true; Read; ; ; Argument[-1]; Argument[*0]; taint; manual | +| 69 | Summary: Azure::Core::IO; BodyStream; true; ReadToCount; ; ; Argument[-1]; Argument[*0]; taint; manual | +| 70 | Summary: Azure::Core::IO; BodyStream; true; ReadToEnd; ; ; Argument[-1]; ReturnValue.Element; taint; manual | +| 71 | Summary: Azure; Nullable; true; Value; ; ; Argument[-1]; ReturnValue[*]; taint; manual | +| 72 | Summary: boost::asio; ; false; buffer; ; ; Argument[*0]; ReturnValue; taint; manual | edges -| asio_streams.cpp:87:34:87:44 | read_until output argument | asio_streams.cpp:91:7:91:17 | recv_buffer | provenance | Src:MaD:32 | -| asio_streams.cpp:87:34:87:44 | read_until output argument | asio_streams.cpp:93:29:93:39 | *recv_buffer | provenance | Src:MaD:32 Sink:MaD:2 | +| asio_streams.cpp:87:34:87:44 | read_until output argument | asio_streams.cpp:91:7:91:17 | recv_buffer | provenance | Src:MaD:39 | +| asio_streams.cpp:87:34:87:44 | read_until output argument | asio_streams.cpp:93:29:93:39 | *recv_buffer | provenance | Src:MaD:39 Sink:MaD:2 | | asio_streams.cpp:97:37:97:44 | call to source | asio_streams.cpp:98:7:98:14 | send_str | provenance | TaintFunction | | asio_streams.cpp:97:37:97:44 | call to source | asio_streams.cpp:100:64:100:71 | *send_str | provenance | TaintFunction | | asio_streams.cpp:100:44:100:62 | call to buffer | asio_streams.cpp:100:44:100:62 | call to buffer | provenance | | | asio_streams.cpp:100:44:100:62 | call to buffer | asio_streams.cpp:101:7:101:17 | send_buffer | provenance | | | asio_streams.cpp:100:44:100:62 | call to buffer | asio_streams.cpp:103:29:103:39 | *send_buffer | provenance | Sink:MaD:2 | -| asio_streams.cpp:100:64:100:71 | *send_str | asio_streams.cpp:100:44:100:62 | call to buffer | provenance | MaD:65 | -| azure.cpp:253:48:253:60 | *call to GetBodyStream | azure.cpp:253:48:253:60 | *call to GetBodyStream | provenance | Src:MaD:29 | +| asio_streams.cpp:100:64:100:71 | *send_str | asio_streams.cpp:100:44:100:62 | call to buffer | provenance | MaD:72 | +| azure.cpp:253:48:253:60 | *call to GetBodyStream | azure.cpp:253:48:253:60 | *call to GetBodyStream | provenance | Src:MaD:36 | | azure.cpp:253:48:253:60 | *call to GetBodyStream | azure.cpp:257:5:257:8 | *resp | provenance | | | azure.cpp:253:48:253:60 | *call to GetBodyStream | azure.cpp:262:5:262:8 | *resp | provenance | | | azure.cpp:253:48:253:60 | *call to GetBodyStream | azure.cpp:266:38:266:41 | *resp | provenance | | -| azure.cpp:257:5:257:8 | *resp | azure.cpp:257:16:257:21 | Read output argument | provenance | MaD:61 | +| azure.cpp:257:5:257:8 | *resp | azure.cpp:257:16:257:21 | Read output argument | provenance | MaD:68 | | azure.cpp:257:16:257:21 | Read output argument | azure.cpp:258:10:258:16 | * ... | provenance | | -| azure.cpp:262:5:262:8 | *resp | azure.cpp:262:23:262:28 | ReadToCount output argument | provenance | MaD:62 | +| azure.cpp:262:5:262:8 | *resp | azure.cpp:262:23:262:28 | ReadToCount output argument | provenance | MaD:69 | | azure.cpp:262:23:262:28 | ReadToCount output argument | azure.cpp:263:10:263:16 | * ... | provenance | | -| azure.cpp:266:38:266:41 | *resp | azure.cpp:266:44:266:52 | call to ReadToEnd [element] | provenance | MaD:63 | +| azure.cpp:266:38:266:41 | *resp | azure.cpp:266:44:266:52 | call to ReadToEnd [element] | provenance | MaD:70 | | azure.cpp:266:44:266:52 | call to ReadToEnd [element] | azure.cpp:266:44:266:52 | call to ReadToEnd [element] | provenance | | | azure.cpp:266:44:266:52 | call to ReadToEnd [element] | azure.cpp:267:10:267:12 | vec [element] | provenance | | | azure.cpp:267:10:267:12 | vec [element] | azure.cpp:267:10:267:12 | vec | provenance | | -| azure.cpp:273:62:273:64 | call to GetHeaders | azure.cpp:273:62:273:64 | call to GetHeaders | provenance | Src:MaD:28 | +| azure.cpp:273:62:273:64 | call to GetHeaders | azure.cpp:273:62:273:64 | call to GetHeaders | provenance | Src:MaD:35 | | azure.cpp:273:62:273:64 | call to GetHeaders | azure.cpp:274:14:274:29 | call to operator[] | provenance | TaintFunction | | azure.cpp:273:62:273:64 | call to GetHeaders | azure.cpp:274:14:274:29 | call to operator[] | provenance | TaintFunction | | azure.cpp:273:62:273:64 | call to GetHeaders | azure.cpp:274:14:274:29 | call to operator[] | provenance | TaintFunction | | azure.cpp:274:14:274:29 | call to operator[] | azure.cpp:274:10:274:29 | call to operator[] | provenance | | | azure.cpp:274:14:274:29 | call to operator[] | azure.cpp:274:14:274:29 | call to operator[] | provenance | | -| azure.cpp:277:45:277:47 | call to GetBody | azure.cpp:277:45:277:47 | call to GetBody | provenance | Src:MaD:27 | +| azure.cpp:277:45:277:47 | call to GetBody | azure.cpp:277:45:277:47 | call to GetBody | provenance | Src:MaD:34 | | azure.cpp:277:45:277:47 | call to GetBody | azure.cpp:278:10:278:13 | body | provenance | | | azure.cpp:277:45:277:47 | call to GetBody | azure.cpp:278:10:278:13 | body | provenance | | | azure.cpp:278:10:278:13 | body | azure.cpp:278:10:278:13 | body | provenance | | -| azure.cpp:281:68:281:84 | *call to ExtractBodyStream | azure.cpp:281:68:281:84 | *call to ExtractBodyStream | provenance | Src:MaD:26 | +| azure.cpp:281:68:281:84 | *call to ExtractBodyStream | azure.cpp:281:68:281:84 | *call to ExtractBodyStream | provenance | Src:MaD:33 | | azure.cpp:281:68:281:84 | *call to ExtractBodyStream | azure.cpp:282:21:282:23 | *call to get | provenance | | -| azure.cpp:282:21:282:23 | *call to get | azure.cpp:282:28:282:36 | call to ReadToEnd [element] | provenance | MaD:63 | +| azure.cpp:282:21:282:23 | *call to get | azure.cpp:282:28:282:36 | call to ReadToEnd [element] | provenance | MaD:70 | | azure.cpp:282:28:282:36 | call to ReadToEnd [element] | azure.cpp:282:10:282:38 | call to ReadToEnd | provenance | | | azure.cpp:282:28:282:36 | call to ReadToEnd [element] | azure.cpp:282:28:282:36 | call to ReadToEnd [element] | provenance | | -| azure.cpp:289:24:289:56 | call to GetHeader | azure.cpp:289:63:289:65 | call to Value | provenance | MaD:64 | +| azure.cpp:289:24:289:56 | call to GetHeader | azure.cpp:289:63:289:65 | call to Value | provenance | MaD:71 | | azure.cpp:289:32:289:40 | call to GetHeader | azure.cpp:289:24:289:56 | call to GetHeader | provenance | | -| azure.cpp:289:32:289:40 | call to GetHeader | azure.cpp:289:32:289:40 | call to GetHeader | provenance | Src:MaD:30 | +| azure.cpp:289:32:289:40 | call to GetHeader | azure.cpp:289:32:289:40 | call to GetHeader | provenance | Src:MaD:37 | | azure.cpp:289:63:289:65 | call to Value | azure.cpp:289:63:289:65 | call to Value | provenance | | | azure.cpp:289:63:289:65 | call to Value | azure.cpp:290:10:290:20 | headerValue | provenance | | | azure.cpp:289:63:289:65 | call to Value | azure.cpp:290:10:290:20 | headerValue | provenance | | | azure.cpp:290:10:290:20 | headerValue | azure.cpp:290:10:290:20 | headerValue | provenance | | -| azure.cpp:293:58:293:67 | call to GetHeaders | azure.cpp:293:58:293:67 | call to GetHeaders | provenance | Src:MaD:31 | +| azure.cpp:293:58:293:67 | call to GetHeaders | azure.cpp:293:58:293:67 | call to GetHeaders | provenance | Src:MaD:38 | | azure.cpp:293:58:293:67 | call to GetHeaders | azure.cpp:294:38:294:53 | call to operator[] | provenance | TaintFunction | | azure.cpp:294:38:294:53 | call to operator[] | azure.cpp:295:10:295:20 | contentType | provenance | | | azure.cpp:294:38:294:53 | call to operator[] | azure.cpp:295:10:295:20 | contentType | provenance | | | azure.cpp:295:10:295:20 | contentType | azure.cpp:295:10:295:20 | contentType | provenance | | | test.cpp:7:47:7:52 | value2 | test.cpp:7:64:7:69 | value2 | provenance | | | test.cpp:7:64:7:69 | value2 | test.cpp:7:5:7:30 | *ymlStepGenerated_with_body | provenance | | -| test.cpp:10:10:10:18 | call to ymlSource | test.cpp:10:10:10:18 | call to ymlSource | provenance | Src:MaD:25 | +| test.cpp:10:10:10:18 | call to ymlSource | test.cpp:10:10:10:18 | call to ymlSource | provenance | Src:MaD:32 | | test.cpp:10:10:10:18 | call to ymlSource | test.cpp:14:10:14:10 | x | provenance | Sink:MaD:1 | | test.cpp:10:10:10:18 | call to ymlSource | test.cpp:17:24:17:24 | x | provenance | | | test.cpp:10:10:10:18 | call to ymlSource | test.cpp:21:27:21:27 | x | provenance | | @@ -122,13 +129,13 @@ edges | test.cpp:10:10:10:18 | call to ymlSource | test.cpp:32:41:32:41 | x | provenance | | | test.cpp:17:10:17:22 | call to ymlStepManual | test.cpp:17:10:17:22 | call to ymlStepManual | provenance | | | test.cpp:17:10:17:22 | call to ymlStepManual | test.cpp:18:10:18:10 | y | provenance | Sink:MaD:1 | -| test.cpp:17:24:17:24 | x | test.cpp:17:10:17:22 | call to ymlStepManual | provenance | MaD:53 | +| test.cpp:17:24:17:24 | x | test.cpp:17:10:17:22 | call to ymlStepManual | provenance | MaD:60 | | test.cpp:21:10:21:25 | call to ymlStepGenerated | test.cpp:21:10:21:25 | call to ymlStepGenerated | provenance | | | test.cpp:21:10:21:25 | call to ymlStepGenerated | test.cpp:22:10:22:10 | z | provenance | Sink:MaD:1 | -| test.cpp:21:27:21:27 | x | test.cpp:21:10:21:25 | call to ymlStepGenerated | provenance | MaD:52 | +| test.cpp:21:27:21:27 | x | test.cpp:21:10:21:25 | call to ymlStepGenerated | provenance | MaD:59 | | test.cpp:25:11:25:33 | call to ymlStepManual_with_body | test.cpp:25:11:25:33 | call to ymlStepManual_with_body | provenance | | | test.cpp:25:11:25:33 | call to ymlStepManual_with_body | test.cpp:26:10:26:11 | y2 | provenance | Sink:MaD:1 | -| test.cpp:25:35:25:35 | x | test.cpp:25:11:25:33 | call to ymlStepManual_with_body | provenance | MaD:54 | +| test.cpp:25:35:25:35 | x | test.cpp:25:11:25:33 | call to ymlStepManual_with_body | provenance | MaD:61 | | test.cpp:32:11:32:36 | call to ymlStepGenerated_with_body | test.cpp:32:11:32:36 | call to ymlStepGenerated_with_body | provenance | | | test.cpp:32:11:32:36 | call to ymlStepGenerated_with_body | test.cpp:33:10:33:11 | z2 | provenance | Sink:MaD:1 | | test.cpp:32:41:32:41 | x | test.cpp:7:47:7:52 | value2 | provenance | | @@ -138,73 +145,73 @@ edges | test.cpp:48:13:48:13 | *s [x] | test.cpp:48:16:48:16 | x | provenance | Sink:MaD:1 | | test.cpp:56:2:56:2 | *s [post update] [x] | test.cpp:59:55:59:64 | *& ... [x] | provenance | | | test.cpp:56:2:56:18 | ... = ... | test.cpp:56:2:56:2 | *s [post update] [x] | provenance | | -| test.cpp:56:8:56:16 | call to ymlSource | test.cpp:56:2:56:18 | ... = ... | provenance | Src:MaD:25 | -| test.cpp:59:55:59:64 | *& ... [x] | test.cpp:46:30:46:32 | *arg [x] | provenance | MaD:49 | +| test.cpp:56:8:56:16 | call to ymlSource | test.cpp:56:2:56:18 | ... = ... | provenance | Src:MaD:32 | +| test.cpp:59:55:59:64 | *& ... [x] | test.cpp:46:30:46:32 | *arg [x] | provenance | MaD:56 | | test.cpp:68:22:68:22 | y | test.cpp:69:11:69:11 | y | provenance | Sink:MaD:1 | | test.cpp:74:22:74:22 | y | test.cpp:75:11:75:11 | y | provenance | Sink:MaD:1 | | test.cpp:82:22:82:22 | y | test.cpp:83:11:83:11 | y | provenance | Sink:MaD:1 | | test.cpp:88:22:88:22 | y | test.cpp:89:11:89:11 | y | provenance | Sink:MaD:1 | -| test.cpp:94:10:94:18 | call to ymlSource | test.cpp:94:10:94:18 | call to ymlSource | provenance | Src:MaD:25 | +| test.cpp:94:10:94:18 | call to ymlSource | test.cpp:94:10:94:18 | call to ymlSource | provenance | Src:MaD:32 | | test.cpp:94:10:94:18 | call to ymlSource | test.cpp:97:26:97:26 | x | provenance | | | test.cpp:94:10:94:18 | call to ymlSource | test.cpp:101:26:101:26 | x | provenance | | | test.cpp:94:10:94:18 | call to ymlSource | test.cpp:103:63:103:63 | x | provenance | | | test.cpp:94:10:94:18 | call to ymlSource | test.cpp:104:62:104:62 | x | provenance | | -| test.cpp:97:26:97:26 | x | test.cpp:68:22:68:22 | y | provenance | MaD:47 | -| test.cpp:101:26:101:26 | x | test.cpp:74:22:74:22 | y | provenance | MaD:47 | -| test.cpp:103:63:103:63 | x | test.cpp:82:22:82:22 | y | provenance | MaD:47 | -| test.cpp:104:62:104:62 | x | test.cpp:88:22:88:22 | y | provenance | MaD:47 | -| test.cpp:114:10:114:18 | call to ymlSource | test.cpp:114:10:114:18 | call to ymlSource | provenance | Src:MaD:25 | +| test.cpp:97:26:97:26 | x | test.cpp:68:22:68:22 | y | provenance | MaD:54 | +| test.cpp:101:26:101:26 | x | test.cpp:74:22:74:22 | y | provenance | MaD:54 | +| test.cpp:103:63:103:63 | x | test.cpp:82:22:82:22 | y | provenance | MaD:54 | +| test.cpp:104:62:104:62 | x | test.cpp:88:22:88:22 | y | provenance | MaD:54 | +| test.cpp:114:10:114:18 | call to ymlSource | test.cpp:114:10:114:18 | call to ymlSource | provenance | Src:MaD:32 | | test.cpp:114:10:114:18 | call to ymlSource | test.cpp:118:44:118:44 | *x | provenance | | | test.cpp:118:11:118:42 | call to callWithNonTypeTemplate | test.cpp:118:11:118:42 | call to callWithNonTypeTemplate | provenance | | | test.cpp:118:11:118:42 | call to callWithNonTypeTemplate | test.cpp:119:10:119:11 | y2 | provenance | Sink:MaD:1 | -| test.cpp:118:44:118:44 | *x | test.cpp:118:11:118:42 | call to callWithNonTypeTemplate | provenance | MaD:48 | -| test.cpp:133:10:133:18 | call to ymlSource | test.cpp:133:10:133:18 | call to ymlSource | provenance | Src:MaD:25 | +| test.cpp:118:44:118:44 | *x | test.cpp:118:11:118:42 | call to callWithNonTypeTemplate | provenance | MaD:55 | +| test.cpp:133:10:133:18 | call to ymlSource | test.cpp:133:10:133:18 | call to ymlSource | provenance | Src:MaD:32 | | test.cpp:133:10:133:18 | call to ymlSource | test.cpp:134:45:134:45 | x | provenance | | | test.cpp:134:13:134:43 | call to templateFunction | test.cpp:134:13:134:43 | call to templateFunction | provenance | | | test.cpp:134:13:134:43 | call to templateFunction | test.cpp:135:10:135:10 | y | provenance | Sink:MaD:1 | -| test.cpp:134:45:134:45 | x | test.cpp:134:13:134:43 | call to templateFunction | provenance | MaD:59 | -| test.cpp:146:10:146:18 | call to ymlSource | test.cpp:146:10:146:18 | call to ymlSource | provenance | Src:MaD:25 | +| test.cpp:134:45:134:45 | x | test.cpp:134:13:134:43 | call to templateFunction | provenance | MaD:66 | +| test.cpp:146:10:146:18 | call to ymlSource | test.cpp:146:10:146:18 | call to ymlSource | provenance | Src:MaD:32 | | test.cpp:146:10:146:18 | call to ymlSource | test.cpp:148:26:148:26 | x | provenance | | | test.cpp:148:10:148:27 | call to function | test.cpp:148:10:148:27 | call to function | provenance | | | test.cpp:148:10:148:27 | call to function | test.cpp:149:10:149:10 | z | provenance | Sink:MaD:1 | -| test.cpp:148:26:148:26 | x | test.cpp:148:10:148:27 | call to function | provenance | MaD:60 | -| test.cpp:155:10:155:18 | call to ymlSource | test.cpp:155:10:155:18 | call to ymlSource | provenance | Src:MaD:25 | +| test.cpp:148:26:148:26 | x | test.cpp:148:10:148:27 | call to function | provenance | MaD:67 | +| test.cpp:155:10:155:18 | call to ymlSource | test.cpp:155:10:155:18 | call to ymlSource | provenance | Src:MaD:32 | | test.cpp:155:10:155:18 | call to ymlSource | test.cpp:157:26:157:26 | x | provenance | | | test.cpp:157:13:157:20 | call to function | test.cpp:157:13:157:20 | call to function | provenance | | | test.cpp:157:13:157:20 | call to function | test.cpp:158:10:158:10 | z | provenance | Sink:MaD:1 | -| test.cpp:157:26:157:26 | x | test.cpp:157:13:157:20 | call to function | provenance | MaD:60 | +| test.cpp:157:26:157:26 | x | test.cpp:157:13:157:20 | call to function | provenance | MaD:67 | | test.cpp:164:34:164:34 | x | test.cpp:165:69:165:69 | x | provenance | | | test.cpp:165:12:165:64 | call to templateFunction2 | test.cpp:164:7:164:7 | *templateFunction3 | provenance | | | test.cpp:165:12:165:64 | call to templateFunction2 | test.cpp:165:12:165:64 | call to templateFunction2 | provenance | | -| test.cpp:165:69:165:69 | x | test.cpp:165:12:165:64 | call to templateFunction2 | provenance | MaD:58 | -| test.cpp:170:10:170:18 | call to ymlSource | test.cpp:170:10:170:18 | call to ymlSource | provenance | Src:MaD:25 | +| test.cpp:165:69:165:69 | x | test.cpp:165:12:165:64 | call to templateFunction2 | provenance | MaD:65 | +| test.cpp:170:10:170:18 | call to ymlSource | test.cpp:170:10:170:18 | call to ymlSource | provenance | Src:MaD:32 | | test.cpp:170:10:170:18 | call to ymlSource | test.cpp:172:51:172:51 | x | provenance | | | test.cpp:172:13:172:44 | call to templateFunction3 | test.cpp:172:13:172:44 | call to templateFunction3 | provenance | | | test.cpp:172:13:172:44 | call to templateFunction3 | test.cpp:173:10:173:10 | y | provenance | Sink:MaD:1 | | test.cpp:172:51:172:51 | x | test.cpp:164:34:164:34 | x | provenance | | -| test.cpp:172:51:172:51 | x | test.cpp:172:13:172:44 | call to templateFunction3 | provenance | MaD:58 | +| test.cpp:172:51:172:51 | x | test.cpp:172:13:172:44 | call to templateFunction3 | provenance | MaD:65 | | test.cpp:186:2:186:2 | *s [post update] [myField] | test.cpp:187:33:187:34 | *& ... [myField] | provenance | | | test.cpp:186:2:186:24 | ... = ... | test.cpp:186:2:186:2 | *s [post update] [myField] | provenance | | -| test.cpp:186:14:186:22 | call to ymlSource | test.cpp:186:2:186:24 | ... = ... | provenance | Src:MaD:25 | +| test.cpp:186:14:186:22 | call to ymlSource | test.cpp:186:2:186:24 | ... = ... | provenance | Src:MaD:32 | | test.cpp:187:10:187:31 | call to read_field_from_struct | test.cpp:187:10:187:31 | call to read_field_from_struct | provenance | | | test.cpp:187:10:187:31 | call to read_field_from_struct | test.cpp:188:10:188:10 | x | provenance | Sink:MaD:1 | -| test.cpp:187:33:187:34 | *& ... [myField] | test.cpp:187:10:187:31 | call to read_field_from_struct | provenance | MaD:50 | +| test.cpp:187:33:187:34 | *& ... [myField] | test.cpp:187:10:187:31 | call to read_field_from_struct | provenance | MaD:57 | | test.cpp:199:2:199:2 | *s [post update] [myField] | test.cpp:200:35:200:36 | *& ... [myField] | provenance | | | test.cpp:199:2:199:24 | ... = ... | test.cpp:199:2:199:2 | *s [post update] [myField] | provenance | | -| test.cpp:199:14:199:22 | call to ymlSource | test.cpp:199:2:199:24 | ... = ... | provenance | Src:MaD:25 | +| test.cpp:199:14:199:22 | call to ymlSource | test.cpp:199:2:199:24 | ... = ... | provenance | Src:MaD:32 | | test.cpp:200:10:200:33 | call to read_field_from_struct_2 | test.cpp:200:10:200:33 | call to read_field_from_struct_2 | provenance | | | test.cpp:200:10:200:33 | call to read_field_from_struct_2 | test.cpp:201:10:201:10 | x | provenance | Sink:MaD:1 | -| test.cpp:200:35:200:36 | *& ... [myField] | test.cpp:200:10:200:33 | call to read_field_from_struct_2 | provenance | MaD:51 | +| test.cpp:200:35:200:36 | *& ... [myField] | test.cpp:200:10:200:33 | call to read_field_from_struct_2 | provenance | MaD:58 | | test.cpp:216:3:216:4 | get_ptr output argument [value] | test.cpp:217:11:217:12 | *rf [value] | provenance | | -| test.cpp:216:3:216:28 | ... = ... | test.cpp:216:3:216:4 | get_ptr output argument [value] | provenance | MaD:57 | -| test.cpp:216:18:216:26 | call to ymlSource | test.cpp:216:3:216:28 | ... = ... | provenance | Src:MaD:25 | +| test.cpp:216:3:216:28 | ... = ... | test.cpp:216:3:216:4 | get_ptr output argument [value] | provenance | MaD:64 | +| test.cpp:216:18:216:26 | call to ymlSource | test.cpp:216:3:216:28 | ... = ... | provenance | Src:MaD:32 | | test.cpp:217:11:217:12 | *rf [value] | test.cpp:217:14:217:18 | value | provenance | | | test.cpp:217:14:217:18 | value | test.cpp:217:14:217:18 | value | provenance | | | test.cpp:217:14:217:18 | value | test.cpp:218:11:218:11 | x | provenance | Sink:MaD:1 | | test.cpp:222:3:222:3 | operator[] output argument | test.cpp:223:12:223:12 | *s | provenance | | -| test.cpp:222:3:222:20 | ... = ... | test.cpp:222:3:222:3 | operator[] output argument | provenance | MaD:56 | -| test.cpp:222:10:222:20 | call to ymlSource | test.cpp:222:3:222:20 | ... = ... | provenance | Src:MaD:25 | -| test.cpp:223:12:223:12 | *s | test.cpp:223:13:223:15 | call to operator[] | provenance | MaD:55 | +| test.cpp:222:3:222:20 | ... = ... | test.cpp:222:3:222:3 | operator[] output argument | provenance | MaD:63 | +| test.cpp:222:10:222:20 | call to ymlSource | test.cpp:222:3:222:20 | ... = ... | provenance | Src:MaD:32 | +| test.cpp:223:12:223:12 | *s | test.cpp:223:13:223:15 | call to operator[] | provenance | MaD:62 | | test.cpp:223:13:223:15 | call to operator[] | test.cpp:223:13:223:15 | call to operator[] | provenance | | | test.cpp:223:13:223:15 | call to operator[] | test.cpp:224:11:224:11 | c | provenance | Sink:MaD:1 | | windows.cpp:22:15:22:29 | *call to GetCommandLineA | windows.cpp:22:15:22:29 | *call to GetCommandLineA | provenance | Src:MaD:3 | @@ -212,7 +219,7 @@ edges | windows.cpp:22:15:22:29 | *call to GetCommandLineA | windows.cpp:27:36:27:38 | *cmd | provenance | | | windows.cpp:27:17:27:34 | **call to CommandLineToArgvA | windows.cpp:27:17:27:34 | **call to CommandLineToArgvA | provenance | | | windows.cpp:27:17:27:34 | **call to CommandLineToArgvA | windows.cpp:30:8:30:15 | * ... | provenance | | -| windows.cpp:27:36:27:38 | *cmd | windows.cpp:27:17:27:34 | **call to CommandLineToArgvA | provenance | MaD:33 | +| windows.cpp:27:36:27:38 | *cmd | windows.cpp:27:17:27:34 | **call to CommandLineToArgvA | provenance | MaD:40 | | windows.cpp:34:17:34:38 | *call to GetEnvironmentStringsA | windows.cpp:34:17:34:38 | *call to GetEnvironmentStringsA | provenance | Src:MaD:4 | | windows.cpp:34:17:34:38 | *call to GetEnvironmentStringsA | windows.cpp:36:10:36:13 | * ... | provenance | | | windows.cpp:39:36:39:38 | GetEnvironmentVariableA output argument | windows.cpp:41:10:41:13 | * ... | provenance | Src:MaD:5 | @@ -232,11 +239,11 @@ edges | windows.cpp:189:21:189:26 | ReadFile output argument | windows.cpp:190:5:190:56 | *... = ... | provenance | Src:MaD:17 | | windows.cpp:190:5:190:14 | *overlapped [post update] [*hEvent] | windows.cpp:192:53:192:63 | *& ... [*hEvent] | provenance | | | windows.cpp:190:5:190:56 | *... = ... | windows.cpp:190:5:190:14 | *overlapped [post update] [*hEvent] | provenance | | -| windows.cpp:192:53:192:63 | *& ... [*hEvent] | windows.cpp:147:16:147:27 | *lpOverlapped [*hEvent] | provenance | MaD:37 | +| windows.cpp:192:53:192:63 | *& ... [*hEvent] | windows.cpp:147:16:147:27 | *lpOverlapped [*hEvent] | provenance | MaD:44 | | windows.cpp:198:21:198:26 | ReadFile output argument | windows.cpp:199:5:199:57 | ... = ... | provenance | Src:MaD:17 | | windows.cpp:199:5:199:14 | *overlapped [post update] [hEvent] | windows.cpp:201:53:201:63 | *& ... [hEvent] | provenance | | | windows.cpp:199:5:199:57 | ... = ... | windows.cpp:199:5:199:14 | *overlapped [post update] [hEvent] | provenance | | -| windows.cpp:201:53:201:63 | *& ... [hEvent] | windows.cpp:157:16:157:27 | *lpOverlapped [hEvent] | provenance | MaD:37 | +| windows.cpp:201:53:201:63 | *& ... [hEvent] | windows.cpp:157:16:157:27 | *lpOverlapped [hEvent] | provenance | MaD:44 | | windows.cpp:209:84:209:89 | NtReadFile output argument | windows.cpp:211:10:211:16 | * ... | provenance | Src:MaD:16 | | windows.cpp:286:23:286:35 | *call to MapViewOfFile | windows.cpp:286:23:286:35 | *call to MapViewOfFile | provenance | Src:MaD:12 | | windows.cpp:286:23:286:35 | *call to MapViewOfFile | windows.cpp:287:20:287:52 | *pMapView | provenance | | @@ -273,9 +280,9 @@ edges | windows.cpp:431:3:431:3 | *s [post update] [x] | windows.cpp:464:7:464:8 | *& ... [x] | provenance | | | windows.cpp:431:3:431:16 | ... = ... | windows.cpp:431:3:431:3 | *s [post update] [x] | provenance | | | windows.cpp:431:9:431:14 | call to source | windows.cpp:431:3:431:16 | ... = ... | provenance | | -| windows.cpp:439:7:439:8 | *& ... [x] | windows.cpp:403:26:403:36 | *lpParameter [x] | provenance | MaD:36 | -| windows.cpp:451:7:451:8 | *& ... [x] | windows.cpp:410:26:410:36 | *lpParameter [x] | provenance | MaD:34 | -| windows.cpp:464:7:464:8 | *& ... [x] | windows.cpp:417:26:417:36 | *lpParameter [x] | provenance | MaD:35 | +| windows.cpp:439:7:439:8 | *& ... [x] | windows.cpp:403:26:403:36 | *lpParameter [x] | provenance | MaD:43 | +| windows.cpp:451:7:451:8 | *& ... [x] | windows.cpp:410:26:410:36 | *lpParameter [x] | provenance | MaD:41 | +| windows.cpp:464:7:464:8 | *& ... [x] | windows.cpp:417:26:417:36 | *lpParameter [x] | provenance | MaD:42 | | windows.cpp:533:11:533:16 | call to source | windows.cpp:533:11:533:16 | call to source | provenance | | | windows.cpp:533:11:533:16 | call to source | windows.cpp:537:40:537:41 | *& ... | provenance | | | windows.cpp:533:11:533:16 | call to source | windows.cpp:542:38:542:39 | *& ... | provenance | | @@ -284,39 +291,39 @@ edges | windows.cpp:533:11:533:16 | call to source | windows.cpp:568:32:568:33 | *& ... | provenance | | | windows.cpp:533:11:533:16 | call to source | windows.cpp:573:40:573:41 | *& ... | provenance | | | windows.cpp:537:27:537:37 | RtlCopyVolatileMemory output argument | windows.cpp:538:10:538:23 | access to array | provenance | | -| windows.cpp:537:40:537:41 | *& ... | windows.cpp:537:27:537:37 | RtlCopyVolatileMemory output argument | provenance | MaD:42 | +| windows.cpp:537:40:537:41 | *& ... | windows.cpp:537:27:537:37 | RtlCopyVolatileMemory output argument | provenance | MaD:49 | | windows.cpp:542:25:542:35 | RtlCopyDeviceMemory output argument | windows.cpp:543:10:543:23 | access to array | provenance | | -| windows.cpp:542:38:542:39 | *& ... | windows.cpp:542:25:542:35 | RtlCopyDeviceMemory output argument | provenance | MaD:38 | +| windows.cpp:542:38:542:39 | *& ... | windows.cpp:542:25:542:35 | RtlCopyDeviceMemory output argument | provenance | MaD:45 | | windows.cpp:547:19:547:29 | RtlCopyMemory output argument | windows.cpp:548:10:548:23 | access to array | provenance | | -| windows.cpp:547:32:547:33 | *& ... | windows.cpp:547:19:547:29 | RtlCopyMemory output argument | provenance | MaD:39 | +| windows.cpp:547:32:547:33 | *& ... | windows.cpp:547:19:547:29 | RtlCopyMemory output argument | provenance | MaD:46 | | windows.cpp:552:30:552:40 | RtlCopyMemoryNonTemporal output argument | windows.cpp:553:10:553:23 | access to array | provenance | | -| windows.cpp:552:43:552:44 | *& ... | windows.cpp:552:30:552:40 | RtlCopyMemoryNonTemporal output argument | provenance | MaD:40 | +| windows.cpp:552:43:552:44 | *& ... | windows.cpp:552:30:552:40 | RtlCopyMemoryNonTemporal output argument | provenance | MaD:47 | | windows.cpp:559:5:559:24 | ... = ... | windows.cpp:561:39:561:44 | *buffer | provenance | | | windows.cpp:559:17:559:24 | call to source | windows.cpp:559:5:559:24 | ... = ... | provenance | | | windows.cpp:561:26:561:36 | RtlInitUnicodeString output argument [*Buffer] | windows.cpp:562:10:562:19 | *src_string [*Buffer] | provenance | | | windows.cpp:561:26:561:36 | RtlInitUnicodeString output argument [*Buffer] | windows.cpp:563:40:563:50 | *& ... [*Buffer] | provenance | | -| windows.cpp:561:39:561:44 | *buffer | windows.cpp:561:26:561:36 | RtlInitUnicodeString output argument [*Buffer] | provenance | MaD:43 | +| windows.cpp:561:39:561:44 | *buffer | windows.cpp:561:26:561:36 | RtlInitUnicodeString output argument [*Buffer] | provenance | MaD:50 | | windows.cpp:562:10:562:19 | *src_string [*Buffer] | windows.cpp:562:10:562:29 | access to array | provenance | | | windows.cpp:562:10:562:19 | *src_string [*Buffer] | windows.cpp:562:21:562:26 | *Buffer | provenance | | | windows.cpp:562:21:562:26 | *Buffer | windows.cpp:562:10:562:29 | access to array | provenance | | | windows.cpp:563:26:563:37 | RtlCopyUnicodeString output argument [*Buffer] | windows.cpp:564:10:564:20 | *dest_string [*Buffer] | provenance | | -| windows.cpp:563:40:563:50 | *& ... [*Buffer] | windows.cpp:563:26:563:37 | RtlCopyUnicodeString output argument [*Buffer] | provenance | MaD:41 | +| windows.cpp:563:40:563:50 | *& ... [*Buffer] | windows.cpp:563:26:563:37 | RtlCopyUnicodeString output argument [*Buffer] | provenance | MaD:48 | | windows.cpp:564:10:564:20 | *dest_string [*Buffer] | windows.cpp:564:10:564:30 | access to array | provenance | | | windows.cpp:564:10:564:20 | *dest_string [*Buffer] | windows.cpp:564:22:564:27 | *Buffer | provenance | | | windows.cpp:564:22:564:27 | *Buffer | windows.cpp:564:10:564:30 | access to array | provenance | | | windows.cpp:568:19:568:29 | RtlMoveMemory output argument | windows.cpp:569:10:569:23 | access to array | provenance | | -| windows.cpp:568:32:568:33 | *& ... | windows.cpp:568:19:568:29 | RtlMoveMemory output argument | provenance | MaD:44 | +| windows.cpp:568:32:568:33 | *& ... | windows.cpp:568:19:568:29 | RtlMoveMemory output argument | provenance | MaD:51 | | windows.cpp:573:27:573:37 | RtlMoveVolatileMemory output argument | windows.cpp:574:10:574:23 | access to array | provenance | | -| windows.cpp:573:40:573:41 | *& ... | windows.cpp:573:27:573:37 | RtlMoveVolatileMemory output argument | provenance | MaD:45 | -| windows.cpp:645:45:645:50 | WinHttpReadData output argument | windows.cpp:647:10:647:16 | * ... | provenance | Src:MaD:23 | -| windows.cpp:652:48:652:53 | WinHttpReadDataEx output argument | windows.cpp:654:10:654:16 | * ... | provenance | Src:MaD:24 | -| windows.cpp:659:47:659:52 | WinHttpQueryHeaders output argument | windows.cpp:661:10:661:16 | * ... | provenance | Src:MaD:19 | -| windows.cpp:669:70:669:79 | WinHttpQueryHeadersEx output argument | windows.cpp:673:10:673:29 | * ... | provenance | Src:MaD:21 | -| windows.cpp:669:82:669:87 | WinHttpQueryHeadersEx output argument | windows.cpp:671:10:671:16 | * ... | provenance | Src:MaD:22 | -| windows.cpp:669:105:669:112 | WinHttpQueryHeadersEx output argument | windows.cpp:675:10:675:27 | * ... | provenance | Src:MaD:20 | +| windows.cpp:573:40:573:41 | *& ... | windows.cpp:573:27:573:37 | RtlMoveVolatileMemory output argument | provenance | MaD:52 | +| windows.cpp:645:45:645:50 | WinHttpReadData output argument | windows.cpp:647:10:647:16 | * ... | provenance | Src:MaD:30 | +| windows.cpp:652:48:652:53 | WinHttpReadDataEx output argument | windows.cpp:654:10:654:16 | * ... | provenance | Src:MaD:31 | +| windows.cpp:659:47:659:52 | WinHttpQueryHeaders output argument | windows.cpp:661:10:661:16 | * ... | provenance | Src:MaD:26 | +| windows.cpp:669:70:669:79 | WinHttpQueryHeadersEx output argument | windows.cpp:673:10:673:29 | * ... | provenance | Src:MaD:28 | +| windows.cpp:669:82:669:87 | WinHttpQueryHeadersEx output argument | windows.cpp:671:10:671:16 | * ... | provenance | Src:MaD:29 | +| windows.cpp:669:105:669:112 | WinHttpQueryHeadersEx output argument | windows.cpp:675:10:675:27 | * ... | provenance | Src:MaD:27 | | windows.cpp:728:5:728:28 | ... = ... | windows.cpp:729:35:729:35 | *x | provenance | | | windows.cpp:728:12:728:28 | call to source | windows.cpp:728:5:728:28 | ... = ... | provenance | | -| windows.cpp:729:35:729:35 | *x | windows.cpp:729:44:729:57 | WinHttpCrackUrl output argument | provenance | MaD:46 | +| windows.cpp:729:35:729:35 | *x | windows.cpp:729:44:729:57 | WinHttpCrackUrl output argument | provenance | MaD:53 | | windows.cpp:729:44:729:57 | WinHttpCrackUrl output argument | windows.cpp:731:10:731:36 | * ... | provenance | | | windows.cpp:729:44:729:57 | WinHttpCrackUrl output argument | windows.cpp:733:10:733:35 | * ... | provenance | | | windows.cpp:729:44:729:57 | WinHttpCrackUrl output argument | windows.cpp:735:10:735:37 | * ... | provenance | | @@ -337,6 +344,13 @@ edges | windows.cpp:936:70:936:78 | HttpReceiveClientCertificate output argument | windows.cpp:937:15:937:48 | *& ... | provenance | Src:MaD:6 | | windows.cpp:936:70:936:78 | HttpReceiveClientCertificate output argument | windows.cpp:941:10:941:31 | * ... | provenance | Src:MaD:6 | | windows.cpp:937:15:937:48 | *& ... | windows.cpp:939:10:939:11 | * ... | provenance | | +| windows.cpp:994:35:994:38 | RegQueryValueA output argument | windows.cpp:996:10:996:14 | * ... | provenance | Src:MaD:22 | +| windows.cpp:1001:36:1001:39 | RegQueryValueW output argument | windows.cpp:1003:10:1003:14 | * ... | provenance | Src:MaD:25 | +| windows.cpp:1009:53:1009:56 | RegQueryValueExA output argument | windows.cpp:1011:10:1011:14 | * ... | provenance | Src:MaD:23 | +| windows.cpp:1017:54:1017:57 | RegQueryValueExW output argument | windows.cpp:1019:10:1019:14 | * ... | provenance | Src:MaD:24 | +| windows.cpp:1025:46:1025:49 | RegQueryMultipleValuesA output argument | windows.cpp:1027:10:1027:14 | * ... | provenance | Src:MaD:20 | +| windows.cpp:1033:46:1033:49 | RegQueryMultipleValuesW output argument | windows.cpp:1035:10:1035:14 | * ... | provenance | Src:MaD:21 | +| windows.cpp:1041:53:1041:56 | RegGetValueA output argument | windows.cpp:1043:10:1043:14 | * ... | provenance | Src:MaD:19 | nodes | asio_streams.cpp:87:34:87:44 | read_until output argument | semmle.label | read_until output argument | | asio_streams.cpp:91:7:91:17 | recv_buffer | semmle.label | recv_buffer | @@ -653,6 +667,20 @@ nodes | windows.cpp:937:15:937:48 | *& ... | semmle.label | *& ... | | windows.cpp:939:10:939:11 | * ... | semmle.label | * ... | | windows.cpp:941:10:941:31 | * ... | semmle.label | * ... | +| windows.cpp:994:35:994:38 | RegQueryValueA output argument | semmle.label | RegQueryValueA output argument | +| windows.cpp:996:10:996:14 | * ... | semmle.label | * ... | +| windows.cpp:1001:36:1001:39 | RegQueryValueW output argument | semmle.label | RegQueryValueW output argument | +| windows.cpp:1003:10:1003:14 | * ... | semmle.label | * ... | +| windows.cpp:1009:53:1009:56 | RegQueryValueExA output argument | semmle.label | RegQueryValueExA output argument | +| windows.cpp:1011:10:1011:14 | * ... | semmle.label | * ... | +| windows.cpp:1017:54:1017:57 | RegQueryValueExW output argument | semmle.label | RegQueryValueExW output argument | +| windows.cpp:1019:10:1019:14 | * ... | semmle.label | * ... | +| windows.cpp:1025:46:1025:49 | RegQueryMultipleValuesA output argument | semmle.label | RegQueryMultipleValuesA output argument | +| windows.cpp:1027:10:1027:14 | * ... | semmle.label | * ... | +| windows.cpp:1033:46:1033:49 | RegQueryMultipleValuesW output argument | semmle.label | RegQueryMultipleValuesW output argument | +| windows.cpp:1035:10:1035:14 | * ... | semmle.label | * ... | +| windows.cpp:1041:53:1041:56 | RegGetValueA output argument | semmle.label | RegGetValueA output argument | +| windows.cpp:1043:10:1043:14 | * ... | semmle.label | * ... | subpaths | test.cpp:32:41:32:41 | x | test.cpp:7:47:7:52 | value2 | test.cpp:7:5:7:30 | *ymlStepGenerated_with_body | test.cpp:32:11:32:36 | call to ymlStepGenerated_with_body | | test.cpp:172:51:172:51 | x | test.cpp:164:34:164:34 | x | test.cpp:164:7:164:7 | *templateFunction3 | test.cpp:172:13:172:44 | call to templateFunction3 | diff --git a/cpp/ql/test/library-tests/dataflow/external-models/sources.expected b/cpp/ql/test/library-tests/dataflow/external-models/sources.expected index 6585c88dc6d..54320be2033 100644 --- a/cpp/ql/test/library-tests/dataflow/external-models/sources.expected +++ b/cpp/ql/test/library-tests/dataflow/external-models/sources.expected @@ -43,3 +43,10 @@ | windows.cpp:900:64:900:77 | HttpReceiveHttpRequest output argument | remote | | windows.cpp:929:70:929:75 | HttpReceiveRequestEntityBody output argument | remote | | windows.cpp:936:70:936:78 | HttpReceiveClientCertificate output argument | remote | +| windows.cpp:994:35:994:38 | RegQueryValueA output argument | local | +| windows.cpp:1001:36:1001:39 | RegQueryValueW output argument | local | +| windows.cpp:1009:53:1009:56 | RegQueryValueExA output argument | local | +| windows.cpp:1017:54:1017:57 | RegQueryValueExW output argument | local | +| windows.cpp:1025:46:1025:49 | RegQueryMultipleValuesA output argument | local | +| windows.cpp:1033:46:1033:49 | RegQueryMultipleValuesW output argument | local | +| windows.cpp:1041:53:1041:56 | RegGetValueA output argument | local | diff --git a/cpp/ql/test/library-tests/dataflow/external-models/windows.cpp b/cpp/ql/test/library-tests/dataflow/external-models/windows.cpp index f098f7344e4..5afa72723d6 100644 --- a/cpp/ql/test/library-tests/dataflow/external-models/windows.cpp +++ b/cpp/ql/test/library-tests/dataflow/external-models/windows.cpp @@ -940,4 +940,106 @@ void test_http_server_api(HANDLE hRequestQueue) { sink(certInfo.pCertEncoded); sink(*certInfo.pCertEncoded); // $ ir } +} + +using HKEY = void*; +using BYTE = unsigned char; +using LPBYTE = BYTE*; +using PLONG = LONG*; + +typedef struct value_entA { + LPSTR ve_valuename; + DWORD ve_valuelen; + DWORD_PTR ve_valueptr; + DWORD ve_type; +} VALENTA, *PVALENTA; + +typedef struct value_entW { + LPWSTR ve_valuename; + DWORD ve_valuelen; + DWORD_PTR ve_valueptr; + DWORD ve_type; +} VALENTW, *PVALENTW; + +LONG RegQueryValueA(HKEY hKey, LPCSTR lpSubKey, LPSTR lpData, PLONG lpcbData); +LONG RegQueryValueW(HKEY hKey, LPCWSTR lpSubKey, LPWSTR lpData, PLONG lpcbData); + +LONG RegQueryValueExA( + HKEY hKey, LPCSTR lpValueName, LPDWORD lpReserved, LPDWORD lpType, LPBYTE lpData, + LPDWORD lpcbData +); + +LONG RegQueryValueExW( + HKEY hKey, LPCWSTR lpValueName, LPDWORD lpReserved, LPDWORD lpType, LPBYTE lpData, + LPDWORD lpcbData +); + +LONG RegGetValueA( + HKEY hKey, LPCSTR lpSubKey, LPCSTR lpValue, DWORD flags, LPDWORD lpType, PVOID lpData, + LPDWORD lpcbData +); + +LONG RegQueryMultipleValuesA( + HKEY hKey, PVALENTA valList, DWORD numVals, LPSTR valueBuffer, LPDWORD totalSize +); + +LONG RegQueryMultipleValuesW( + HKEY hKey, PVALENTW valList, DWORD numVals, LPWSTR valueBuffer, LPDWORD totalSize +); + +void test_registry_queries(HKEY hKey) { + { + char data[256]; + LONG dataSize = sizeof(data); + RegQueryValueA(hKey, "value", data, &dataSize); + sink(data); // clean + sink(*data); // $ ir + } + { + wchar_t data[256]; + LONG dataSize = sizeof(data); + RegQueryValueW(hKey, L"value", data, &dataSize); + sink(data); // clean + sink(*data); // $ ir + } + { + BYTE data[256]; + DWORD dataSize = sizeof(data); + DWORD type; + RegQueryValueExA(hKey, "value", nullptr, &type, data, &dataSize); + sink(data); // clean + sink(*data); // $ ir + } + { + BYTE data[256]; + DWORD dataSize = sizeof(data); + DWORD type; + RegQueryValueExW(hKey, L"value", nullptr, &type, data, &dataSize); + sink(data); // clean + sink(*data); // $ ir + } + { + VALENTA values[1]; + char data[256]; + DWORD dataSize = sizeof(data); + RegQueryMultipleValuesA(hKey, values, 1, data, &dataSize); + sink(data); // clean + sink(*data); // $ ir + } + { + VALENTW values[1]; + wchar_t data[256]; + DWORD dataSize = sizeof(data); + RegQueryMultipleValuesW(hKey, values, 1, data, &dataSize); + sink(data); // clean + sink(*data); // $ ir + } + { + BYTE data[256]; + DWORD dataSize = sizeof(data); + DWORD type; + RegGetValueA(hKey, "subkey", "value", 0, &type, data, &dataSize); + sink(data); // clean + sink(*data); // $ ir + } } \ No newline at end of file diff --git a/csharp/ql/integration-tests/windows/subst/file.expected b/csharp/ql/integration-tests/windows/subst/file.expected index 6421dce4121..d14faca364b 100644 --- a/csharp/ql/integration-tests/windows/subst/file.expected +++ b/csharp/ql/integration-tests/windows/subst/file.expected @@ -1,8 +1,8 @@ -| code/Program.cs:0:0:0:0 | code/Program.cs | | -| code/obj/Debug/net9.0/.NETCoreApp,Version=v9.0.AssemblyAttributes.cs:0:0:0:0 | code/obj/Debug/net9.0/.NETCoreApp,Version=v9.0.AssemblyAttributes.cs | | -| code/obj/Debug/net9.0/dotnet_build.AssemblyInfo.cs:0:0:0:0 | code/obj/Debug/net9.0/dotnet_build.AssemblyInfo.cs | | -| code/obj/Debug/net9.0/dotnet_build.GlobalUsings.g.cs:0:0:0:0 | code/obj/Debug/net9.0/dotnet_build.GlobalUsings.g.cs | | -| code/obj/Debug/net9.0/dotnet_build.dll:0:0:0:0 | code/obj/Debug/net9.0/dotnet_build.dll | | +| code/Program.cs:0:0:0:0 | code/Program.cs | relative | +| code/dotnet_build.csproj:0:0:0:0 | code/dotnet_build.csproj | relative | +| code/obj/Debug/net9.0/.NETCoreApp,Version=v9.0.AssemblyAttributes.cs:0:0:0:0 | code/obj/Debug/net9.0/.NETCoreApp,Version=v9.0.AssemblyAttributes.cs | relative | +| code/obj/Debug/net9.0/dotnet_build.AssemblyInfo.cs:0:0:0:0 | code/obj/Debug/net9.0/dotnet_build.AssemblyInfo.cs | relative | +| code/obj/Debug/net9.0/dotnet_build.GlobalUsings.g.cs:0:0:0:0 | code/obj/Debug/net9.0/dotnet_build.GlobalUsings.g.cs | relative | +| code/obj/Debug/net9.0/dotnet_build.dll:0:0:0:0 | code/obj/Debug/net9.0/dotnet_build.dll | relative | +| code/obj/dotnet_build.csproj.nuget.g.props:0:0:0:0 | code/obj/dotnet_build.csproj.nuget.g.props | relative | | file://:0:0:0:0 | | | -| file://Z:/dotnet_build.csproj:0:0:0:0 | Z:/dotnet_build.csproj | relative | -| file://Z:/obj/dotnet_build.csproj.nuget.g.props:0:0:0:0 | Z:/obj/dotnet_build.csproj.nuget.g.props | relative | diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index 8c217837614..4efa1daac56 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -770,7 +770,7 @@ func normalizedPath(ast *ast.File, fset *token.FileSet) string { if err != nil { return file } - return path + return util.ResolvePath(path) } // extractFile extracts AST information for the given file diff --git a/go/extractor/go.mod b/go/extractor/go.mod index b45815abbc9..c92b654c8c6 100644 --- a/go/extractor/go.mod +++ b/go/extractor/go.mod @@ -13,7 +13,10 @@ require ( golang.org/x/tools v0.48.0 ) -require github.com/stretchr/testify v1.11.1 +require ( + github.com/stretchr/testify v1.11.1 + golang.org/x/sys v0.47.0 +) require ( github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go/extractor/go.sum b/go/extractor/go.sum index 76fbec137b4..b57649cbb35 100644 --- a/go/extractor/go.sum +++ b/go/extractor/go.sum @@ -10,6 +10,8 @@ golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/go/extractor/util/BUILD.bazel b/go/extractor/util/BUILD.bazel index ccebf5ebd86..6d6c6c7a993 100644 --- a/go/extractor/util/BUILD.bazel +++ b/go/extractor/util/BUILD.bazel @@ -9,11 +9,20 @@ go_library( "logging.go", "overlays.go", "semver.go", + "subst_other.go", + "subst_windows.go", "util.go", ], importpath = "github.com/github/codeql-go/extractor/util", visibility = ["//visibility:public"], - deps = ["@org_golang_x_mod//semver"], + deps = [ + "@org_golang_x_mod//semver", + ] + select({ + "@rules_go//go/platform:windows": [ + "@org_golang_x_sys//windows", + ], + "//conditions:default": [], + }), ) go_test( diff --git a/go/extractor/util/subst_other.go b/go/extractor/util/subst_other.go new file mode 100644 index 00000000000..b32420a7746 --- /dev/null +++ b/go/extractor/util/subst_other.go @@ -0,0 +1,6 @@ +//go:build !windows + +package util + +// ResolvePath is a no-op on non-Windows platforms. +func ResolvePath(path string) string { return path } diff --git a/go/extractor/util/subst_windows.go b/go/extractor/util/subst_windows.go new file mode 100644 index 00000000000..a64f21576fd --- /dev/null +++ b/go/extractor/util/subst_windows.go @@ -0,0 +1,91 @@ +//go:build windows + +package util + +import ( + "os" + "path/filepath" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +var ( + dll *syscall.DLL + procResolve *syscall.Proc + procFree *syscall.Proc + available bool +) + +func init() { + dist := os.Getenv("CODEQL_DIST") + if dist == "" { + return + } + dllPath := filepath.Join(dist, "tools", "win64", "canonicalize.dll") + d, err := syscall.LoadDLL(dllPath) + if err != nil { + return + } + p, err := d.FindProc("resolve_subst") + if err != nil { + d.Release() + return + } + f, err := d.FindProc("resolve_subst_free") + if err != nil { + d.Release() + return + } + dll = d + procResolve = p + procFree = f + available = true +} + +// If "path" is an absolute path starting with a "subst"ed drive letter, return an +// equivalent path with the drive letter replaced by its target. Otherwise return +// "path" unchanged. +func ResolvePath(path string) string { + if len(path) < 3 { + return path + } + if path[1] != ':' { + return path + } + if path[2] != '\\' && path[2] != '/' { + return path + } + c := path[0] + if !((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')) { + return path + } + + resolved, ok := resolveDrive(path[:3]) + if !ok { + return path + } + return resolved + path[2:] +} + +// Given a drive root like "X:\" (or "X:/"), returns the path that drive is +// "subst"ed to. Returns false if the drive is not "subst"ed or an error occurred. +func resolveDrive(driveRoot string) (string, bool) { + if !available { + return "", false + } + driveBytes, err := windows.ByteSliceFromString(driveRoot) + if err != nil { + return "", false + } + ret, _, _ := procResolve.Call(uintptr(unsafe.Pointer(&driveBytes[0]))) + if ret == 0 { + return "", false + } + result := windows.BytePtrToString((*byte)(unsafe.Pointer(ret))) + if procFree != nil { + procFree.Call(ret) + } + return result, true +} diff --git a/go/ql/integration-tests/subst/file.expected b/go/ql/integration-tests/subst/file.expected index b86b34681e4..96cce26cff5 100644 --- a/go/ql/integration-tests/subst/file.expected +++ b/go/ql/integration-tests/subst/file.expected @@ -1 +1 @@ -| file://Z:/main.go:0:0:0:0 | Z:/main.go | relative | +| code/main.go:0:0:0:0 | code/main.go | relative | diff --git a/java/kotlin-extractor/src/main/java/com/semmle/util/files/FileUtil.java b/java/kotlin-extractor/src/main/java/com/semmle/util/files/FileUtil.java index 79ce2d8d8d3..19bdc786c5e 100644 --- a/java/kotlin-extractor/src/main/java/com/semmle/util/files/FileUtil.java +++ b/java/kotlin-extractor/src/main/java/com/semmle/util/files/FileUtil.java @@ -1237,13 +1237,14 @@ public class FileUtil /** * Try to convert a file into a canonical file. Handles the possible IO exception by just making - * the path absolute. + * the path absolute. Also resolves subst drives on Windows. */ public static File tryMakeCanonical (File f) { try { - return f.getCanonicalFile(); - } + // getCanonicalFile does not canonicalize subst drives on Windows, so do this separately. This + // is a no-op on non-Windows platforms. + return SubstResolver.resolve(f.getCanonicalFile()); } catch (IOException ignored) { Exceptions.ignore(ignored, "Can't log error: Could be too verbose."); return new File(simplifyPath(f)); diff --git a/java/kotlin-extractor/src/main/java/com/semmle/util/files/SubstResolver.java b/java/kotlin-extractor/src/main/java/com/semmle/util/files/SubstResolver.java new file mode 100644 index 00000000000..017d4fb8399 --- /dev/null +++ b/java/kotlin-extractor/src/main/java/com/semmle/util/files/SubstResolver.java @@ -0,0 +1,87 @@ +package com.semmle.util.files; + +import com.semmle.util.exception.Exceptions; +import java.io.File; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +/** + * Resolves Windows {@code subst}ed drive letters to their underlying paths. On non-Windows + * platforms, or when the native library failed to load, resolving will be a no-op. + */ +public class SubstResolver { + private static final boolean available; + + static { + boolean loaded = false; + // Cheap check to see that we are on Windows. Avoids static initialization of {@code Env}. + if (File.separatorChar == '\\') { + String dist = System.getenv("CODEQL_DIST"); + if (dist != null && !dist.isEmpty()) { + try { + Path library = Paths.get(dist).resolve("tools") + .resolve("win64").resolve("canonicalize.dll").toAbsolutePath(); + System.load(library.toString()); + loaded = true; + } catch (RuntimeException | UnsatisfiedLinkError ignored) { + Exceptions.ignore(ignored, "Fall back to resolution being a no-op."); + } + } + } + available = loaded; + } + + private SubstResolver() {} + + /** + * Given a drive root like {@code "X:\\"} (or {@code "X:/"}), returns the path that drive is + * {@code subst}ed to, or {@code null} if the drive root was not subst drive or if an internal + * error occurred. + */ + private static native String nativeResolveSubst(String driveRoot); + + /** + * If {@code f} is an absolute path starting with a {@code subst}ed drive letter, return an + * equivalent path with the drive letter replaced by its target. Otherwise return {@code f} + * unchanged. + */ + public static File resolve(File f) { + if (!available) { + return f; + } + String path = f.getPath(); + if (path.length() < 3 || path.charAt(1) != ':') { + return f; + } + char sep = path.charAt(2); + if (sep != '\\' && sep != '/') { + return f; + } + if (!isDriveLetter(path.charAt(0))) { + return f; + } + + String resolved; + try { + resolved = nativeResolveSubst(path.substring(0, 3)); + } catch (RuntimeException | UnsatisfiedLinkError ignored) { + Exceptions.ignore(ignored, "Fall back to resolution being a no-op."); + + return f; + } + if (resolved == null) { + return f; + } + + // Append the remainder of the original path. The native side strips away + // any trailing separator. + return new File(resolved + path.substring(2)); + } + + private static boolean isDriveLetter(char c) { + return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); + } +} diff --git a/java/ql/integration-tests/java/subst/file.expected b/java/ql/integration-tests/java/subst/file.expected index 467876a7445..02d4543cfca 100644 --- a/java/ql/integration-tests/java/subst/file.expected +++ b/java/ql/integration-tests/java/subst/file.expected @@ -1,5 +1,5 @@ +| code/Test.class:0:0:0:0 | Test | relative | +| code/test1.java:0:0:0:0 | test1 | relative | +| code/test2.kt:0:0:0:0 | test2 | relative | | file://:0:0:0:0 | | | | file://:0:0:0:0 | | | -| file://Z:/Test.class:0:0:0:0 | Test | relative | -| file://Z:/test1.java:0:0:0:0 | test1 | relative | -| file://Z:/test2.kt:0:0:0:0 | test2 | relative | diff --git a/javascript/ql/integration-tests/subst/file.expected b/javascript/ql/integration-tests/subst/file.expected index 10416eb5e73..6d3663acac8 100644 --- a/javascript/ql/integration-tests/subst/file.expected +++ b/javascript/ql/integration-tests/subst/file.expected @@ -1,2 +1,2 @@ -| file://Z:/main.js:0:0:0:0 | Z:/main.js | relative | -| file://Z:/test.ts:0:0:0:0 | Z:/test.ts | relative | +| code/main.js:0:0:0:0 | code/main.js | relative | +| code/test.ts:0:0:0:0 | code/test.ts | relative | diff --git a/python/ql/consistency-queries/CfgConsistency.ql b/python/ql/consistency-queries/CfgConsistency.ql new file mode 100644 index 00000000000..ab13eddf190 --- /dev/null +++ b/python/ql/consistency-queries/CfgConsistency.ql @@ -0,0 +1,2 @@ +import semmle.python.controlflow.internal.AstNodeImpl +import ControlFlow::Consistency diff --git a/python/ql/integration-tests/subst/file.expected b/python/ql/integration-tests/subst/file.expected index 1bc4d03914a..e001d4959bb 100644 --- a/python/ql/integration-tests/subst/file.expected +++ b/python/ql/integration-tests/subst/file.expected @@ -1 +1 @@ -| code/main.py:0:0:0:0 | code/main.py | | +| code/main.py:0:0:0:0 | code/main.py | relative | diff --git a/python/ql/lib/ide-contextual-queries/printCfg.ql b/python/ql/lib/ide-contextual-queries/printCfg.ql new file mode 100644 index 00000000000..819022c750c --- /dev/null +++ b/python/ql/lib/ide-contextual-queries/printCfg.ql @@ -0,0 +1,41 @@ +/** + * @name Print CFG + * @description Produces a representation of a file's Control Flow Graph. + * This query is used by the VS Code extension. + * @id py/print-cfg + * @kind graph + * @tags ide-contextual-queries/print-cfg + */ + +import semmle.python.Files as Files +import semmle.python.controlflow.internal.AstNodeImpl + +external string selectedSourceFile(); + +private predicate selectedSourceFileAlias = selectedSourceFile/0; + +external int selectedSourceLine(); + +private predicate selectedSourceLineAlias = selectedSourceLine/0; + +external int selectedSourceColumn(); + +private predicate selectedSourceColumnAlias = selectedSourceColumn/0; + +module ViewCfgQueryInput implements ControlFlow::ViewCfgQueryInputSig { + predicate selectedSourceFile = selectedSourceFileAlias/0; + + predicate selectedSourceLine = selectedSourceLineAlias/0; + + predicate selectedSourceColumn = selectedSourceColumnAlias/0; + + predicate cfgScopeSpan( + Ast::Callable scope, Files::File file, int startLine, int startColumn, int endLine, + int endColumn + ) { + file = scope.getLocation().getFile() and + scope.getLocation().hasLocationInfo(_, startLine, startColumn, endLine, endColumn) + } +} + +import ControlFlow::ViewCfgQuery diff --git a/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll b/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll new file mode 100644 index 00000000000..78218f48921 --- /dev/null +++ b/python/ql/lib/semmle/python/controlflow/internal/AstNodeImpl.qll @@ -0,0 +1,1772 @@ +/** + * Provides classes for the shared control-flow library, mediating between + * the Python AST and `AstSig`. + * + * The `Ast` module wraps Python's `Stmt`, `Expr`, `Scope`, and `Pattern`, + * and adds two synthetic kinds of node: + * - `BlockStmt`, identifying a body slot of a parent AST node (e.g. an + * `if`'s then or else branch). `Py::StmtList` itself is not directly + * wrapped. + * - Intermediate nodes for multi-operand boolean expressions. + */ +overlay[local?] +module; + +private import python as Py +private import codeql.controlflow.ControlFlowGraph +private import codeql.controlflow.SuccessorType +private import codeql.util.Void + +/** + * Gets the bound `Name` of a PEP 695 type parameter (`TypeVar`, + * `ParamSpec`, or `TypeVarTuple`). The base `TypeParameter` class does + * not expose `getName()`; this helper dispatches over the subtypes. + */ +private Py::Name typeParameterName(Py::TypeParameter tp) { + result = tp.(Py::TypeVar).getName() + or + result = tp.(Py::ParamSpec).getName() + or + result = tp.(Py::TypeVarTuple).getName() +} + +/** Provides the Python implementation of the shared CFG `AstSig`. */ +module Ast implements AstSig { + private newtype TAstNode = + TPyStmt(Py::Stmt s) or + TPyExpr(Py::Expr e) { not e instanceof Py::BoolExpr } or + TScope(Py::Scope sc) or + TPattern(Py::Pattern p) or + /** + * A synthetic node representing an operand pair of an `and`/`or` + * expression. For `a and b and c` (operands 0, 1, 2) we model the + * operation as a right-nested tree: pair 0 represents the whole + * expression with left=a and right=pair 1; pair 1 represents + * `b and c` with left=b and right=c. Each Python `Py::BoolExpr` + * with `n` operands has `n - 1` such pairs (indices `0 .. n - 2`). + */ + TBoolExprPair(Py::BoolExpr be, int index) { index = [0 .. count(be.getAValue()) - 2] } or + /** + * A synthetic block statement, wrapping a `Py::StmtList`. Each list of + * statements that represents an imperative block (a function/class/module + * body, an `if`/`while`/`for` branch, a `try`/`except`/`finally` body, + * etc.) becomes one `BlockStmt` node in the CFG. `Py::StmtList`s used + * in other roles - `Try.getHandlers()` (iterated via `getCatch`) and + * `MatchStmt.getCases()` (iterated via `getCase`) - are excluded, as + * the shared library's `Try`/`Switch` logic walks their items + * individually. + */ + TBlockStmt(Py::StmtList sl) { + not sl = any(Py::Try t).getHandlers() and + not sl = any(Py::MatchStmt m).getCases() + } + + /** + * The union of `TPyStmt` (wrapping `Py::Stmt`) and `TBlockStmt` (wrapping + * `Py::StmtList`). Both represent the kinds of node that can appear in + * a `Stmt` position in the CFG. + */ + private class TStmt = TPyStmt or TBlockStmt; + + /** + * The union of `TPyExpr` (wrapping non-boolean `Py::Expr`) and + * `TBoolExprPair` (synthetic operand pairs of `and`/`or` expressions). + * Both represent the kinds of node that can appear in an `Expr` + * position in the CFG. + */ + private class TExpr = TPyExpr or TBoolExprPair; + + /** + * An AST node visible to the shared CFG. + * + * This is the abstract implementation class. It enforces that each + * concrete subclass provides `toString`, `getLocation`, and + * `getEnclosingCallable` (one subclass per `TAstNode` newtype branch). + * The public alias `AstNode` is what users (and the `AstSig` signature) + * see; subclasses inside this module extend `AstNodeImpl` directly. + */ + abstract private class AstNodeImpl extends TAstNode { + /** Gets a textual representation of this AST node. */ + abstract string toString(); + + /** Gets the location of this AST node. */ + abstract Py::Location getLocation(); + + /** Gets the enclosing callable that contains this node, if any. */ + abstract Callable getEnclosingCallable(); + + /** Gets the underlying Python `Stmt`, if this node wraps one. */ + Py::Stmt asStmt() { this = TPyStmt(result) } + + /** + * Gets the underlying Python `Expr`, if this node wraps one. Boolean + * expressions are represented by `TBoolExprPair(_, 0)`; this + * predicate also recovers the underlying `Py::BoolExpr` from such a + * representation. + */ + Py::Expr asExpr() { + this = TPyExpr(result) + or + this = TBoolExprPair(result, 0) + } + + /** Gets the underlying Python `Scope`, if this node wraps one. */ + Py::Scope asScope() { this = TScope(result) } + + /** Gets the underlying Python `Pattern`, if this node wraps one. */ + Py::Pattern asPattern() { this = TPattern(result) } + + /** Gets the underlying Python `StmtList`, if this node is a `BlockStmt`. */ + Py::StmtList asStmtList() { this = TBlockStmt(result) } + + /** + * Gets the child of this AST node at the specified (zero-based) + * index, in evaluation order. Subclasses with children override + * this method. + */ + AstNode getChild(int index) { none() } + } + + /** An AST node visible to the shared CFG. */ + final class AstNode = AstNodeImpl; + + /** Gets the immediately enclosing callable that contains `node`. */ + Callable getEnclosingCallable(AstNode node) { result = node.getEnclosingCallable() } + + /** + * A callable: a function, class, or module scope. + * + * In Python, all three are executable scopes with statement bodies. + */ + class Callable extends AstNodeImpl, TScope { + private Py::Scope sc; + + Callable() { this = TScope(sc) } + + override string toString() { result = sc.toString() } + + override Py::Location getLocation() { result = sc.getLocation() } + + override Callable getEnclosingCallable() { result.asScope() = sc.getEnclosingScope() } + } + + /** Gets the body of callable `c`. */ + AstNode callableGetBody(Callable c) { result.asStmtList() = c.asScope().getBody() } + + /** + * A parameter of a callable. + * + * Modeled per the C# template (`csharp/.../ControlFlowGraph.qll`): + * each Python parameter (the `Py::Parameter` AST node, which is a `Name` + * or — Python 2 only — a `Tuple` in store context) becomes a CFG node + * at a stable position in the enclosing callable's entry sequence. + * + * Default-value expressions for positional and keyword-only parameters + * are wired separately on the `FunctionDefExpr` / `LambdaExpr` wrappers + * (they evaluate at function-definition time, not at call time). + * `Parameter::getDefaultValue()` returns `none()` here, signalling to + * the shared library that the parameter never falls back to a default + * during call binding. This mirrors C# for non-optional parameters. + */ + class Parameter extends Expr { + private Py::Parameter param; + + Parameter() { this = TPyExpr(param) } + + /** Gets the underlying Python parameter. */ + Py::Parameter asParameter() { result = param } + + /** + * Gets the default-value expression of this parameter, if any. + * + * Returns `none()`: defaults evaluate at function-definition time and + * are wired into the CFG via `FunctionDefExpr.getDefault` / + * `LambdaExpr.getDefault`. The shared library calls this predicate + * to model the "missing argument → evaluate default" fallback during + * call binding, which Python does not model at the CFG level. + */ + Expr getDefaultValue() { none() } + + /** + * Gets the pattern for this parameter. In Python, there is no destructuring + * pattern syntax for parameters, so the pattern is the parameter itself. + */ + AstNode getPattern() { result = this } + } + + /** + * Gets the `index`th parameter of callable `c`, ordered as Python binds + * them at call time: positional, then vararg (`*args`), then + * keyword-only, then kwarg (`**kwargs`). + */ + Parameter callableGetParameter(Callable c, int index) { + exists(Py::Function f | f = c.asScope() | + result.asParameter() = + rank[index + 1](Py::Parameter p, int subOrder, int subIndex | + // positional parameters first + p = f.getArg(subIndex) and subOrder = 0 + or + // then *args + p = f.getVararg() and subOrder = 1 and subIndex = 0 + or + // then keyword-only parameters + p = f.getKeywordOnlyArg(subIndex) and subOrder = 2 + or + // finally **kwargs + p = f.getKwarg() and subOrder = 3 and subIndex = 0 + | + p order by subOrder, subIndex + ) + ) + } + + /** A statement. */ + class Stmt extends AstNodeImpl, TStmt { + // For `TPyStmt` instances, delegate to the wrapped Python statement. + // `BlockStmt` (the only `TBlockStmt` subclass) provides its own overrides. + override string toString() { result = this.asStmt().toString() } + + override Py::Location getLocation() { result = this.asStmt().getLocation() } + + override Callable getEnclosingCallable() { result.asScope() = this.asStmt().getScope() } + } + + /** An expression. */ + class Expr extends AstNodeImpl, TExpr { + // For `TPyExpr` instances, delegate to the wrapped Python expression. + // `BinaryExpr` (the only `TBoolExprPair` subclass) provides its own overrides. + override string toString() { result = this.asExpr().toString() } + + override Py::Location getLocation() { result = this.asExpr().getLocation() } + + override Callable getEnclosingCallable() { result.asScope() = this.asExpr().getScope() } + } + + /** A pattern in a `match` statement. */ + additional class Pattern extends AstNodeImpl, TPattern { + private Py::Pattern p; + + Pattern() { this = TPattern(p) } + + override string toString() { result = p.toString() } + + override Py::Location getLocation() { result = p.getLocation() } + + override Callable getEnclosingCallable() { result.asScope() = p.getScope() } + } + + /** + * A `case x` pattern that binds `x` to the matched value. + */ + additional class MatchCapturePattern extends Pattern { + private Py::MatchCapturePattern cap; + + MatchCapturePattern() { this = TPattern(cap) } + + /** Gets the bound Name expression. */ + Expr getVariable() { result.asExpr() = cap.getVariable() } + + override AstNode getChild(int index) { index = 0 and result = this.getVariable() } + } + + /** + * A `case pattern as name` pattern. + */ + additional class MatchAsPattern extends Pattern { + private Py::MatchAsPattern asp; + + MatchAsPattern() { this = TPattern(asp) } + + /** Gets the inner pattern. */ + AstNode getPattern() { result.asPattern() = asp.getPattern() } + + /** Gets the bound Name expression. */ + Expr getAlias() { result.asExpr() = asp.getAlias() } + + override AstNode getChild(int index) { + index = 0 and result = this.getPattern() + or + index = 1 and result = this.getAlias() + } + } + + /** + * A `case [a, b, *rest]` star pattern. Binds `rest` to the remaining + * elements of the sequence. + */ + additional class MatchStarPattern extends Pattern { + private Py::MatchStarPattern starp; + + MatchStarPattern() { this = TPattern(starp) } + + /** Gets the target Pattern (a `MatchCapturePattern` if `*rest`). */ + AstNode getTarget() { result.asPattern() = starp.getTarget() } + + override AstNode getChild(int index) { index = 0 and result = this.getTarget() } + } + + /** + * A `case [a, b, ...]` sequence pattern. Recurses into the sub-patterns. + */ + additional class MatchSequencePattern extends Pattern { + private Py::MatchSequencePattern seqp; + + MatchSequencePattern() { this = TPattern(seqp) } + + /** Gets the `n`th sub-pattern. */ + AstNode getPattern(int n) { result.asPattern() = seqp.getPattern(n) } + + override AstNode getChild(int index) { result = this.getPattern(index) } + } + + /** + * A `case Cls(a, b, x=y)` class pattern. + */ + additional class MatchClassPattern extends Pattern { + private Py::MatchClassPattern clsp; + + MatchClassPattern() { this = TPattern(clsp) } + + /** Gets the class expression of this class pattern. */ + Expr getClass() { result.asExpr() = clsp.getClass() } + + /** Gets the `n`th positional sub-pattern. */ + AstNode getPositional(int n) { result.asPattern() = clsp.getPositional(n) } + + /** Gets the `n`th keyword sub-pattern. */ + AstNode getKeyword(int n) { result.asPattern() = clsp.getKeyword(n) } + + private int numPositional() { result = count(int i | exists(clsp.getPositional(i))) } + + override AstNode getChild(int index) { + index = 0 and result = this.getClass() + or + result = this.getPositional(index - 1) and index >= 1 + or + result = this.getKeyword(index - 1 - this.numPositional()) and + index >= 1 + this.numPositional() + } + } + + /** + * A `case {k: v}` mapping pattern. + */ + additional class MatchMappingPattern extends Pattern { + private Py::MatchMappingPattern mapp; + + MatchMappingPattern() { this = TPattern(mapp) } + + AstNode getMapping(int n) { result.asPattern() = mapp.getMapping(n) } + + override AstNode getChild(int index) { result = this.getMapping(index) } + } + + /** + * A key-value pair inside a `case {k: v}` mapping pattern. + */ + additional class MatchKeyValuePattern extends Pattern { + private Py::MatchKeyValuePattern kvp; + + MatchKeyValuePattern() { this = TPattern(kvp) } + + AstNode getKey() { result.asPattern() = kvp.getKey() } + + AstNode getValue() { result.asPattern() = kvp.getValue() } + + override AstNode getChild(int index) { + index = 0 and result = this.getKey() + or + index = 1 and result = this.getValue() + } + } + + /** + * A `case Cls(name=value)` keyword sub-pattern. + */ + additional class MatchKeywordPattern extends Pattern { + private Py::MatchKeywordPattern kwp; + + MatchKeywordPattern() { this = TPattern(kwp) } + + Expr getAttribute() { result.asExpr() = kwp.getAttribute() } + + AstNode getValue() { result.asPattern() = kwp.getValue() } + + override AstNode getChild(int index) { + index = 0 and result = this.getAttribute() + or + index = 1 and result = this.getValue() + } + } + + /** A `case **rest` double-star mapping sub-pattern. */ + additional class MatchDoubleStarPattern extends Pattern { + private Py::MatchDoubleStarPattern dsp; + + MatchDoubleStarPattern() { this = TPattern(dsp) } + + AstNode getTarget() { result.asPattern() = dsp.getTarget() } + + override AstNode getChild(int index) { index = 0 and result = this.getTarget() } + } + + /** A `case p1 | p2 | …` or-pattern. */ + additional class MatchOrPattern extends Pattern { + private Py::MatchOrPattern orp; + + MatchOrPattern() { this = TPattern(orp) } + + AstNode getPattern(int n) { result.asPattern() = orp.getPattern(n) } + + override AstNode getChild(int index) { result = this.getPattern(index) } + } + + /** A `case 1` literal pattern. */ + additional class MatchLiteralPattern extends Pattern { + private Py::MatchLiteralPattern litp; + + MatchLiteralPattern() { this = TPattern(litp) } + + Expr getLiteral() { result.asExpr() = litp.getLiteral() } + + override AstNode getChild(int index) { index = 0 and result = this.getLiteral() } + } + + /** A `case Cls.NAME` value pattern. */ + additional class MatchValuePattern extends Pattern { + private Py::MatchValuePattern vp; + + MatchValuePattern() { this = TPattern(vp) } + + Expr getValue() { result.asExpr() = vp.getValue() } + + override AstNode getChild(int index) { index = 0 and result = this.getValue() } + } + + /** + * A block statement, modeling the body of a parent AST node as a + * sequence of statements. + */ + class BlockStmt extends Stmt, TBlockStmt { + private Py::StmtList sl; + + BlockStmt() { this = TBlockStmt(sl) } + + /** Gets the `n`th (zero-based) statement in this block. */ + Stmt getStmt(int n) { result.asStmt() = sl.getItem(n) } + + /** Gets the last statement in this block. */ + Stmt getLastStmt() { result.asStmt() = sl.getLastItem() } + + override string toString() { result = sl.toString() } + + // `Py::StmtList` has no native location; approximate with the first + // item's location. + override Py::Location getLocation() { result = sl.getItem(0).getLocation() } + + override Callable getEnclosingCallable() { + result.asScope() = sl.getParent().(Py::Scope) + or + result.asScope() = sl.getParent().(Py::Stmt).getScope() + } + + override AstNode getChild(int index) { result = this.getStmt(index) } + } + + /** An expression statement. */ + class ExprStmt extends Stmt { + private Py::ExprStmt exprStmt; + + ExprStmt() { this = TPyStmt(exprStmt) } + + /** Gets the expression in this expression statement. */ + Expr getExpr() { result.asExpr() = exprStmt.getValue() } + + override AstNode getChild(int index) { index = 0 and result = this.getExpr() } + } + + /** An assignment statement (`x = y = expr`). */ + additional class AssignStmt extends Stmt { + private Py::Assign assign; + + AssignStmt() { this = TPyStmt(assign) } + + Expr getValue() { result.asExpr() = assign.getValue() } + + Expr getTarget(int n) { result.asExpr() = assign.getTarget(n) } + + int getNumberOfTargets() { result = count(assign.getATarget()) } + + override AstNode getChild(int index) { + index = 0 and result = this.getValue() + or + result = this.getTarget(index - 1) and index >= 1 + } + } + + /** An augmented assignment statement (`x += expr`). */ + additional class AugAssignStmt extends Stmt { + private Py::AugAssign augAssign; + + AugAssignStmt() { this = TPyStmt(augAssign) } + + Expr getOperation() { result.asExpr() = augAssign.getOperation() } + + override AstNode getChild(int index) { index = 0 and result = this.getOperation() } + } + + /** + * An annotated assignment statement (`x: T = expr`, or `x: T` without + * value). The evaluation order follows CPython: annotation first, then + * the optional value, then the target binding. + */ + additional class AnnAssignStmt extends Stmt { + private Py::AnnAssign annAssign; + + AnnAssignStmt() { this = TPyStmt(annAssign) } + + Expr getAnnotation() { result.asExpr() = annAssign.getAnnotation() } + + Expr getValue() { result.asExpr() = annAssign.getValue() } + + Expr getTarget() { result.asExpr() = annAssign.getTarget() } + + override AstNode getChild(int index) { + index = 0 and result = this.getAnnotation() + or + index = 1 and result = this.getValue() + or + index = 2 and result = this.getTarget() + } + } + + /** An assignment expression / walrus operator (`x := expr`). */ + additional class NamedExpr extends Expr { + private Py::AssignExpr assignExpr; + + NamedExpr() { this = TPyExpr(assignExpr) } + + Expr getValue() { result.asExpr() = assignExpr.getValue() } + + Expr getTarget() { result.asExpr() = assignExpr.getTarget() } + + override AstNode getChild(int index) { + index = 0 and result = this.getValue() + or + index = 1 and result = this.getTarget() + } + } + + /** + * An `if` statement. + * + * Python's `elif` chains are represented as nested `If` nodes in the + * else branch's `StmtList`. The shared CFG library handles this + * naturally: `getElse()` returns the `BlockStmt` wrapping the else + * branch, and if that block contains a single `If`, the result is + * a chained conditional. + */ + class IfStmt extends Stmt { + private Py::If ifStmt; + + IfStmt() { this = TPyStmt(ifStmt) } + + /** Gets the underlying Python `If` statement. */ + Py::If asIf() { result = ifStmt } + + /** Gets the condition of this `if` statement. */ + Expr getCondition() { result.asExpr() = ifStmt.getTest() } + + /** Gets the `then` (true) branch of this `if` statement. */ + Stmt getThen() { result.asStmtList() = ifStmt.getBody() } + + /** Gets the `else` (false) branch, if any. */ + Stmt getElse() { result.asStmtList() = ifStmt.getOrelse() } + + override AstNode getChild(int index) { + index = 0 and result = this.getCondition() + or + index = 1 and result = this.getThen() + or + index = 2 and result = this.getElse() + } + } + + /** A loop statement. */ + final class LoopStmt = LoopStmtImpl; + + abstract private class LoopStmtImpl extends Stmt { + LoopStmtImpl() { + this = TPyStmt(any(Py::While w)) + or + this = TPyStmt(any(Py::For f)) + } + + /** Gets the body of this loop statement. */ + Stmt getBody() { none() } + } + + /** A `while` loop statement. */ + class WhileStmt extends LoopStmtImpl { + private Py::While whileStmt; + + WhileStmt() { this = TPyStmt(whileStmt) } + + /** Gets the boolean condition of this `while` loop. */ + Expr getCondition() { result.asExpr() = whileStmt.getTest() } + + override Stmt getBody() { result.asStmtList() = whileStmt.getBody() } + + /** Gets the `else` branch of this `while` loop, if any. */ + Stmt getElse() { result.asStmtList() = whileStmt.getOrelse() } + + override AstNode getChild(int index) { + index = 0 and result = this.getCondition() + or + index = 1 and result = this.getBody() + or + index = 2 and result = this.getElse() + } + } + + /** + * A `do-while` loop statement. Python has no do-while construct. + */ + class DoStmt extends LoopStmtImpl { + DoStmt() { none() } + + Expr getCondition() { none() } + } + + /** An `until` loop. Python has no `until` loop. */ + class UntilStmt extends LoopStmtImpl { + UntilStmt() { none() } + + Expr getCondition() { none() } + } + + /** A C-style `for` loop. Python has no C-style for loop. */ + class ForStmt extends LoopStmtImpl { + ForStmt() { none() } + + AstNode getInit(int index) { none() } + + Expr getCondition() { none() } + + AstNode getUpdate(int index) { none() } + } + + /** A for-each loop (`for x in iterable:`). */ + class ForeachStmt extends LoopStmtImpl { + private Py::For forStmt; + + ForeachStmt() { this = TPyStmt(forStmt) } + + /** Gets the loop variable. */ + Expr getVariable() { result.asExpr() = forStmt.getTarget() } + + /** Gets the collection being iterated. */ + Expr getCollection() { result.asExpr() = forStmt.getIter() } + + override Stmt getBody() { result.asStmtList() = forStmt.getBody() } + + /** Gets the `else` branch of this `for` loop, if any. */ + Stmt getElse() { result.asStmtList() = forStmt.getOrelse() } + + override AstNode getChild(int index) { + index = 0 and result = this.getCollection() + or + index = 1 and result = this.getVariable() + or + index = 2 and result = this.getBody() + or + index = 3 and result = this.getElse() + } + } + + /** A `break` statement. */ + class BreakStmt extends Stmt { + BreakStmt() { this = TPyStmt(any(Py::Break b)) } + } + + /** A `continue` statement. */ + class ContinueStmt extends Stmt { + ContinueStmt() { this = TPyStmt(any(Py::Continue c)) } + } + + /** A `goto` statement. Python has no goto. */ + class GotoStmt extends Stmt { + GotoStmt() { none() } + } + + /** A `return` statement. */ + class ReturnStmt extends Stmt { + private Py::Return ret; + + ReturnStmt() { this = TPyStmt(ret) } + + /** Gets the expression being returned, if any. */ + Expr getExpr() { result.asExpr() = ret.getValue() } + + override AstNode getChild(int index) { index = 0 and result = this.getExpr() } + } + + /** A `raise` statement (mapped to `Throw`). */ + class Throw extends Stmt { + private Py::Raise raise; + + Throw() { this = TPyStmt(raise) } + + /** Gets the expression being raised. */ + Expr getExpr() { result.asExpr() = raise.getException() } + + /** Gets the cause of this `raise`, if any. */ + Expr getCause() { result.asExpr() = raise.getCause() } + + override AstNode getChild(int index) { + index = 0 and result = this.getExpr() + or + index = 1 and result = this.getCause() + } + } + + /** + * An `import` statement (`import a, b` or `from m import a, b`). + * + * Each alias contributes two children in evaluation order: first the + * value expression (which performs the import side-effect), then the + * bound `asname` Name (the in-scope binding). This makes both reachable + * from the CFG and allows `Name.defines(v)` for `asname` Names to have + * corresponding CFG nodes — which is essential for SSA to see import + * bindings. + */ + additional class ImportStmt extends Stmt { + private Py::Import imp; + + ImportStmt() { this = TPyStmt(imp) } + + /** Gets the value (module/member expression) of the `n`th alias. */ + Expr getValue(int n) { result.asExpr() = imp.getName(n).getValue() } + + /** Gets the bound `asname` of the `n`th alias. */ + Expr getAsname(int n) { result.asExpr() = imp.getName(n).getAsname() } + + /** Gets the number of aliases in this import statement. */ + int getNumberOfAliases() { result = count(int i | exists(imp.getName(i))) } + + override AstNode getChild(int index) { + exists(int i | + index = 2 * i and result = this.getValue(i) + or + index = 2 * i + 1 and result = this.getAsname(i) + ) + } + } + + /** + * A `from m import *` statement. Evaluates the module expression but + * binds no name (the bindings happen by side-effect at runtime, which + * is not modeled at the CFG level). + */ + additional class ImportStarStmt extends Stmt { + private Py::ImportStar imp; + + ImportStarStmt() { this = TPyStmt(imp) } + + Expr getModule() { result.asExpr() = imp.getModule() } + + override AstNode getChild(int index) { index = 0 and result = this.getModule() } + } + + /** A `with` statement. */ + additional class WithStmt extends Stmt { + private Py::With withStmt; + + WithStmt() { this = TPyStmt(withStmt) } + + Expr getContextExpr() { result.asExpr() = withStmt.getContextExpr() } + + Expr getOptionalVars() { result.asExpr() = withStmt.getOptionalVars() } + + Stmt getBody() { result.asStmtList() = withStmt.getBody() } + + override AstNode getChild(int index) { + index = 0 and result = this.getContextExpr() + or + index = 1 and result = this.getOptionalVars() + or + index = 2 and result = this.getBody() + } + } + + /** An `assert` statement. */ + additional class AssertStmt extends Stmt { + private Py::Assert assertStmt; + + AssertStmt() { this = TPyStmt(assertStmt) } + + Expr getTest() { result.asExpr() = assertStmt.getTest() } + + Expr getMsg() { result.asExpr() = assertStmt.getMsg() } + + override AstNode getChild(int index) { + index = 0 and result = this.getTest() + or + index = 1 and result = this.getMsg() + } + } + + /** A `delete` statement. */ + additional class DeleteStmt extends Stmt { + private Py::Delete del; + + DeleteStmt() { this = TPyStmt(del) } + + Expr getTarget(int n) { result.asExpr() = del.getTarget(n) } + + override AstNode getChild(int index) { result = this.getTarget(index) } + } + + /** + * A PEP 695 `type` statement (`type Alias[T1, T2] = value`). + * + * The type parameters bind at statement-evaluation time. The value + * expression is captured for lazy evaluation but the alias `Name` + * itself binds the resulting `TypeAliasType` object — so the CFG must + * visit at minimum the type-parameter names and the alias name. + */ + additional class TypeAliasStmt extends Stmt { + private Py::TypeAlias ta; + + TypeAliasStmt() { this = TPyStmt(ta) } + + /** Gets the alias `Name` bound by this statement. */ + Expr getName() { result.asExpr() = ta.getName() } + + /** + * Gets the `n`th PEP 695 type-parameter name (a `Name` in store + * context), in declaration order. + */ + Expr getTypeParamName(int n) { result.asExpr() = typeParameterName(ta.getTypeParameter(n)) } + + int getNumberOfTypeParams() { result = count(ta.getATypeParameter()) } + + override AstNode getChild(int index) { + result = this.getTypeParamName(index) + or + index = this.getNumberOfTypeParams() and result = this.getName() + } + } + + /** A `try` statement. */ + class TryStmt extends Stmt { + private Py::Try tryStmt; + + TryStmt() { this = TPyStmt(tryStmt) } + + AstNode getBody(int index) { index = 0 and result.asStmtList() = tryStmt.getBody() } + + /** Gets the `else` branch of this `try` statement, if any. */ + Stmt getElse() { result.asStmtList() = tryStmt.getOrelse() } + + Stmt getFinally() { result.asStmtList() = tryStmt.getFinalbody() } + + CatchClause getCatch(int index) { result.asStmt() = tryStmt.getHandler(index) } + + override AstNode getChild(int index) { + index = 0 and result = this.getBody(0) + or + result = this.getCatch(index - 1) and index >= 1 + or + index = -1 and result = this.getFinally() + or + index = -2 and result = this.getElse() + } + } + + /** + * Gets the `else` branch of `try` statement `try`, if any. + */ + AstNode getTryElse(TryStmt try) { result = try.getElse() } + + /** + * Gets the `else` branch of loop `loop`, if any. + * + * Python's `while`/`for` loops may have an `else` block that runs when the + * loop completes without `break`. + */ + AstNode getLoopElse(LoopStmt loop) { + result = loop.(WhileStmt).getElse() + or + result = loop.(ForeachStmt).getElse() + } + + /** An exception handler (`except` or `except*`). */ + class CatchClause extends Stmt { + private Py::ExceptionHandler handler; + + CatchClause() { this = TPyStmt(handler) } + + /** + * Gets the type-test pattern of this exception handler, if any. + * + * This is the single syntactic type expression (`except TypeError:` → + * `TypeError`; `except (A, B):` → the `(A, B)` tuple). A bare `except:` + * has no pattern and is therefore a catch-all that matches any + * exception. The type test and the variable binding are independent + * child nodes in Python, matching the shared CFG model. + * + * We read the raw type child directly (extractor relation `py_exprs` + * at child index 1) rather than the public `getType()` accessor: the + * latter flattens a tuple of exception types (`except (A, B):`) into + * its individual elements, which would yield several patterns for one + * handler and violate the shared CFG's single-pattern-per-catch + * contract. The raw child is the one tuple node, modeling the + * runtime's single `isinstance(exc, (A, B))` test. + */ + AstNode getPattern() { + exists(Py::Expr rawType | py_exprs(rawType, _, handler, 1) | result.asExpr() = rawType) + } + + /** Gets the variable name of this exception handler, if any. */ + AstNode getVariable() { result.asExpr() = handler.getName() } + + /** Holds: catch clauses do not have a `Condition` in Python's model. */ + Expr getCondition() { none() } + + /** Gets the body of this exception handler. */ + Stmt getBody() { + result.asStmtList() = handler.(Py::ExceptStmt).getBody() + or + result.asStmtList() = handler.(Py::ExceptGroupStmt).getBody() + } + + override AstNode getChild(int index) { + index = 0 and result = this.getPattern() + or + index = 1 and result = this.getVariable() + or + index = 2 and result = this.getBody() + } + } + + /** A `match` statement, mapped to the shared CFG's `Switch`. */ + class Switch extends Stmt { + private Py::MatchStmt matchStmt; + + Switch() { this = TPyStmt(matchStmt) } + + Expr getExpr() { result.asExpr() = matchStmt.getSubject() } + + Case getCase(int index) { result.asStmt() = matchStmt.getCase(index) } + + Stmt getStmt(int index) { none() } + + override AstNode getChild(int index) { + index = 0 and result = this.getExpr() + or + result = this.getCase(index - 1) and index >= 1 + } + } + + /** A `case` clause in a match statement. */ + class Case extends Stmt { + private Py::Case caseStmt; + + Case() { this = TPyStmt(caseStmt) } + + AstNode getPattern(int index) { index = 0 and result.asPattern() = caseStmt.getPattern() } + + Expr getGuard() { result.asExpr() = caseStmt.getGuard().(Py::Guard).getTest() } + + AstNode getBody() { result.asStmtList() = caseStmt.getBody() } + + /** Holds if this case is a wildcard pattern (`case _:`). */ + predicate isWildcard() { caseStmt.getPattern() instanceof Py::MatchWildcardPattern } + + override AstNode getChild(int index) { + index = 0 and result = this.getPattern(0) + or + index = 1 and result = this.getGuard() + or + index = 2 and result = this.getBody() + } + } + + /** A wildcard case (`case _:`). */ + class DefaultCase extends Case { + DefaultCase() { this.isWildcard() } + } + + /** A conditional expression (`x if cond else y`). */ + class ConditionalExpr extends Expr { + private Py::IfExp ifExp; + + ConditionalExpr() { this = TPyExpr(ifExp) } + + /** Gets the condition of this expression. */ + Expr getCondition() { result.asExpr() = ifExp.getTest() } + + /** Gets the true branch of this expression. */ + Expr getThen() { result.asExpr() = ifExp.getBody() } + + /** Gets the false branch of this expression. */ + Expr getElse() { result.asExpr() = ifExp.getOrelse() } + + override AstNode getChild(int index) { + index = 0 and result = this.getCondition() + or + index = 1 and result = this.getThen() + or + index = 2 and result = this.getElse() + } + } + + /** + * A binary expression for the shared CFG. In Python, this covers all + * `and`/`or` expression operand pairs. + */ + class BinaryExpr extends Expr, TBoolExprPair { + private Py::BoolExpr be; + private int index; + + BinaryExpr() { this = TBoolExprPair(be, index) } + + /** Gets the underlying Python `BoolExpr`. */ + Py::BoolExpr getBoolExpr() { result = be } + + /** Gets the (zero-based) index of this pair within its `BoolExpr`. */ + int getIndex() { result = index } + + override string toString() { result = be.getOperator() } + + override Py::Location getLocation() { result = be.getValue(index).getLocation() } + + override Callable getEnclosingCallable() { result.asScope() = be.getScope() } + + /** Gets the left operand of this binary expression. */ + Expr getLeftOperand() { result.asExpr() = be.getValue(index) } + + /** Gets the right operand of this binary expression. */ + Expr getRightOperand() { + // Last pair: right operand is the final value. + index = count(be.getAValue()) - 2 and result.asExpr() = be.getValue(index + 1) + or + // Non-last pair: right operand is the next synthetic pair. + index < count(be.getAValue()) - 2 and + exists(BinaryExpr next | + next.getBoolExpr() = be and next.getIndex() = index + 1 and result = next + ) + } + + override AstNode getChild(int childIndex) { + childIndex = 0 and result = this.getLeftOperand() + or + childIndex = 1 and result = this.getRightOperand() + } + } + + /** A short-circuiting logical `and` expression. */ + class LogicalAndExpr extends BinaryExpr { + LogicalAndExpr() { this.getBoolExpr().getOp() instanceof Py::And } + } + + /** A short-circuiting logical `or` expression. */ + class LogicalOrExpr extends BinaryExpr { + LogicalOrExpr() { this.getBoolExpr().getOp() instanceof Py::Or } + } + + /** A null-coalescing expression. Python has no null-coalescing operator. */ + class NullCoalescingExpr extends BinaryExpr { + NullCoalescingExpr() { none() } + } + + /** + * A unary expression. Currently only used for the `not` subclass. + */ + class UnaryExpr extends Expr { + UnaryExpr() { exists(Py::UnaryExpr u | this = TPyExpr(u) and u.getOp() instanceof Py::Not) } + + /** Gets the operand of this unary expression. */ + Expr getOperand() { result.asExpr() = this.asExpr().(Py::UnaryExpr).getOperand() } + + override AstNode getChild(int index) { index = 0 and result = this.getOperand() } + } + + /** A logical `not` expression. */ + class LogicalNotExpr extends UnaryExpr { } + + /** + * An assignment expression. + * + * Empty in Python: `x = y` and `x += y` are statements (`AssignStmt` and + * `AugAssignStmt`), not expressions, and the walrus `x := y` is modeled + * separately as `NamedExpr`. The shared library's `Assignment` extends + * `BinaryExpr`, so it cannot share instances with our `Stmt`-based + * assignment forms. + */ + class Assignment extends BinaryExpr { + Assignment() { none() } + } + + /** A simple assignment expression. Empty in Python (see `Assignment`). */ + class AssignExpr extends Assignment { } + + /** A compound assignment expression. Empty in Python (see `Assignment`). */ + class CompoundAssignment extends Assignment { } + + /** + * A short-circuiting logical AND compound assignment expression (`&&=`). + * Python has no such operator. + */ + class AssignLogicalAndExpr extends CompoundAssignment { } + + /** + * A short-circuiting logical OR compound assignment expression (`||=`). + * Python has no such operator. + */ + class AssignLogicalOrExpr extends CompoundAssignment { } + + /** + * A short-circuiting null-coalescing compound assignment expression + * (`??=`). Python has no such operator. + */ + class AssignNullCoalescingExpr extends CompoundAssignment { } + + /** A boolean literal expression (`True` or `False`). */ + class BooleanLiteral extends Expr { + BooleanLiteral() { this = TPyExpr(any(Py::True t)) or this = TPyExpr(any(Py::False f)) } + + /** Gets the boolean value of this literal. */ + boolean getValue() { + this.asExpr() instanceof Py::True and result = true + or + this.asExpr() instanceof Py::False and result = false + } + } + + /** A pattern match expression. Python has no `instanceof`-style pattern match expression. */ + class PatternMatchExpr extends Expr { + PatternMatchExpr() { none() } + + Expr getExpr() { none() } + + AstNode getPattern() { none() } + } + + // ===== Python-specific expression classes (used by `getChild`) ===== + /** A Python binary expression (arithmetic, bitwise, matmul, etc.). */ + additional class ArithBinaryExpr extends Expr { + private Py::BinaryExpr binExpr; + + ArithBinaryExpr() { this = TPyExpr(binExpr) } + + Expr getLeft() { result.asExpr() = binExpr.getLeft() } + + Expr getRight() { result.asExpr() = binExpr.getRight() } + + override AstNode getChild(int index) { + index = 0 and result = this.getLeft() + or + index = 1 and result = this.getRight() + } + } + + /** A call expression (`func(args...)`). */ + additional class CallExpr extends Expr { + private Py::Call call; + + CallExpr() { this = TPyExpr(call) } + + Expr getFunc() { result.asExpr() = call.getFunc() } + + Expr getPositionalArg(int n) { result.asExpr() = call.getPositionalArg(n) } + + int getNumberOfPositionalArgs() { result = count(call.getAPositionalArg()) } + + Expr getKeywordValue(int n) { + result.asExpr() = call.getNamedArg(n).(Py::Keyword).getValue() + or + result.asExpr() = call.getNamedArg(n).(Py::DictUnpacking).getValue() + } + + int getNumberOfNamedArgs() { result = count(call.getANamedArg()) } + + override AstNode getChild(int index) { + index = 0 and result = this.getFunc() + or + result = this.getPositionalArg(index - 1) and index >= 1 + or + result = this.getKeywordValue(index - 1 - this.getNumberOfPositionalArgs()) and + index >= 1 + this.getNumberOfPositionalArgs() + } + } + + /** A subscript expression (`obj[index]`). */ + additional class SubscriptExpr extends Expr { + private Py::Subscript sub; + + SubscriptExpr() { this = TPyExpr(sub) } + + Expr getObject() { result.asExpr() = sub.getObject() } + + Expr getIndex() { result.asExpr() = sub.getIndex() } + + override AstNode getChild(int index) { + index = 0 and result = this.getObject() + or + index = 1 and result = this.getIndex() + } + } + + /** An attribute access (`obj.name`). */ + additional class AttributeExpr extends Expr { + private Py::Attribute attr; + + AttributeExpr() { this = TPyExpr(attr) } + + Expr getObject() { result.asExpr() = attr.getObject() } + + override AstNode getChild(int index) { index = 0 and result = this.getObject() } + } + + /** + * An `import x.y` module expression. Modeled as a leaf — the dotted + * name is just a string. + */ + additional class ImportExpression extends Expr { + ImportExpression() { this.asExpr() instanceof Py::ImportExpr } + } + + /** + * A `from m import x` member access. The module sub-expression is a + * child so that the CFG visits both the module load and this + * attribute selection. + */ + additional class ImportMemberExpr extends Expr { + private Py::ImportMember im; + + ImportMemberExpr() { this = TPyExpr(im) } + + /** Gets the module expression `m` in `from m import x`. */ + Expr getModule() { result.asExpr() = im.getModule() } + + override AstNode getChild(int index) { index = 0 and result = this.getModule() } + } + + /** A tuple literal. */ + additional class TupleExpr extends Expr { + private Py::Tuple tuple; + + TupleExpr() { this = TPyExpr(tuple) } + + Expr getElt(int n) { result.asExpr() = tuple.getElt(n) } + + override AstNode getChild(int index) { result = this.getElt(index) } + } + + /** A list literal. */ + additional class ListExpr extends Expr { + private Py::List list; + + ListExpr() { this = TPyExpr(list) } + + Expr getElt(int n) { result.asExpr() = list.getElt(n) } + + override AstNode getChild(int index) { result = this.getElt(index) } + } + + /** A set literal. */ + additional class SetExpr extends Expr { + private Py::Set set; + + SetExpr() { this = TPyExpr(set) } + + Expr getElt(int n) { result.asExpr() = set.getElt(n) } + + override AstNode getChild(int index) { result = this.getElt(index) } + } + + /** A dict literal. */ + additional class DictExpr extends Expr { + private Py::Dict dict; + + DictExpr() { this = TPyExpr(dict) } + + /** + * Gets the key of the `n`th item (at child index `2*n`); the value is + * at child index `2*n + 1`. + */ + Expr getKey(int n) { result.asExpr() = dict.getItem(n).(Py::KeyValuePair).getKey() } + + Expr getValue(int n) { result.asExpr() = dict.getItem(n).(Py::KeyValuePair).getValue() } + + int getNumberOfItems() { result = count(dict.getAnItem()) } + + override AstNode getChild(int index) { + exists(int item | + index = 2 * item and result = this.getKey(item) + or + index = 2 * item + 1 and result = this.getValue(item) + ) + } + } + + /** A unary expression other than `not` (e.g., `-x`, `+x`, `~x`). */ + additional class ArithUnaryExpr extends Expr { + private Py::UnaryExpr unaryExpr; + + ArithUnaryExpr() { this = TPyExpr(unaryExpr) and not unaryExpr.getOp() instanceof Py::Not } + + Expr getOperand() { result.asExpr() = unaryExpr.getOperand() } + + override AstNode getChild(int index) { index = 0 and result = this.getOperand() } + } + + /** + * A comprehension or generator expression. The iterable is evaluated in + * the enclosing scope; the body runs in a nested synthetic function + * scope handled by its own CFG. + */ + additional class Comprehension extends Expr { + private Py::Expr iterable; + + Comprehension() { + exists(Py::Expr c | this = TPyExpr(c) | + iterable = c.(Py::ListComp).getIterable() + or + iterable = c.(Py::SetComp).getIterable() + or + iterable = c.(Py::DictComp).getIterable() + or + iterable = c.(Py::GeneratorExp).getIterable() + ) + } + + Expr getIterable() { result.asExpr() = iterable } + + override AstNode getChild(int index) { index = 0 and result = this.getIterable() } + } + + /** A comparison expression (`a < b`, `a < b < c`, etc.). */ + additional class CompareExpr extends Expr { + private Py::Compare cmp; + + CompareExpr() { this = TPyExpr(cmp) } + + Expr getLeft() { result.asExpr() = cmp.getLeft() } + + Expr getComparator(int n) { result.asExpr() = cmp.getComparator(n) } + + override AstNode getChild(int index) { + index = 0 and result = this.getLeft() + or + result = this.getComparator(index - 1) and index >= 1 + } + } + + /** A slice expression (`start:stop:step`). */ + additional class SliceExpr extends Expr { + private Py::Slice slice; + + SliceExpr() { this = TPyExpr(slice) } + + Expr getStart() { result.asExpr() = slice.getStart() } + + Expr getStop() { result.asExpr() = slice.getStop() } + + Expr getStep() { result.asExpr() = slice.getStep() } + + override AstNode getChild(int index) { + index = 0 and result = this.getStart() + or + index = 1 and result = this.getStop() + or + index = 2 and result = this.getStep() + } + } + + /** A starred expression (`*x`). */ + additional class StarredExpr extends Expr { + private Py::Starred starred; + + StarredExpr() { this = TPyExpr(starred) } + + Expr getValue() { result.asExpr() = starred.getValue() } + + override AstNode getChild(int index) { index = 0 and result = this.getValue() } + } + + /** A formatted string literal (`f"...{expr}..."`). */ + additional class FstringExpr extends Expr { + private Py::Fstring fstring; + + FstringExpr() { this = TPyExpr(fstring) } + + Expr getValue(int n) { result.asExpr() = fstring.getValue(n) } + + override AstNode getChild(int index) { result = this.getValue(index) } + } + + /** A formatted value inside an f-string (`{expr}` or `{expr:spec}`). */ + additional class FormattedValueExpr extends Expr { + private Py::FormattedValue fv; + + FormattedValueExpr() { this = TPyExpr(fv) } + + Expr getValue() { result.asExpr() = fv.getValue() } + + Expr getFormatSpec() { result.asExpr() = fv.getFormatSpec() } + + override AstNode getChild(int index) { + index = 0 and result = this.getValue() + or + index = 1 and result = this.getFormatSpec() + } + } + + /** A `yield` expression. */ + additional class YieldExpr extends Expr { + private Py::Yield yield; + + YieldExpr() { this = TPyExpr(yield) } + + Expr getValue() { result.asExpr() = yield.getValue() } + + override AstNode getChild(int index) { index = 0 and result = this.getValue() } + } + + /** A `yield from` expression. */ + additional class YieldFromExpr extends Expr { + private Py::YieldFrom yieldFrom; + + YieldFromExpr() { this = TPyExpr(yieldFrom) } + + Expr getValue() { result.asExpr() = yieldFrom.getValue() } + + override AstNode getChild(int index) { index = 0 and result = this.getValue() } + } + + /** An `await` expression. */ + additional class AwaitExpr extends Expr { + private Py::Await await; + + AwaitExpr() { this = TPyExpr(await) } + + Expr getValue() { result.asExpr() = await.getValue() } + + override AstNode getChild(int index) { index = 0 and result = this.getValue() } + } + + /** + * A class definition expression (visits bases, but NOT PEP 695 type + * parameters — those bind in an annotation scope that nests the class + * body, so they belong to the inner scope's CFG, not the enclosing + * scope's; the legacy CFG also omitted them). + */ + additional class ClassDefExpr extends Expr { + private Py::ClassExpr classExpr; + + ClassDefExpr() { this = TPyExpr(classExpr) } + + Expr getBase(int n) { result.asExpr() = classExpr.getBase(n) } + + override AstNode getChild(int index) { result = this.getBase(index) } + } + + /** + * A function definition expression (visits positional and keyword + * defaults followed by parameter and return type annotations, but NOT + * PEP 695 type parameters — those bind in an annotation scope that + * nests the function body, so they belong to the inner scope's CFG, + * not the enclosing scope's; the legacy CFG also omitted them). + * + * Evaluation order follows CPython: defaults are pushed first, then + * keyword-only defaults, then annotations (the `__annotations__` dict + * is built last, before `MAKE_FUNCTION`). Annotations are emitted as + * CFG nodes so that flows from a class reference into a parameter's + * type annotation are visible to dataflow (e.g. so that framework + * models like FastAPI's `Depends()` can use a parameter's type hint + * to track that the parameter receives an instance of the annotated + * class — see `LocalSources::annotatedInstance`). + */ + additional class FunctionDefExpr extends Expr { + private Py::FunctionExpr funcExpr; + + FunctionDefExpr() { this = TPyExpr(funcExpr) } + + /** + * Gets the `n`th default for a positional argument, in evaluation + * order. Note that `Args.getDefault(int)` is indexed by argument + * position (with gaps for arguments without defaults), so we must + * renumber here to obtain contiguous indices. + */ + Expr getDefault(int n) { + result.asExpr() = + rank[n + 1](Py::Expr d, int i | d = funcExpr.getArgs().getDefault(i) | d order by i) + } + + /** Gets the `n`th default for a keyword-only argument, in evaluation order. */ + Expr getKwDefault(int n) { + result.asExpr() = + rank[n + 1](Py::Expr d, int i | d = funcExpr.getArgs().getKwDefault(i) | d order by i) + } + + /** + * Gets the `n`th annotation expression, in CPython evaluation + * order: positional parameter annotations (by argument position), + * `*args` annotation, keyword-only parameter annotations (by + * argument position), `**kwargs` annotation, then the return + * annotation. Each annotation appears at most once. + */ + Expr getAnnotation(int n) { + result.asExpr() = + rank[n + 1](Py::Expr a, int subOrder, int subIndex | + functionAnnotation(funcExpr, a, subOrder, subIndex) + | + a order by subOrder, subIndex + ) + } + + int getNumberOfDefaults() { result = count(funcExpr.getArgs().getADefault()) } + + int getNumberOfKwDefaults() { result = count(funcExpr.getArgs().getAKwDefault()) } + + int getNumberOfAnnotations() { + result = count(Py::Expr a | functionAnnotation(funcExpr, a, _, _)) + } + + override AstNode getChild(int index) { + result = this.getDefault(index) + or + result = this.getKwDefault(index - this.getNumberOfDefaults()) + or + result = this.getAnnotation(index - this.getNumberOfDefaults() - this.getNumberOfKwDefaults()) + } + } + + /** + * Holds if `a` is an annotation of `funcExpr` in slot + * `(subOrder, subIndex)`. Slots are CPython evaluation order: + * positional param annotations (subOrder 0, subIndex = argument + * position), `*args` annotation (1, 0), keyword-only annotations + * (2, position), `**kwargs` annotation (3, 0), return annotation + * (4, 0). + */ + private predicate functionAnnotation( + Py::FunctionExpr funcExpr, Py::Expr a, int subOrder, int subIndex + ) { + a = funcExpr.getArgs().getAnnotation(subIndex) and subOrder = 0 + or + a = funcExpr.getArgs().getVarargannotation() and subOrder = 1 and subIndex = 0 + or + a = funcExpr.getArgs().getKwAnnotation(subIndex) and subOrder = 2 + or + a = funcExpr.getArgs().getKwargannotation() and subOrder = 3 and subIndex = 0 + or + a = funcExpr.getReturns() and subOrder = 4 and subIndex = 0 + } + + /** A lambda expression (has default args evaluated at definition time). */ + additional class LambdaExpr extends Expr { + private Py::Lambda lambda; + + LambdaExpr() { this = TPyExpr(lambda) } + + /** Gets the `n`th default for a positional argument, in evaluation order. */ + Expr getDefault(int n) { + result.asExpr() = + rank[n + 1](Py::Expr d, int i | d = lambda.getArgs().getDefault(i) | d order by i) + } + + /** Gets the `n`th default for a keyword-only argument, in evaluation order. */ + Expr getKwDefault(int n) { + result.asExpr() = + rank[n + 1](Py::Expr d, int i | d = lambda.getArgs().getKwDefault(i) | d order by i) + } + + int getNumberOfDefaults() { result = count(lambda.getArgs().getADefault()) } + + override AstNode getChild(int index) { + result = this.getDefault(index) + or + result = this.getKwDefault(index - this.getNumberOfDefaults()) + } + } + + /** Gets the child of `n` at the specified (zero-based) index. */ + AstNode getChild(AstNode n, int index) { result = n.getChild(index) } +} + +private module Cfg0 = Make0; + +private import Cfg0 + +private module Cfg1 = Make1; + +private import Cfg1 + +private module Cfg2 = Make2; + +private import Cfg2 + +private module Input implements InputSig1, InputSig2 { + predicate cfgCachedStageRef() { CfgCachedStage::ref() } + + private newtype TLabel = TNone() + + class Label extends TLabel { + string toString() { result = "label" } + } + + class CallableContext = Void; + + predicate inConditionalContext(Ast::AstNode n, ConditionKind kind) { + kind.isBoolean() and + n = any(Ast::AssertStmt a).getTest() + } + + private string assertThrowTag() { result = "[assert-throw]" } + + /** + * Holds if the expression node `e` may raise an exception at runtime as part of + * its normal evaluation (not via an explicit `raise`/`assert`, which are + * modeled separately). + * + * The set mirrors what the legacy CFG used to flag implicitly: function + * calls (anything can raise), attribute access (`AttributeError`), + * subscript access (`IndexError`/`KeyError`/`TypeError`), arithmetic and + * comparison operators (`TypeError`/`ZeroDivisionError`), imports + * (`ImportError`/`ModuleNotFoundError`), and generator/coroutine + * suspension points (`await`/`yield`/`yield from`). + * + * Bare `Name` reads are intentionally excluded — modeling every name + * read as `mayThrow` would explode CFG edge count for negligible + * analysis value. `BoolExpr`/`IfExp` containers are also excluded; the + * operands they evaluate contribute their own exception edges. + */ + private predicate exprMayThrow(Py::Expr e) { + e instanceof Py::Call + or + e instanceof Py::Attribute + or + e instanceof Py::Subscript + or + e instanceof Py::BinaryExpr + or + e instanceof Py::UnaryExpr + or + e instanceof Py::Compare + or + e instanceof Py::ImportExpr + or + e instanceof Py::ImportMember + or + e instanceof Py::Await + or + e instanceof Py::Yield + or + e instanceof Py::YieldFrom + } + + /** + * Holds if the statement `s` may raise an exception at runtime as part + * of its normal evaluation. Currently restricted to `from m import *` + * (which performs the import as a statement-level side effect). + */ + private predicate stmtMayThrow(Py::Stmt s) { s instanceof Py::ImportStar } + + /** + * Holds if `py` is syntactically inside the body, handlers, `else`, or + * `finally` of a `try` statement (or the body of a `with` statement, + * which compiles to an implicit try/finally for `__exit__`) in the + * same scope. + * + * This mirrors Java's `ControlFlowGraph::mayThrow`, which only emits + * exception edges when there is local exception handling that would + * observe them. Outside such contexts, exception edges would add CFG + * complexity (weakening BarrierGuard precision and breaking SSA + * continuity around augmented assignments and subscript stores) + * without any analysis benefit, since exceptions just propagate to + * the function exit anyway. + */ + private predicate inExceptionContext(Py::AstNode py) { + exists(Py::Try t | t.containsInScope(py)) + or + exists(Py::With w | w.containsInScope(py)) + } + + /** + * Holds if `n` may raise an exception during normal evaluation. See + * `exprMayThrow` and `stmtMayThrow` for the included AST classes. + * + * Restricted to nodes inside a `try`/`with` statement: matches Java's + * approach of only modeling exception flow where it can be observed + * by local handling. + */ + private predicate mayThrow(Ast::AstNode n) { + exists(Py::AstNode py | py = n.asExpr() or py = n.asStmt() | + (exprMayThrow(py) or stmtMayThrow(py)) and + inExceptionContext(py) + ) + } + + predicate additionalNode(Ast::AstNode n, string tag, NormalSuccessor t) { + n instanceof Ast::AssertStmt and tag = assertThrowTag() and t instanceof DirectSuccessor + } + + predicate beginAbruptCompletion( + Ast::AstNode ast, PreControlFlowNode n, AbruptCompletion c, boolean always + ) { + ast instanceof Ast::AssertStmt and + n.isAdditional(ast, assertThrowTag()) and + c.asSimpleAbruptCompletion() instanceof ExceptionSuccessor and + always = true + or + mayThrow(ast) and + n.isIn(ast) and + c.asSimpleAbruptCompletion() instanceof ExceptionSuccessor and + always = false + } + + predicate endAbruptCompletion(Ast::AstNode ast, PreControlFlowNode n, AbruptCompletion c) { + none() + } + + predicate step(PreControlFlowNode n1, PreControlFlowNode n2) { + exists(Ast::AssertStmt assertStmt | + n1.isBefore(assertStmt) and + n2.isBefore(assertStmt.getTest()) + or + n1.isAfterTrue(assertStmt.getTest()) and + n2.isAfter(assertStmt) + or + n1.isAfterFalse(assertStmt.getTest()) and + ( + n2.isBefore(assertStmt.getMsg()) + or + not exists(assertStmt.getMsg()) and + n2.isAdditional(assertStmt, assertThrowTag()) + ) + or + n1.isAfter(assertStmt.getMsg()) and + n2.isAdditional(assertStmt, assertThrowTag()) + ) + } +} + +import Public + +/** + * Maps a CFG AST wrapper node to the corresponding Python AST node, if any. + * Entry, exit, and synthetic nodes have no corresponding Python AST node. + */ +Py::AstNode astNodeToPyNode(Ast::AstNode n) { + result = n.asExpr() + or + result = n.asStmt() + or + result = n.asScope() + or + result = n.asPattern() +} diff --git a/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll b/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll new file mode 100644 index 00000000000..3441ad0203c --- /dev/null +++ b/python/ql/lib/semmle/python/controlflow/internal/Cfg.qll @@ -0,0 +1,926 @@ +/** + * Provides a Python control flow graph facade backed by the shared + * `codeql.controlflow.ControlFlowGraph` library (via `AstNodeImpl.qll`). + * + * This module re-exposes the same API surface as `semmle/python/Flow.qll` + * (the legacy CFG), but is implemented on the new shared CFG. It is + * intended as a drop-in replacement for use by the Python dataflow library + * and other downstream code. + * + * Layering follows the Java pattern (`java/ql/lib/semmle/code/java/Expr.qll` + * and `SsaImpl.qll`): variable identity and similar AST-level semantics + * live on the Python AST classes (`Name.defines(v)`, `Name.uses(v)`, ...); + * the CFG layer is purely positional, with `toAst` / `getNode` bridging + * back to the AST. The shared SSA library can then be parameterized on + * (`BasicBlock`, `int`) directly, with no CFG-level variable predicates. + */ +overlay[local?] +module; + +private import python as Py +private import semmle.python.controlflow.internal.AstNodeImpl as CfgImpl +private import codeql.controlflow.SuccessorType + +/** + * A control flow node. + * + * This is the full set of CFG nodes from the shared library — it includes + * before-nodes, in-order/post-order nodes, after-value-split nodes, and + * entry/exit nodes. This enables full control-flow-level reasoning and + * compatibility with the shared control-flow reachability library. + * + * AST-level semantics (`getNode()`, `isLoad()`, typed wrappers, etc.) + * are available only on the `injects` (canonical) node for each AST node. + * Non-injects nodes are purely positional CFG nodes with no AST mapping. + */ +class ControlFlowNode extends CfgImpl::ControlFlowNode { + /** Gets the syntactic element corresponding to this flow node, if any. */ + Py::AstNode getNode() { + exists(CfgImpl::Ast::AstNode n | this.injects(n) | result = CfgImpl::astNodeToPyNode(n)) + } + + Py::Expr asPyExpr() { result = this.getNode() } + + /** Gets a predecessor of this flow node. */ + ControlFlowNode getAPredecessor() { this = result.getASuccessor() } + + /** Gets a successor of this flow node. */ + ControlFlowNode getASuccessor() { result = super.getASuccessor() } + + /** Gets a successor for this node if the relevant condition is True. */ + ControlFlowNode getATrueSuccessor() { + result = super.getASuccessor(any(BooleanSuccessor t | t.getValue() = true)) + } + + /** Gets a successor for this node if the relevant condition is False. */ + ControlFlowNode getAFalseSuccessor() { + result = super.getASuccessor(any(BooleanSuccessor t | t.getValue() = false)) + } + + /** Gets a successor for this node if an exception is raised. */ + ControlFlowNode getAnExceptionalSuccessor() { result = super.getAnExceptionSuccessor() } + + /** Gets a successor for this node if no exception is raised. */ + ControlFlowNode getANormalSuccessor() { result = super.getANormalSuccessor() } + + /** Gets the basic block containing this flow node. */ + BasicBlock getBasicBlock() { result = super.getBasicBlock() } + + /** Gets the scope containing this flow node. */ + Py::Scope getScope() { result = super.getEnclosingCallable().asScope() } + + /** Gets the enclosing module. */ + Py::Module getEnclosingModule() { result = this.getScope().getEnclosingModule() } + + /** Gets the immediate dominator of this flow node. */ + ControlFlowNode getImmediateDominator() { + // Defined positionally via the basic-block dominance tree. + exists(BasicBlock bb, int i | bb.getNode(i) = this | + // Predecessor within the same basic block. + i > 0 and result = bb.getNode(i - 1) + or + // First node of `bb`: dominator is the last node of the immediate dominator block. + i = 0 and result = bb.getImmediateDominator().getLastNode() + ) + } + + /** Holds if this strictly dominates `other`. */ + overlay[caller?] + pragma[inline] + predicate strictlyDominates(ControlFlowNode other) { super.strictlyDominates(other) } + + /** Holds if this dominates `other` (reflexively). */ + overlay[caller?] + pragma[inline] + predicate dominates(ControlFlowNode other) { super.dominates(other) } + + /** Holds if this is the first node in its enclosing scope. */ + predicate isEntryNode() { this instanceof CfgImpl::ControlFlow::EntryNode } + + /** Holds if this is the first node of a module. */ + predicate isModuleEntry() { + this.isEntryNode() and super.getAstNode().asScope() instanceof Py::Module + } + + /** Holds if this node may exit its scope by raising an exception. */ + predicate isExceptionalExit(Py::Scope s) { + this instanceof CfgImpl::ControlFlow::ExceptionalExitNode and + super.getEnclosingCallable().asScope() = s + } + + /** Holds if this node is a normal (non-exceptional) exit. */ + predicate isNormalExit() { this instanceof CfgImpl::ControlFlow::NormalExitNode } + + // ===== AST-shape predicates (bridges to the wrapped Python AST) ===== + /** + * Holds if this flow node is a load (including those in augmented + * assignments). + * + * Note: an augmented-assignment target (`x[i]` in `x[i] += 1`) is + * both a load and a store — `isLoad` and `isStore` both hold on the + * canonical CFG node. This mirrors Java's `VarAccess.isVarRead`, + * which holds on the destination of compound and unary assignments + * even though the destination is also a write. + */ + predicate isLoad() { py_expr_contexts(_, 3, this.asPyExpr()) } + + /** Holds if this flow node is a store (including those in augmented assignments). */ + predicate isStore() { py_expr_contexts(_, 5, this.asPyExpr()) or augstore(_, this) } + + /** Holds if this flow node is a delete. */ + predicate isDelete() { py_expr_contexts(_, 2, this.asPyExpr()) } + + /** Holds if this flow node is a parameter. */ + predicate isParameter() { py_expr_contexts(_, 4, this.asPyExpr()) } + + /** Holds if this flow node is a store in an augmented assignment. */ + predicate isAugStore() { augstore(_, this) } + + /** Holds if this flow node is a load in an augmented assignment. */ + predicate isAugLoad() { augstore(this, _) } + + /** Holds if this flow node corresponds to a literal. */ + predicate isLiteral() { + this.getNode() instanceof Py::Bytes or + this.getNode() instanceof Py::Dict or + this.getNode() instanceof Py::DictComp or + this.getNode() instanceof Py::Set or + this.getNode() instanceof Py::SetComp or + this.getNode() instanceof Py::Ellipsis or + this.getNode() instanceof Py::GeneratorExp or + this.getNode() instanceof Py::Lambda or + this.getNode() instanceof Py::ListComp or + this.getNode() instanceof Py::List or + this.getNode() instanceof Py::Num or + this.getNode() instanceof Py::Tuple or + this.getNode() instanceof Py::Unicode or + this.getNode() instanceof Py::NameConstant + } + + /** Holds if this flow node corresponds to an attribute expression. */ + predicate isAttribute() { this.getNode() instanceof Py::Attribute } + + /** Holds if this flow node corresponds to a subscript expression. */ + predicate isSubscript() { this.getNode() instanceof Py::Subscript } + + /** Holds if this flow node corresponds to an import member. */ + predicate isImportMember() { this.getNode() instanceof Py::ImportMember } + + /** Holds if this flow node corresponds to a call. */ + predicate isCall() { this.getNode() instanceof Py::Call } + + /** Holds if this flow node corresponds to an import. */ + predicate isImport() { this.getNode() instanceof Py::ImportExpr } + + /** Holds if this flow node corresponds to a conditional expression. */ + predicate isIfExp() { this.getNode() instanceof Py::IfExp } + + /** Holds if this flow node corresponds to a function definition expression. */ + predicate isFunction() { this.getNode() instanceof Py::FunctionExpr } + + /** Holds if this flow node corresponds to a class definition expression. */ + predicate isClass() { this.getNode() instanceof Py::ClassExpr } + + /** + * Holds if this flow node is a branch (i.e. has both a true and a + * false successor). + */ + predicate isBranch() { exists(this.getATrueSuccessor()) or exists(this.getAFalseSuccessor()) } + + /** + * Gets a CFG child of this node, defined as a CFG node whose AST node + * is a child of this CFG node's AST node, restricted to nodes that + * dominate this one (so the child has been evaluated by the time we + * reach this node). + * + * Mirrors `Flow.qll`'s `getAChild`. UnaryExprNode is excluded because + * its operand is its CFG predecessor (handled separately). + */ + pragma[nomagic] + ControlFlowNode getAChild() { + this.getNode().(Py::Expr).getAChildNode() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) and + not this instanceof UnaryExprNode + } + + /** Holds if this flow node strictly reaches `other`. */ + predicate strictlyReaches(ControlFlowNode other) { this.getASuccessor+() = other } +} + +/** + * Holds if `load` is the load half of an augmented-assignment target, + * and `store` is the corresponding store half. + * + * In the legacy CFG (`Flow.qll`) the same Python `Name` had two + * distinct CFG nodes — a load node (context 3) earlier in the BB, and + * a store node (context 5) later. The legacy `augstore` related the + * pair via dominance. + * + * In the new (shared) CFG, the canonical node for an AST expression is + * unique, so `load` and `store` collapse onto the same CFG node. The + * predicate is therefore reflexive on the augmented-assignment + * target's canonical node. + */ +private predicate augstore(ControlFlowNode load, ControlFlowNode store) { + exists(Py::AugAssign aa | aa.getTarget() = load.getNode()) and + load = store +} + +/** + * A basic block — a maximal-length sequence of control flow nodes such + * that no node except the first has a predecessor outside the sequence, + * and no node except the last has a successor outside the sequence. + */ +class BasicBlock extends CfgImpl::BasicBlock { + /** Gets the `n`th node in this basic block. */ + ControlFlowNode getNode(int n) { result = super.getNode(n) } + + /** Gets a node in this basic block. */ + ControlFlowNode getANode() { result = super.getNode(_) } + + /** Gets the first node in this basic block. */ + ControlFlowNode firstNode() { result = this.getNode(0) } + + /** Gets the last node in this basic block. */ + ControlFlowNode getLastNode() { result = super.getLastNode() } + + /** Holds if this basic block contains `node`. */ + predicate contains(ControlFlowNode node) { node = this.getANode() } + + // Inherited from the shared library's `BasicBlock`: + // getASuccessor(), getASuccessor(SuccessorType), getAPredecessor(), + // strictlyDominates(), dominates(), getImmediateDominator(), + // length(), inLoop(). + // We shadow `getNode(int)` etc. to return `ControlFlowNode` (this + // facade's type) and add Python-style helpers below. + /** Gets a true successor to this basic block. */ + BasicBlock getATrueSuccessor() { + result = super.getASuccessor(any(BooleanSuccessor t | t.getValue() = true)) + } + + /** Gets a false successor to this basic block. */ + BasicBlock getAFalseSuccessor() { + result = super.getASuccessor(any(BooleanSuccessor t | t.getValue() = false)) + } + + /** Gets an unconditional successor to this basic block. */ + BasicBlock getAnUnconditionalSuccessor() { + result = super.getASuccessor() and + not result = this.getATrueSuccessor() and + not result = this.getAFalseSuccessor() + } + + /** Gets an exceptional successor to this basic block. */ + BasicBlock getAnExceptionalSuccessor() { result = super.getASuccessor(any(ExceptionSuccessor t)) } + + /** + * Holds if this basic block is in the dominance frontier of `df`. + * + * Note: implemented locally rather than via the shared lib, which + * doesn't currently expose a `dominanceFrontier` predicate at this + * level. + */ + predicate inDominanceFrontier(BasicBlock df) { super.inDominanceFrontier(df) } + + /** Holds if this basic block strictly reaches `other`. */ + predicate strictlyReaches(BasicBlock other) { super.getASuccessor+() = other } + + /** Holds if this basic block reaches `other` (reflexively). */ + predicate reaches(BasicBlock other) { this = other or this.strictlyReaches(other) } + + /** Holds if flow from this basic block reaches a normal exit from its scope. */ + predicate reachesExit() { + this.getANode() instanceof CfgImpl::ControlFlow::NormalExitNode + or + exists(BasicBlock succ | succ = super.getASuccessor() and succ.reachesExit()) + } + + /** Gets the scope of this basic block. */ + Py::Scope getScope() { exists(ControlFlowNode n | n = this.getANode() | result = n.getScope()) } + + /** Holds if flow from this BasicBlock always reaches `succ`. */ + predicate alwaysReaches(BasicBlock succ) { + succ = this + or + strictcount(BasicBlock s | s = super.getASuccessor()) = 1 and + succ = super.getASuccessor() + or + forex(BasicBlock immsucc | immsucc = super.getASuccessor() | immsucc.alwaysReaches(succ)) + } + + /** + * Holds if this basic block ends in a node that branches on a boolean + * outcome, and `other` is dominated by the corresponding successor + * for `branch` while not being reachable from the other branch + * without going through this BB. + * + * In other words: any execution that reaches `other` must have just + * evaluated the last node of this BB and taken the `branch` outcome. + * This mirrors the legacy `ConditionBlock.controls(BB, branch)`. + */ + predicate controls(BasicBlock other, boolean branch) { + super.edgeDominates(other, any(BooleanSuccessor t | t.getValue() = branch)) + } +} + +// =========================================================================== +// Re-exports for SSA / dominance consumers +// +// The shared `BB::CfgSig` requires `EntryBasicBlock` and `dominatingEdge` in +// addition to the BasicBlock class we already expose. They are provided by +// the shared CFG library on the `BB::Make` instantiation produced by +// `AstNodeImpl.qll`. +// =========================================================================== +/** An entry basic block, that is, a basic block whose first node is an entry node. */ +class EntryBasicBlock = CfgImpl::Cfg::EntryBasicBlock; + +/** + * Holds if `bb1` has `bb2` as a direct successor and the edge between `bb1` + * and `bb2` is a dominating edge. + */ +predicate dominatingEdge = CfgImpl::Cfg::dominatingEdge/2; + +// =========================================================================== +// AST-shape subclasses of ControlFlowNode +// +// Each class is a thin wrapper around the canonical CFG node for a given +// kind of Python AST node. Methods that take/return CFG nodes look up +// related CFG nodes by AST identity (via `getNode()`), and the dominance +// constraint from the old CFG (`result.getBasicBlock().dominates(this.getBasicBlock())`) +// is preserved. +// =========================================================================== +/** Gets the canonical `ControlFlowNode` for AST expression `e`. */ +ControlFlowNode astExprToCfg(Py::Expr e) { result.getNode() = e } + +/** A control flow node corresponding to a `Name` or `PlaceHolder` expression. */ +class NameNode extends ControlFlowNode { + NameNode() { + this.getNode() instanceof Py::Name + or + this.getNode() instanceof Py::PlaceHolder + } + + /** + * Holds if this flow node defines the variable `v`. + * + * This includes augmented-assignment targets — `n += 1` is both a + * read and a write of `n`, so `defines(n)` and `uses(n)` both hold + * on the same canonical CFG node. Mirrors Java's `VariableUpdate` + * semantics where compound assignments register both a write + * (`VarWrite`) and a read (`VarRead`) on the destination. + */ + predicate defines(Py::Variable v) { exists(Py::Name n | n = this.getNode() and n.defines(v)) } + + /** Holds if this flow node deletes the variable `v`. */ + predicate deletes(Py::Variable v) { exists(Py::Name n | n = this.getNode() and n.deletes(v)) } + + /** Holds if this flow node uses the variable `v`. */ + predicate uses(Py::Variable v) { + this.isLoad() and + exists(Py::Name u | u = this.getNode() and u.uses(v)) + or + exists(Py::PlaceHolder u | + u = this.getNode() and u.getVariable() = v and u.getCtx() instanceof Py::Load + ) + } + + /** Gets the identifier of this name node. */ + string getId() { + result = this.getNode().(Py::Name).getId() + or + result = this.getNode().(Py::PlaceHolder).getId() + } + + /** Holds if this is a use of a local variable. */ + predicate isLocal() { exists(Py::Variable v | this.uses(v) and v instanceof Py::LocalVariable) } + + /** Holds if this is a use of a non-local variable. */ + predicate isNonLocal() { + exists(Py::Variable v | this.uses(v) and v.getScope() != this.getScope()) + } + + /** Holds if this is a use of a global (including builtin) variable. */ + predicate isGlobal() { exists(Py::Variable v | this.uses(v) and v instanceof Py::GlobalVariable) } + + /** + * Holds if this is a use of `self` — the first parameter of an + * enclosing method. + * + * AST-level approximation: matches when the Name uses a `Variable` + * that is the first parameter of an enclosing `Function` defined + * inside a `Class`. + */ + predicate isSelf() { + exists(Py::Variable v, Py::Function f, Py::Class c | + this.uses(v) and + f = c.getAMethod() and + v.getScope() = f and + v = f.getArg(0).(Py::Name).getVariable() + ) + } +} + +/** A control flow node corresponding to a named constant (`None`, `True`, `False`). */ +class NameConstantNode extends NameNode { + NameConstantNode() { this.getNode() instanceof Py::NameConstant } +} + +/** A control flow node corresponding to a call. */ +class CallNode extends ControlFlowNode { + CallNode() { super.getNode() instanceof Py::Call } + + override Py::Call getNode() { result = super.getNode() } + + /** Gets the underlying Python `Call`. */ + Py::Call getCall() { result = this.getNode() } + + /** Gets the flow node for the function component of this call. */ + ControlFlowNode getFunction() { + this.getCall().getFunc() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + } + + /** Gets the flow node for the `n`th positional argument. */ + ControlFlowNode getArg(int n) { + this.getCall().getArg(n) = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + } + + /** Gets the flow node for the named argument with name `name`. */ + ControlFlowNode getArgByName(string name) { + exists(Py::Keyword k | + k = this.getCall().getANamedArg() and + k.getValue() = result.getNode() and + k.getArg() = name and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } + + /** Gets a flow node corresponding to any argument. */ + ControlFlowNode getAnArg() { result = this.getArg(_) or result = this.getArgByName(_) } + + /** Gets the first tuple (`*args`) argument, if any. */ + ControlFlowNode getStarArg() { + this.getCall().getStarArg() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + } + + /** Gets a dictionary (`**kwargs`) argument, if any. */ + ControlFlowNode getKwargs() { + this.getCall().getKwargs() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + } + + /** Holds if this call is a decorator call applied to a class or a function. */ + predicate isDecoratorCall() { this.isClassDecoratorCall() or this.isFunctionDecoratorCall() } + + /** Holds if this call is a decorator call applied to a class. */ + predicate isClassDecoratorCall() { + exists(Py::ClassExpr cls | this.getNode() = cls.getADecoratorCall()) + } + + /** Holds if this call is a decorator call applied to a function. */ + predicate isFunctionDecoratorCall() { + exists(Py::FunctionExpr func | this.getNode() = func.getADecoratorCall()) + } +} + +/** A control flow node corresponding to an attribute expression. */ +class AttrNode extends ControlFlowNode { + AttrNode() { super.getNode() instanceof Py::Attribute } + + override Py::Attribute getNode() { result = super.getNode() } + + /** Gets the flow node for the object of the attribute expression. */ + ControlFlowNode getObject() { + this.getNode().getObject() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + } + + /** Gets the flow node for the object of this attribute expression, with the matching name. */ + ControlFlowNode getObject(string name) { + this.getName() = name and + result = this.getObject() + } + + /** Gets the attribute name. */ + string getName() { result = this.getNode().getName() } +} + +/** A control flow node corresponding to an import statement (`import x`). */ +class ImportExprNode extends ControlFlowNode { + ImportExprNode() { super.getNode() instanceof Py::ImportExpr } + + override Py::ImportExpr getNode() { result = super.getNode() } +} + +/** A control flow node corresponding to a `from ... import name` expression. */ +class ImportMemberNode extends ControlFlowNode { + ImportMemberNode() { super.getNode() instanceof Py::ImportMember } + + override Py::ImportMember getNode() { result = super.getNode() } + + /** Gets the flow node for the module being imported from, with the matching name. */ + ControlFlowNode getModule(string name) { + exists(Py::ImportMember i | + i = this.getNode() and + i.getModule() = result.getNode() and + i.getName() = name and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } +} + +/** A control flow node corresponding to a `from ... import *` statement. */ +class ImportStarNode extends ControlFlowNode { + ImportStarNode() { super.getNode() instanceof Py::ImportStar } + + override Py::ImportStar getNode() { result = super.getNode() } + + /** Gets the flow node for the module being imported from. */ + ControlFlowNode getModule() { + this.getNode().getModuleExpr() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + } +} + +/** A control flow node corresponding to a subscript expression. */ +class SubscriptNode extends ControlFlowNode { + SubscriptNode() { super.getNode() instanceof Py::Subscript } + + override Py::Subscript getNode() { result = super.getNode() } + + /** Gets the flow node for the value being subscripted. */ + ControlFlowNode getObject() { + this.getNode().getObject() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + } + + /** Gets the flow node for the index expression. */ + ControlFlowNode getIndex() { + this.getNode().getIndex() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + } +} + +/** A control flow node corresponding to a comparison operation. */ +class CompareNode extends ControlFlowNode { + CompareNode() { super.getNode() instanceof Py::Compare } + + override Py::Compare getNode() { result = super.getNode() } + + /** Holds if `left` and `right` are a pair of operands for this comparison. */ + predicate operands(ControlFlowNode left, Py::Cmpop op, ControlFlowNode right) { + exists(Py::Compare c, Py::Expr eleft, Py::Expr eright | + c = this.getNode() and eleft = left.getNode() and eright = right.getNode() + | + eleft = c.getLeft() and eright = c.getComparator(0) and op = c.getOp(0) + or + exists(int i | + eleft = c.getComparator(i - 1) and eright = c.getComparator(i) and op = c.getOp(i) + ) + ) and + left.getBasicBlock().dominates(this.getBasicBlock()) and + right.getBasicBlock().dominates(this.getBasicBlock()) + } +} + +/** A control flow node corresponding to a conditional expression (`x if c else y`). */ +class IfExprNode extends ControlFlowNode { + IfExprNode() { super.getNode() instanceof Py::IfExp } + + override Py::IfExp getNode() { result = super.getNode() } + + /** Gets the flow node for one of the value operands (true-branch or false-branch). */ + ControlFlowNode getAnOperand() { + exists(Py::IfExp ie | + ie = this.getNode() and + (result.getNode() = ie.getBody() or result.getNode() = ie.getOrelse()) + ) + } +} + +/** A control flow node corresponding to an assignment expression (walrus `:=`). */ +class AssignmentExprNode extends ControlFlowNode { + AssignmentExprNode() { super.getNode() instanceof Py::AssignExpr } + + override Py::AssignExpr getNode() { result = super.getNode() } + + /** Gets the flow node for the left-hand side. */ + ControlFlowNode getTarget() { + this.getNode().getTarget() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + } + + /** Gets the flow node for the right-hand side. */ + ControlFlowNode getValue() { + this.getNode().getValue() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + } +} + +/** A control flow node corresponding to a binary expression (`a + b` etc.). */ +class BinaryExprNode extends ControlFlowNode { + BinaryExprNode() { super.getNode() instanceof Py::BinaryExpr } + + override Py::BinaryExpr getNode() { result = super.getNode() } + + ControlFlowNode getLeft() { + this.getNode().getLeft() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + } + + ControlFlowNode getRight() { + this.getNode().getRight() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + } + + Py::Operator getOp() { result = this.getNode().(Py::BinaryExpr).getOp() } + + /** Holds if `left` and `right` are the operands and `op` is the operator. */ + predicate operands(ControlFlowNode left, Py::Operator op, ControlFlowNode right) { + left = this.getLeft() and right = this.getRight() and op = this.getOp() + } + + /** Gets either operand. */ + ControlFlowNode getAnOperand() { result = this.getLeft() or result = this.getRight() } +} + +/** A control flow node corresponding to a boolean expression (`a and b`, `a or b`). */ +class BoolExprNode extends ControlFlowNode { + BoolExprNode() { super.getNode() instanceof Py::BoolExpr } + + override Py::BoolExpr getNode() { result = super.getNode() } + + Py::Boolop getOp() { result = this.getNode().(Py::BoolExpr).getOp() } + + /** Gets any operand of this boolean expression. */ + ControlFlowNode getAnOperand() { this.getNode().getAValue() = result.getNode() } +} + +/** A control flow node corresponding to a unary expression (`-x`, `not x`, etc.). */ +class UnaryExprNode extends ControlFlowNode { + UnaryExprNode() { super.getNode() instanceof Py::UnaryExpr } + + override Py::UnaryExpr getNode() { result = super.getNode() } + + ControlFlowNode getOperand() { + this.getNode().getOperand() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + } + + Py::Unaryop getOp() { result = this.getNode().(Py::UnaryExpr).getOp() } +} + +/** + * A control flow node that is a definition: it appears in a context that + * binds a variable (assignment target, parameter, etc.). + */ +class DefinitionNode extends ControlFlowNode { + DefinitionNode() { this.isStore() or this.isParameter() } + + /** Gets the value assigned, if any. */ + ControlFlowNode getValue() { + // For-target: the value is the for-loop's iter expression (which + // is also where `Cfg::ForNode` lives — its `getNode()` returns the + // enclosing `Py::For` statement). Treated specially because there + // is no AST node holding the result of `iter(next(seq))`; we use + // the iter expression's CFG node as the stand-in. + exists(Py::For f | + f.getTarget() = this.getNode() and + result.getNode() = f.getIter() + ) + or + exists(Py::AstNode value | value = assignedValue(this.getNode()) | + result.getNode() = value and + ( + result.getBasicBlock().dominates(this.getBasicBlock()) + or + result.isImport() + or + // The default value for a parameter is evaluated in the same basic block as + // the function definition, but the parameter belongs to the basic block of the + // function, so there is no dominance relationship between the two. + exists(Py::Parameter param | this.getNode() = param.asName()) + ) + ) + } +} + +/** + * Gets the AST node that holds the value assigned to `lhs` in a binding + * context. Mirrors `Flow.qll::assigned_value`. + */ +private Py::AstNode assignedValue(Py::Expr lhs) { + // lhs = result + exists(Py::Assign a | a.getATarget() = lhs and result = a.getValue()) + or + // lhs := result + exists(Py::AssignExpr a | a.getTarget() = lhs and result = a.getValue()) + or + // lhs: annotation = result + exists(Py::AnnAssign a | a.getTarget() = lhs and result = a.getValue()) + or + // import result as lhs (also covers plain `import lhs`, where alias.getAsname() = lhs) + exists(Py::Alias a | a.getAsname() = lhs and result = a.getValue()) + or + // lhs += x -> result is the (lhs + x) binary expression + exists(Py::AugAssign a, Py::BinaryExpr b | + b = a.getOperation() and result = b and lhs = b.getLeft() + ) + or + // Nested sequence assign: ..., lhs, ... = ..., result, ... + exists(Py::Assign a | nestedSequenceAssign(a.getATarget(), a.getValue(), lhs, result)) + or + // Parameter default + exists(Py::Parameter param | lhs = param.asName() and result = param.getDefault()) +} + +/** + * Helper for nested sequence assignments such as `(a, b), c = (1, 2), 3`. + */ +private predicate nestedSequenceAssign( + Py::Expr leftParent, Py::Expr rightParent, Py::Expr left, Py::Expr right +) { + exists(int i | + leftParent.(Py::Tuple).getElt(i) = left and rightParent.(Py::Tuple).getElt(i) = right + or + leftParent.(Py::List).getElt(i) = left and rightParent.(Py::List).getElt(i) = right + ) + or + exists(Py::Expr leftMid, Py::Expr rightMid | + nestedSequenceAssign(leftParent, rightParent, leftMid, rightMid) and + nestedSequenceAssign(leftMid, rightMid, left, right) + ) +} + +/** A control flow node corresponding to a deletion (`del x`). */ +class DeletionNode extends ControlFlowNode { + DeletionNode() { this.isDelete() } +} + +/** A control flow node corresponding to a `for` loop target. */ +class ForNode extends ControlFlowNode { + ForNode() { exists(Py::For f | this.getNode() = f.getIter()) } + + /** Gets the iterable expression. */ + ControlFlowNode getIter() { + result = this and result = result // canonical "after" of the iterable + } + + /** Gets the sequence expression (alias for `getIter()`, matches legacy Flow naming). */ + ControlFlowNode getSequence() { result = this.getIter() } + + /** Gets the target (loop variable) of the `for` loop. */ + ControlFlowNode getTarget() { + exists(Py::For f | + f.getIter() = this.getNode() and + f.getTarget() = result.getNode() + ) + } + + /** Holds if `target` is the loop variable and `sequence` is the iterable. */ + predicate iterates(ControlFlowNode target, ControlFlowNode sequence) { + target = this.getTarget() and sequence = this.getSequence() + } +} + +/** A control flow node corresponding to a `raise` statement. */ +class RaiseStmtNode extends ControlFlowNode { + RaiseStmtNode() { super.getNode() instanceof Py::Raise } + + override Py::Raise getNode() { result = super.getNode() } + + /** Gets the exception expression, if any. */ + ControlFlowNode getException() { + this.getNode().getException() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + } +} + +/** A control flow node corresponding to a starred expression (`*x`). */ +class StarredNode extends ControlFlowNode { + StarredNode() { this.getNode() instanceof Py::Starred } + + /** Gets the value being starred. */ + ControlFlowNode getValue() { + exists(Py::Starred s | + s = this.getNode() and + s.getValue() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } +} + +/** A control flow node corresponding to an `except` clause's name binding. */ +class ExceptFlowNode extends ControlFlowNode { + ExceptFlowNode() { exists(Py::ExceptStmt e | this.getNode() = e.getName()) } + + /** Gets the CFG node for the bound `as`-name itself. */ + ControlFlowNode getName() { result = this } + + /** Gets the type expression of this exception handler. */ + ControlFlowNode getType() { + exists(Py::ExceptStmt e | + e.getName() = this.getNode() and + e.getType() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } +} + +/** A control flow node corresponding to an `except*` clause's name binding. */ +class ExceptGroupFlowNode extends ControlFlowNode { + ExceptGroupFlowNode() { exists(Py::ExceptGroupStmt e | this.getNode() = e.getName()) } + + /** Gets the CFG node for the bound `as`-name itself. */ + ControlFlowNode getName() { result = this } +} + +/** A control flow node corresponding to a sequence (tuple, list). */ +abstract class SequenceNode extends ControlFlowNode { + /** Gets the `n`th element of this sequence. */ + abstract ControlFlowNode getElement(int n); + + /** Gets any element of this sequence. */ + ControlFlowNode getAnElement() { result = this.getElement(_) } +} + +/** A control flow node corresponding to a tuple literal. */ +class TupleNode extends SequenceNode { + TupleNode() { this.getNode() instanceof Py::Tuple } + + override ControlFlowNode getElement(int n) { + exists(Py::Tuple t | + t = this.getNode() and + t.getElt(n) = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } +} + +/** A control flow node corresponding to a list literal. */ +class ListNode extends SequenceNode { + ListNode() { this.getNode() instanceof Py::List } + + override ControlFlowNode getElement(int n) { + exists(Py::List l | + l = this.getNode() and + l.getElt(n) = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } +} + +/** A control flow node corresponding to a set literal. */ +class SetNode extends ControlFlowNode { + SetNode() { this.getNode() instanceof Py::Set } + + /** Gets the flow node for an element of the set. */ + ControlFlowNode getAnElement() { + exists(Py::Set s | + s = this.getNode() and + s.getAnElt() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } +} + +/** A control flow node corresponding to a dict literal. */ +class DictNode extends ControlFlowNode { + DictNode() { this.getNode() instanceof Py::Dict } + + /** Gets the flow node for a key of the dict. */ + ControlFlowNode getAKey() { + exists(Py::Dict d | + d = this.getNode() and + d.getAKey() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } + + /** Gets the flow node for a value of the dict. */ + ControlFlowNode getAValue() { + exists(Py::Dict d | + d = this.getNode() and + d.getAValue() = result.getNode() and + result.getBasicBlock().dominates(this.getBasicBlock()) + ) + } +} + +/** A control flow node corresponding to an iterable in a `for` loop. */ +class IterableNode extends ControlFlowNode { + IterableNode() { + this instanceof SequenceNode + or + this instanceof SetNode + } + + /** Gets the control flow node for an element of this iterable. */ + ControlFlowNode getAnElement() { + result = this.(SequenceNode).getAnElement() + or + result = this.(SetNode).getAnElement() + } +} diff --git a/python/ql/test/extractor-tests/syntax_error/CONSISTENCY/CfgConsistency.expected b/python/ql/test/extractor-tests/syntax_error/CONSISTENCY/CfgConsistency.expected new file mode 100644 index 00000000000..91a01a3a3d9 --- /dev/null +++ b/python/ql/test/extractor-tests/syntax_error/CONSISTENCY/CfgConsistency.expected @@ -0,0 +1,4 @@ +consistencyOverview +| deadEnd | 1 | +deadEnd +| without_loop.py:7:5:7:9 | Break | diff --git a/python/ql/test/library-tests/ControlFlow/bindings/BindingsTest.expected b/python/ql/test/library-tests/ControlFlow/bindings/BindingsTest.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/bindings/BindingsTest.ql b/python/ql/test/library-tests/ControlFlow/bindings/BindingsTest.ql new file mode 100644 index 00000000000..a507878911b --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/BindingsTest.ql @@ -0,0 +1,32 @@ +/** + * Phase -1 of the dataflow CFG migration: verifies that every variable + * binding visible to the AST (`Name.defines(v)`) corresponds to a CFG node + * in the new CFG (`semmle.python.controlflow.internal.AstNodeImpl`). + * + * The expected tag is `cfgdefines=`. Each binding annotation in the + * test sources looks like `# $ cfgdefines=x` for a binding currently + * covered by the new CFG, or `# $ MISSING: cfgdefines=x` for a binding + * that is known to be uncovered (a "red" test case that should be + * green-flipped once the corresponding `cfg-ext-*` extension lands). + */ + +import python +import semmle.python.controlflow.internal.AstNodeImpl as CfgImpl +import utils.test.InlineExpectationsTest + +module CfgBindingsTest implements TestSig { + string getARelevantTag() { result = "cfgdefines" } + + predicate hasActualResult(Location location, string element, string tag, string value) { + exists(Name n, Variable v, CfgImpl::ControlFlowNode cfg | + n.defines(v) and + cfg.getAstNode().asExpr() = n and + location = n.getLocation() and + element = n.toString() and + tag = "cfgdefines" and + value = v.getId() + ) + } +} + +import MakeTest diff --git a/python/ql/test/library-tests/ControlFlow/bindings/annassign.py b/python/ql/test/library-tests/ControlFlow/bindings/annassign.py new file mode 100644 index 00000000000..7a9ae3ab6c7 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/annassign.py @@ -0,0 +1,13 @@ +# Annotated assignment (PEP 526). Both with and without an initializer. + +a: int = 1 # $ cfgdefines=a +b: str = "hi" # $ cfgdefines=b + +# Annotation without value: the AST records `c` as defined, +# and the new CFG now visits it via the AnnAssignStmt wrapper. +c: int # $ cfgdefines=c + +class K: # $ cfgdefines=K + field: int = 0 # $ cfgdefines=field + + diff --git a/python/ql/test/library-tests/ControlFlow/bindings/compound.py b/python/ql/test/library-tests/ControlFlow/bindings/compound.py new file mode 100644 index 00000000000..cb2f36f12ff --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/compound.py @@ -0,0 +1,14 @@ +# Compound (tuple/list) assignment targets — actually wired in the new CFG. + +a, b = (1, 2) # $ cfgdefines=a cfgdefines=b +[c, d] = [3, 4] # $ cfgdefines=c cfgdefines=d + +# Nested unpacking. +(e, (f, g)) = (1, (2, 3)) # $ cfgdefines=e cfgdefines=f cfgdefines=g + +# Star unpacking. +h, *i = [1, 2, 3] # $ cfgdefines=h cfgdefines=i + +# Chained assignment with compound target. +j = k, l = (5, 6) # $ cfgdefines=j cfgdefines=k cfgdefines=l + diff --git a/python/ql/test/library-tests/ControlFlow/bindings/comprehension.py b/python/ql/test/library-tests/ControlFlow/bindings/comprehension.py new file mode 100644 index 00000000000..6b5f722c1f7 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/comprehension.py @@ -0,0 +1,21 @@ +# Comprehension and `for` loop targets — wired in the new CFG. +# Comprehensions are nested function scopes with a synthetic `.0` parameter +# bound to the iterable. + +# Bare-name `for` target. +for i in range(3): # $ cfgdefines=i + pass + +# Compound `for` target. +for k, v in [(1, 2)]: # $ cfgdefines=k cfgdefines=v + pass + +# Comprehension targets. +_ = [x for x in range(3)] # $ cfgdefines=_ cfgdefines=x cfgdefines=.0 +_ = {y: z for y, z in []} # $ cfgdefines=_ cfgdefines=y cfgdefines=z cfgdefines=.0 +_ = (a for a in []) # $ cfgdefines=_ cfgdefines=a cfgdefines=.0 + +# Nested comprehensions. +_ = [b for c in [] for b in c] # $ cfgdefines=_ cfgdefines=c cfgdefines=b cfgdefines=.0 + + diff --git a/python/ql/test/library-tests/ControlFlow/bindings/dead_under_no_raise.py b/python/ql/test/library-tests/ControlFlow/bindings/dead_under_no_raise.py new file mode 100644 index 00000000000..9058f2b7116 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/dead_under_no_raise.py @@ -0,0 +1,53 @@ +# Reachability of code following a try whose body always returns. +# +# The new CFG models exception edges for raise-prone expressions when +# they appear inside a `try` (or `with`) statement, mirroring Java's +# `mayThrow`. This means the body of a `try` has both a normal +# completion edge and an exception edge to its handlers, so code +# following the try-statement is reachable via the except-handler path +# even when the try-body would otherwise always return. +# +# Code that is not reachable under either normal or exception flow +# (for example, the `else` clause of a try whose body unconditionally +# raises) remains correctly classified as dead. + + +def f(obj): # $ cfgdefines=f cfgdefines=obj + try: + return len(obj) + except TypeError: + pass + + # The try-body always returns, but `len(obj)` can raise (it is + # inside the try, so we model its exception edge). The + # `except TypeError: pass` handler falls through to here, making + # the code below reachable. + try: + hint = type(obj).__length_hint__ # $ cfgdefines=hint + except AttributeError: + return None + return hint + + +def g(): # $ cfgdefines=g + try: + raise Exception("inner") + except: + raise Exception("outer") + else: + # Unreachable: the inner try body always raises (via an explicit + # `raise`, which is modelled unconditionally), so the `else:` + # clause never runs. + hit_inner_else = True + + +def h(cache, key): # $ cfgdefines=h cfgdefines=cache cfgdefines=key + try: + return cache[key] + except KeyError: + pass + + # Same pattern as `f`: reachable via the except-handler fall-through. + value = compute(key) # $ cfgdefines=value + cache[key] = value + return value diff --git a/python/ql/test/library-tests/ControlFlow/bindings/decorated.py b/python/ql/test/library-tests/ControlFlow/bindings/decorated.py new file mode 100644 index 00000000000..9b93c166ace --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/decorated.py @@ -0,0 +1,30 @@ +# Decorated `def`/`class` — wired in the new CFG. + + +def deco(f): # $ cfgdefines=deco cfgdefines=f + return f + + +@deco +def decorated_func(): # $ cfgdefines=decorated_func + pass + + +@deco +class DecoratedClass: # $ cfgdefines=DecoratedClass + pass + + +# Stacked decorators. +@deco +@deco +def doubly(): # $ cfgdefines=doubly + pass + + +# Inside a class body. +class Outer: # $ cfgdefines=Outer + @staticmethod + def inner(): # $ cfgdefines=inner + pass + diff --git a/python/ql/test/library-tests/ControlFlow/bindings/except_handler.py b/python/ql/test/library-tests/ControlFlow/bindings/except_handler.py new file mode 100644 index 00000000000..57b6c99fe9b --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/except_handler.py @@ -0,0 +1,19 @@ +# Exception-handler name bindings. These are already wired in the new +# CFG provided the try body can raise; `raise` statements are reliably +# treated as exception sources. + +try: + raise ValueError("oops") +except ValueError as e: # $ cfgdefines=e + pass + +try: + raise TypeError("oops") +except (TypeError, KeyError) as err: # $ cfgdefines=err + pass + +# Exception groups (Python 3.11+). +try: + raise ValueError("oops") +except* ValueError as eg: # $ cfgdefines=eg + pass diff --git a/python/ql/test/library-tests/ControlFlow/bindings/imports.py b/python/ql/test/library-tests/ControlFlow/bindings/imports.py new file mode 100644 index 00000000000..c8834b5332a --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/imports.py @@ -0,0 +1,14 @@ +# Import aliases — all bound names below are now reachable via the new +# CFG's `ImportStmt` wrapper. + +import os # $ cfgdefines=os +import os.path # $ cfgdefines=os +import os as o # $ cfgdefines=o +from os import path # $ cfgdefines=path +from os import path as p # $ cfgdefines=p +from os import sep, linesep # $ cfgdefines=sep cfgdefines=linesep +from os import ( + getcwd, # $ cfgdefines=getcwd + getcwdb, # $ cfgdefines=getcwdb +) + diff --git a/python/ql/test/library-tests/ControlFlow/bindings/match_pattern.py b/python/ql/test/library-tests/ControlFlow/bindings/match_pattern.py new file mode 100644 index 00000000000..0868a2680d0 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/match_pattern.py @@ -0,0 +1,24 @@ +# Match-statement pattern bindings — wired in the new CFG. + +def f(subject): # $ cfgdefines=f cfgdefines=subject + match subject: + case x: # $ cfgdefines=x + pass + case [a, b]: # $ cfgdefines=a cfgdefines=b + pass + case {"k": v}: # $ cfgdefines=v + pass + case Point(p, q): # $ cfgdefines=p cfgdefines=q + pass + case [_, *rest]: # $ cfgdefines=rest + pass + case (1 | 2) as n: # $ cfgdefines=n + pass + + +class Point: # $ cfgdefines=Point + __match_args__ = ("x", "y") # $ cfgdefines=__match_args__ + x: int # $ cfgdefines=x + y: int # $ cfgdefines=y + + diff --git a/python/ql/test/library-tests/ControlFlow/bindings/parameters.py b/python/ql/test/library-tests/ControlFlow/bindings/parameters.py new file mode 100644 index 00000000000..7fe5e01e4c4 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/parameters.py @@ -0,0 +1,42 @@ +# Function parameters. + +def positional(a, b): # $ cfgdefines=positional cfgdefines=a cfgdefines=b + pass + + +def with_default(x=1, y=2): # $ cfgdefines=with_default cfgdefines=x cfgdefines=y + pass + + +def with_vararg(*args): # $ cfgdefines=with_vararg cfgdefines=args + pass + + +def with_kwarg(**kwargs): # $ cfgdefines=with_kwarg cfgdefines=kwargs + pass + + +def with_kwonly(*, k1, k2=5): # $ cfgdefines=with_kwonly cfgdefines=k1 cfgdefines=k2 + pass + + +def kitchen_sink(a, b=2, *args, k1, k2=5, **kw): # $ cfgdefines=kitchen_sink cfgdefines=a cfgdefines=b cfgdefines=args cfgdefines=k1 cfgdefines=k2 cfgdefines=kw + pass + + +# Methods get `self` / `cls`. +class C: # $ cfgdefines=C + def method(self, x): # $ cfgdefines=method cfgdefines=self cfgdefines=x + pass + + @classmethod + def cmethod(cls, x): # $ cfgdefines=cmethod cfgdefines=cls cfgdefines=x + pass + + +# Lambda parameter. +_ = lambda p: p + 1 # $ cfgdefines=_ cfgdefines=p + +# PEP 570 positional-only. +def pos_only(a, b, /, c): # $ cfgdefines=pos_only cfgdefines=a cfgdefines=b cfgdefines=c + pass diff --git a/python/ql/test/library-tests/ControlFlow/bindings/simple.py b/python/ql/test/library-tests/ControlFlow/bindings/simple.py new file mode 100644 index 00000000000..51cb7d828c9 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/simple.py @@ -0,0 +1,14 @@ +# Simple bindings that should already work in the new CFG. +# No MISSING annotations expected. + +x = 1 # $ cfgdefines=x +y = x + 1 # $ cfgdefines=y + +def f(): # $ cfgdefines=f + pass + +class C: # $ cfgdefines=C + pass + +# Re-assignment. +x = 2 # $ cfgdefines=x diff --git a/python/ql/test/library-tests/ControlFlow/bindings/starred.py b/python/ql/test/library-tests/ControlFlow/bindings/starred.py new file mode 100644 index 00000000000..ac38c74ef12 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/starred.py @@ -0,0 +1,4 @@ +# Starred-target edge cases — wired in the new CFG. + +# Starred target in a Tuple LHS. +*head, tail = [1, 2, 3] # $ cfgdefines=head cfgdefines=tail diff --git a/python/ql/test/library-tests/ControlFlow/bindings/type_params.py b/python/ql/test/library-tests/ControlFlow/bindings/type_params.py new file mode 100644 index 00000000000..ae48ccf130b --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/type_params.py @@ -0,0 +1,21 @@ +# PEP 695 type parameters (Python 3.12+). + +# PEP 695 type-param names on `def`/`class` bind in an annotation scope +# that nests the function/class body — they have no CFG node in the +# enclosing scope (matching the legacy CFG). +def func[T](x: T) -> T: # $ cfgdefines=func cfgdefines=x + return x + + +class Box[T]: # $ cfgdefines=Box + item: T # $ SPURIOUS: cfgdefines=item + + +# Multi-parameter, with bound and variadics. +def multi[T: int, *Ts, **P](x: T, *args: *Ts, **kwargs: P.kwargs) -> T: # $ cfgdefines=multi cfgdefines=x cfgdefines=args cfgdefines=kwargs + return x + + +# `type` statement (PEP 695). +type Alias[T] = list[T] # $ cfgdefines=Alias cfgdefines=T + diff --git a/python/ql/test/library-tests/ControlFlow/bindings/walrus.py b/python/ql/test/library-tests/ControlFlow/bindings/walrus.py new file mode 100644 index 00000000000..d370fd342dc --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/walrus.py @@ -0,0 +1,9 @@ +# Walrus edge cases — wired in the new CFG. + +# Walrus in expression context. +if (y := 5) > 0: # $ cfgdefines=y + pass + +# Walrus in a comprehension. The comprehension introduces a synthetic +# `.0` parameter bound to the iterable. +_ = [w for _ in range(3) if (w := 1)] # $ cfgdefines=_ cfgdefines=w cfgdefines=.0 diff --git a/python/ql/test/library-tests/ControlFlow/bindings/with_stmt.py b/python/ql/test/library-tests/ControlFlow/bindings/with_stmt.py new file mode 100644 index 00000000000..5fffe46c5d4 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/bindings/with_stmt.py @@ -0,0 +1,21 @@ +# `with cm() as x:` bindings — wired in the new CFG. + +class CM: # $ cfgdefines=CM + def __enter__(self): return self # $ cfgdefines=__enter__ cfgdefines=self + def __exit__(self, *a): pass # $ cfgdefines=__exit__ cfgdefines=self cfgdefines=a + +with CM() as x: # $ cfgdefines=x + pass + +# Multiple items. +with CM() as a, CM() as b: # $ cfgdefines=a cfgdefines=b + pass + +# Parenthesised form (Python 3.10+). +with (CM() as p, CM() as q): # $ cfgdefines=p cfgdefines=q + pass + +# Compound target in `with`. +with CM() as (m, n): # $ cfgdefines=m cfgdefines=n + pass + diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutivePredecessorTimestamps.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutivePredecessorTimestamps.expected new file mode 100644 index 00000000000..9dc28649061 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutivePredecessorTimestamps.expected @@ -0,0 +1,13 @@ +| test_boolean.py:9:10:9:43 | BoolExpr | $@ in $@ has no consecutive predecessor (expected 1) | test_boolean.py:9:59:9:59 | IntegerLiteral | Timestamp 2 | test_boolean.py:7:1:7:27 | Function test_and_both_sides | test_and_both_sides | +| test_boolean.py:15:10:15:43 | BoolExpr | $@ in $@ has no consecutive predecessor (expected 0) | test_boolean.py:15:50:15:50 | IntegerLiteral | Timestamp 1 | test_boolean.py:13:1:13:30 | Function test_and_short_circuit | test_and_short_circuit | +| test_boolean.py:21:10:21:42 | BoolExpr | $@ in $@ has no consecutive predecessor (expected 0) | test_boolean.py:21:49:21:49 | IntegerLiteral | Timestamp 1 | test_boolean.py:19:1:19:29 | Function test_or_short_circuit | test_or_short_circuit | +| test_boolean.py:27:10:27:34 | BoolExpr | $@ in $@ has no consecutive predecessor (expected 1) | test_boolean.py:27:50:27:50 | IntegerLiteral | Timestamp 2 | test_boolean.py:25:1:25:26 | Function test_or_both_sides | test_or_both_sides | +| test_boolean.py:40:10:40:61 | BoolExpr | $@ in $@ has no consecutive predecessor (expected 2) | test_boolean.py:40:86:40:86 | IntegerLiteral | Timestamp 3 | test_boolean.py:38:1:38:24 | Function test_chained_and | test_chained_and | +| test_boolean.py:46:10:46:61 | BoolExpr | $@ in $@ has no consecutive predecessor (expected 2) | test_boolean.py:46:86:46:86 | IntegerLiteral | Timestamp 3 | test_boolean.py:44:1:44:23 | Function test_chained_or | test_chained_or | +| test_boolean.py:52:10:52:95 | BoolExpr | $@ in $@ has no consecutive predecessor (expected 3) | test_boolean.py:52:120:52:120 | IntegerLiteral | Timestamp 4 | test_boolean.py:50:1:50:25 | Function test_mixed_and_or | test_mixed_and_or | +| test_boolean.py:52:11:52:47 | BoolExpr | $@ in $@ has no consecutive predecessor (expected 1) | test_boolean.py:52:63:52:63 | IntegerLiteral | Timestamp 2 | test_boolean.py:50:1:50:25 | Function test_mixed_and_or | test_mixed_and_or | +| test_boolean.py:52:78:52:79 | IntegerLiteral | $@ in $@ has no consecutive predecessor (expected 2) | test_boolean.py:52:85:52:85 | IntegerLiteral | Timestamp 3 | test_boolean.py:50:1:50:25 | Function test_mixed_and_or | test_mixed_and_or | +| test_comparison.py:37:6:37:41 | Compare | $@ in $@ has no consecutive predecessor (expected 1) | test_comparison.py:37:48:37:48 | IntegerLiteral | Timestamp 2 | test_comparison.py:32:1:32:32 | Function test_three_short_circuit | test_three_short_circuit | +| test_if.py:96:9:96:9 | x | $@ in $@ has no consecutive predecessor (expected 1) | test_if.py:96:15:96:15 | IntegerLiteral | Timestamp 2 | test_if.py:93:1:93:34 | Function test_if_compound_condition | test_if_compound_condition | +| test_if.py:96:9:96:29 | BoolExpr | $@ in $@ has no consecutive predecessor (expected 3) | test_if.py:96:36:96:36 | IntegerLiteral | Timestamp 4 | test_if.py:93:1:93:34 | Function test_if_compound_condition | test_if_compound_condition | +| test_if.py:99:13:99:13 | IntegerLiteral | $@ in $@ has no consecutive predecessor (expected 4) | test_if.py:99:19:99:19 | IntegerLiteral | Timestamp 5 | test_if.py:93:1:93:34 | Function test_if_compound_condition | test_if_compound_condition | diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutivePredecessorTimestamps.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutivePredecessorTimestamps.ql new file mode 100644 index 00000000000..6d90605d685 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutivePredecessorTimestamps.ql @@ -0,0 +1,22 @@ +/** + * Checks that each annotated node (except the minimum timestamp) has a + * predecessor annotation with timestamp `a - 1`. This is the reverse of + * ConsecutiveTimestamps: it catches nodes that are reachable but arrived + * at from the wrong place (skipping an intermediate node). + * + * Only applies to functions where all annotations are in the function's + * own scope (excludes tests with generators, async, comprehensions, or + * lambdas that have annotations in nested scopes). + */ + +import OldCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from TimerAnnotation ann, int a +where consecutivePredecessorTimestamps(ann, a) +select ann, "$@ in $@ has no consecutive predecessor (expected " + (a - 1) + ")", + ann.getTimestampExpr(a), "Timestamp " + a, ann.getTestFunction(), ann.getTestFunction().getName() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutiveTimestamps.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutiveTimestamps.expected index ed22c971ecb..c354396586e 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutiveTimestamps.expected +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/ConsecutiveTimestamps.expected @@ -7,6 +7,7 @@ | test_boolean.py:52:11:52:47 | BoolExpr | $@ in $@ has no consecutive successor (expected 3) | test_boolean.py:52:63:52:63 | IntegerLiteral | Timestamp 2 | test_boolean.py:50:1:50:25 | Function test_mixed_and_or | test_mixed_and_or | | test_boolean.py:52:27:52:31 | False | $@ in $@ has no consecutive successor (expected 2) | test_boolean.py:52:37:52:37 | IntegerLiteral | Timestamp 1 | test_boolean.py:50:1:50:25 | Function test_mixed_and_or | test_mixed_and_or | | test_boolean.py:52:78:52:79 | IntegerLiteral | $@ in $@ has no consecutive successor (expected 4) | test_boolean.py:52:85:52:85 | IntegerLiteral | Timestamp 3 | test_boolean.py:50:1:50:25 | Function test_mixed_and_or | test_mixed_and_or | +| test_comparison.py:37:17:37:17 | IntegerLiteral | $@ in $@ has no consecutive successor (expected 2) | test_comparison.py:37:23:37:23 | IntegerLiteral | Timestamp 1 | test_comparison.py:32:1:32:32 | Function test_three_short_circuit | test_three_short_circuit | | test_if.py:95:9:95:13 | False | $@ in $@ has no consecutive successor (expected 2) | test_if.py:95:19:95:19 | IntegerLiteral | Timestamp 1 | test_if.py:93:1:93:34 | Function test_if_compound_condition | test_if_compound_condition | | test_if.py:96:9:96:29 | BoolExpr | $@ in $@ has no consecutive successor (expected 5) | test_if.py:96:36:96:36 | IntegerLiteral | Timestamp 4 | test_if.py:93:1:93:34 | Function test_if_compound_condition | test_if_compound_condition | | test_if.py:96:22:96:22 | y | $@ in $@ has no consecutive successor (expected 4) | test_if.py:96:28:96:28 | IntegerLiteral | Timestamp 3 | test_if.py:93:1:93:34 | Function test_if_compound_condition | test_if_compound_condition | diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAllLiveReachable.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAllLiveReachable.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAllLiveReachable.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAllLiveReachable.ql new file mode 100644 index 00000000000..883b266c82e --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAllLiveReachable.ql @@ -0,0 +1,13 @@ +/** New-CFG version of AllLiveReachable. */ + +import python +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from TimerCfgNode a, TestFunction f +where allLiveReachable(a, f) +select a, "Unreachable live annotation; entry of $@ does not reach this node", f, f.getName() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAnnotationHasCfgNode.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAnnotationHasCfgNode.expected new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAnnotationHasCfgNode.expected @@ -0,0 +1 @@ + diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAnnotationHasCfgNode.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAnnotationHasCfgNode.ql new file mode 100644 index 00000000000..bf97a079322 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgAnnotationHasCfgNode.ql @@ -0,0 +1,17 @@ +/** + * New-CFG version of AnnotationHasCfgNode. + * + * Checks that every timer annotation has a corresponding CFG node. + */ + +import python +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils::CfgTests + +from TimerAnnotation ann +where annotationWithoutCfgNode(ann) +select ann, "Annotation in $@ has no CFG node", ann.getTestFunction(), + ann.getTestFunction().getName() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.ql new file mode 100644 index 00000000000..e12ab8ff5ef --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockAnnotationGap.ql @@ -0,0 +1,24 @@ +/** + * New-CFG version of BasicBlockAnnotationGap. + * + * Checks that within a basic block, if a node is annotated then its + * successor is also annotated (or excluded). A gap in annotations + * within a basic block indicates a missing annotation, since there + * are no branches to justify the gap. + * + * Nodes with exceptional successors are excluded, as the exception + * edge leaves the basic block and the normal successor may be dead. + */ + +import python +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from TimerCfgNode a, CfgNode succ +where basicBlockAnnotationGap(a, succ) +select a, "Annotated node followed by unannotated $@ in the same basic block", succ, + succ.getNode().toString() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.ql new file mode 100644 index 00000000000..b8254a8e864 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBasicBlockOrdering.ql @@ -0,0 +1,19 @@ +/** + * New-CFG version of BasicBlockOrdering. + * + * Checks that within a single basic block, annotations appear in + * increasing minimum-timestamp order. + */ + +import python +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from TimerCfgNode a, TimerCfgNode b, int minA, int minB +where basicBlockOrdering(a, b, minA, minB) +select a, "Basic block ordering: $@ appears before $@", a.getTimestampExpr(minA), + "timestamp " + minA, b.getTimestampExpr(minB), "timestamp " + minB diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBranchTimestamps.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBranchTimestamps.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBranchTimestamps.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBranchTimestamps.ql new file mode 100644 index 00000000000..2dc8ae144dc --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgBranchTimestamps.ql @@ -0,0 +1,79 @@ +/** + * New-CFG version of BranchTimestamps. + * + * Checks that when a node has both a true and false successor, the + * live timestamps on one branch are marked as dead on the other. + * This ensures that boolean branches are fully annotated with dead() + * markers for the paths not taken. + * + * Limitation: the `@ t[ts, ...]` / `dead(ts)` annotation scheme can only + * model branch-dead-ness for plain boolean control flow that reconverges + * linearly after the split — i.e. `if`-with-else and `if`-expression. + * It cannot model: + * + * * loops (`while` / `for`): body timestamps repeat across iterations, + * so the loop-exit annotation can't list them as dead; + * * `match` statements: each `case` body is a syntactically distinct + * sub-tree, and the branches don't reconverge through a common + * annotation point in the timeline; + * * `try` / `with` and `raise` / `assert`: exception edges are modeled + * as true/false but flow to syntactically distinct handlers, with no + * reconvergence in the linear annotation order; + * * short-circuit `and` / `or` (`BoolExpr`): the branches reconverge at + * the BoolExpr's after-node, so timestamps on one branch are live + * downstream of the other rather than dead; + * * `if` without an `else` clause, and `if`/`elif` chains: the false + * branch reconverges with the true branch at the post-if statement + * (no-else) or fans out across multiple elif-test annotations, + * neither of which fit the binary annotation scheme. + * + * Branch nodes inside those constructs are therefore whitelisted out + * below. The check still fires (and is useful) for plain `if`/`else` + * and conditional-expression branching. + */ + +import python +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +/** + * Holds if `f` contains a construct whose branches the linear-timestamp + * annotation scheme cannot describe (see file-level comment). + */ +private predicate hasUnmodellableBranching(Function f) { + exists(AstNode bad | + bad.getScope() = f and + ( + bad instanceof While + or + bad instanceof For + or + bad instanceof MatchStmt + or + bad instanceof Try + or + bad instanceof With + or + bad instanceof Raise + or + bad instanceof Assert + or + bad instanceof BoolExpr + or + bad instanceof If and + (not exists(bad.(If).getAnOrelse()) or bad.(If).isElif()) + ) + ) +} + +from TimerCfgNode node, int ts, string branch +where + missingBranchTimestamp(node, ts, branch) and + not hasUnmodellableBranching(node.getTestFunction()) +select node, + "Timestamp " + ts + " on true/false branch is missing a dead() annotation on the " + branch + + " successor in $@", node.getTestFunction(), node.getTestFunction().getName() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.expected new file mode 100644 index 00000000000..0c9e31f3771 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.expected @@ -0,0 +1 @@ +| test_comparison.py:37:6:37:41 | Compare | $@ in $@ has no consecutive predecessor (expected 1) | test_comparison.py:37:48:37:48 | IntegerLiteral | Timestamp 2 | test_comparison.py:32:1:32:32 | Function test_three_short_circuit | test_three_short_circuit | diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.ql new file mode 100644 index 00000000000..0eb8ad5a414 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutivePredecessorTimestamps.ql @@ -0,0 +1,21 @@ +/** + * New-CFG version of ConsecutivePredecessorTimestamps. + * + * Checks that each annotated node (except the minimum timestamp) has + * a predecessor annotation with timestamp `a - 1`. This is the reverse + * of ConsecutiveTimestamps: it catches nodes that are reachable but + * arrived at from the wrong place (skipping an intermediate node). + */ + +import python +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from TimerAnnotation ann, int a +where consecutivePredecessorTimestamps(ann, a) +select ann, "$@ in $@ has no consecutive predecessor (expected " + (a - 1) + ")", + ann.getTimestampExpr(a), "Timestamp " + a, ann.getTestFunction(), ann.getTestFunction().getName() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.expected new file mode 100644 index 00000000000..6fdcfc40c4a --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.expected @@ -0,0 +1 @@ +| test_comparison.py:37:17:37:17 | IntegerLiteral | $@ in $@ has no consecutive successor (expected 2) | test_comparison.py:37:23:37:23 | IntegerLiteral | Timestamp 1 | test_comparison.py:32:1:32:32 | Function test_three_short_circuit | test_three_short_circuit | diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.ql new file mode 100644 index 00000000000..8fc13c9acba --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgConsecutiveTimestamps.ql @@ -0,0 +1,27 @@ +/** + * New-CFG version of ConsecutiveTimestamps. + * + * Checks that consecutive annotated nodes have consecutive timestamps: + * for each annotation with timestamp `a`, some CFG node for that annotation + * must have a next annotation containing `a + 1`. + * + * Handles CFG splitting (e.g., finally blocks duplicated for normal/exceptional + * flow) by checking that at least one split has the required successor. + * + * Only applies to functions where all annotations are in the function's + * own scope (excludes tests with generators, async, comprehensions, or + * lambdas that have annotations in nested scopes). + */ + +import python +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from TimerAnnotation ann, int a +where consecutiveTimestamps(ann, a) +select ann, "$@ in $@ has no consecutive successor (expected " + (a + 1) + ")", + ann.getTimestampExpr(a), "Timestamp " + a, ann.getTestFunction(), ann.getTestFunction().getName() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgImpl.qll b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgImpl.qll new file mode 100644 index 00000000000..ae27e8f16a0 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgImpl.qll @@ -0,0 +1,118 @@ +/** + * Implementation of the evaluation-order CFG signature using the new + * shared control flow graph from AstNodeImpl. + */ + +private import python as Py +import TimerUtils +private import semmle.python.controlflow.internal.AstNodeImpl as CfgImpl +private import codeql.controlflow.SuccessorType + +private class NewControlFlowNode = CfgImpl::ControlFlowNode; + +private class NewBasicBlock = CfgImpl::BasicBlock; + +/** New (shared) CFG implementation of the evaluation-order signature. */ +module NewCfg implements EvalOrderCfgSig { + class CfgNode instanceof NewControlFlowNode { + // We must pick a *unique* representative CFG node for each AST node. The + // shared CFG has several nodes per AST node (before / in-post-order / after + // / after-value splits), but the timer test framework keys annotations on + // `getNode()` and assumes one CFG node per annotated AST node. Without a + // filter, an annotated `f()` would map to both `f()` and `After f()`, which + // breaks two framework invariants: (1) the "no shared reachable" check + // requires that two distinct nodes sharing a timestamp be mutually + // unreachable (true/false branches of a condition), but `Before f()`, + // `f()` and `After f()` share the annotation's timestamp *and* lie on one + // linear path; and (2) the annotation walk (`nextTimerAnnotation`) halts at + // the first reachable representative, so a second node for the same AST + // node would stall the walk on the same timestamp instead of advancing to + // the next evaluation event. + // + // We use the "after" node (`isAfter`) rather than the canonical `injects` + // node, because `injects` represents short-circuit / conditional + // expressions (`and`/`or`/`not`/ternary) by their *before* node, placing + // them ahead of their operands — wrong for evaluation order. `isAfter` + // instead picks the post-evaluation node: the merged before/after node for + // simple leaves, the `TAfterNode` for post-order expressions, and the + // `AfterValueNode`(s) for pre-order conditionals, all positioned after the + // operands. The two value-split nodes of a conditional are genuinely + // distinct evaluation outcomes (handled by `getATrueSuccessor` / + // `getAFalseSuccessor`), so they do not violate the uniqueness assumption. + CfgNode() { NewControlFlowNode.super.isAfter(_) } + + string toString() { result = NewControlFlowNode.super.toString() } + + Py::Location getLocation() { result = NewControlFlowNode.super.getLocation() } + + Py::AstNode getNode() { + result = CfgImpl::astNodeToPyNode(NewControlFlowNode.super.getAstNode()) + } + + CfgNode getASuccessor() { nextCfgNode(this, result) } + + CfgNode getATrueSuccessor() { + NewControlFlowNode.super.isAfterTrue(_) and + // Only where there's also a false branch (true boolean split) + exists(NewControlFlowNode other | other.isAfterFalse(NewControlFlowNode.super.getAstNode())) and + nextCfgNodeFrom(this, result) + } + + CfgNode getAFalseSuccessor() { + NewControlFlowNode.super.isAfterFalse(_) and + // Only where there's also a true branch (true boolean split) + exists(NewControlFlowNode other | other.isAfterTrue(NewControlFlowNode.super.getAstNode())) and + nextCfgNodeFrom(this, result) + } + + CfgNode getAnExceptionalSuccessor() { + exists(NewControlFlowNode mid | + mid = NewControlFlowNode.super.getAnExceptionSuccessor() and + nextCfgNodeFrom(mid, result) + ) + } + + Py::Scope getScope() { result = NewControlFlowNode.super.getEnclosingCallable().asScope() } + + BasicBlock getBasicBlock() { exists(NewBasicBlock bb | bb.getNode(_) = this and result = bb) } + } + + /** + * Holds if `next` is the nearest CfgNode reachable from `n` via + * one or more raw CFG successor edges, skipping non-CfgNode intermediaries. + */ + private predicate nextCfgNodeFrom(NewControlFlowNode n, CfgNode next) { + next = n.getASuccessor() + or + exists(NewControlFlowNode mid | + mid = n.getASuccessor() and + not mid instanceof CfgNode and + nextCfgNodeFrom(mid, next) + ) + } + + /** + * Holds if `next` is the nearest CfgNode successor of `n`, + * skipping synthetic intermediate nodes. + */ + private predicate nextCfgNode(CfgNode n, CfgNode next) { nextCfgNodeFrom(n, next) } + + class BasicBlock instanceof NewBasicBlock { + string toString() { result = NewBasicBlock.super.toString() } + + CfgNode getNode(int n) { result = NewBasicBlock.super.getNode(n) } + + predicate reaches(BasicBlock bb) { this = bb or this.strictlyReaches(bb) } + + predicate strictlyReaches(BasicBlock bb) { NewBasicBlock.super.getASuccessor+() = bb } + + predicate strictlyDominates(BasicBlock bb) { NewBasicBlock.super.strictlyDominates(bb) } + } + + CfgNode scopeGetEntryNode(Py::Scope s) { + exists(CfgImpl::ControlFlow::EntryNode entry | + entry.getEnclosingCallable().asScope() = s and + nextCfgNodeFrom(entry, result) + ) + } +} diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.ql new file mode 100644 index 00000000000..db324e6ee76 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNeverReachable.ql @@ -0,0 +1,19 @@ +/** + * New-CFG version of NeverReachable. + * + * Checks that expressions annotated with `t.never` either have no CFG + * node, or if they do, that the node is not reachable from its scope's + * entry (including within the same basic block). + */ + +import python +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils::CfgTests + +from TimerAnnotation ann +where neverReachable(ann) +select ann, "Node annotated with t.never is reachable in $@", ann.getTestFunction(), + ann.getTestFunction().getName() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.ql new file mode 100644 index 00000000000..8cb0dfb143b --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBackwardFlow.ql @@ -0,0 +1,20 @@ +/** + * New-CFG version of NoBackwardFlow. + * + * Checks that time never flows backward between consecutive timer annotations + * in the CFG. For each pair of consecutive annotated nodes (A -> B), there must + * exist timestamps a in A and b in B with a < b. + */ + +import python +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from TimerCfgNode a, TimerCfgNode b, int minA, int maxB +where noBackwardFlow(a, b, minA, maxB) +select a, "Backward flow: $@ flows to $@ (max timestamp $@)", a.getTimestampExpr(minA), + minA.toString(), b, b.getNode().toString(), b.getTimestampExpr(maxB), maxB.toString() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBasicBlock.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBasicBlock.expected new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBasicBlock.expected @@ -0,0 +1 @@ + diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBasicBlock.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBasicBlock.ql new file mode 100644 index 00000000000..c0fc86b5130 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoBasicBlock.ql @@ -0,0 +1,17 @@ +/** + * New-CFG version of NoBasicBlock. + * + * Checks that every annotated CFG node belongs to a basic block. + */ + +import python +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from CfgNode n, TestFunction f +where noBasicBlock(n, f) +select n, "CFG node in $@ does not belong to any basic block", f, f.getName() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.ql new file mode 100644 index 00000000000..dd174d6d913 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgNoSharedReachable.ql @@ -0,0 +1,19 @@ +/** + * New-CFG version of NoSharedReachable. + * + * Checks that two annotations sharing a timestamp value are on + * mutually exclusive CFG paths (neither can reach the other). + */ + +import python +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from TimerCfgNode a, TimerCfgNode b, int ts +where noSharedReachable(a, b, ts) +select a, "Shared timestamp $@ but this node reaches $@", a.getTimestampExpr(ts), ts.toString(), b, + b.getNode().toString() diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.expected b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.ql b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.ql new file mode 100644 index 00000000000..e7f4a12ff81 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/NewCfgStrictForward.ql @@ -0,0 +1,20 @@ +/** + * New-CFG version of StrictForward. + * + * Stronger version of NoBackwardFlow: for consecutive annotated nodes + * A -> B that both have a single timestamp (non-loop code) and B does + * NOT dominate A (forward edge), requires max(A) < min(B). + */ + +import python +import NewCfgImpl + +private module Utils = EvalOrderCfgUtils; + +private import Utils +private import Utils::CfgTests + +from TimerCfgNode a, TimerCfgNode b, int maxA, int minB +where strictForward(a, b, maxA, minB) +select a, "Strict forward violation: $@ flows to $@", a.getTimestampExpr(maxA), "timestamp " + maxA, + b.getTimestampExpr(minB), "timestamp " + minB diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/test_comparison.py b/python/ql/test/library-tests/ControlFlow/evaluation-order/test_comparison.py new file mode 100644 index 00000000000..97866317166 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/test_comparison.py @@ -0,0 +1,37 @@ +"""Comparisons and evaluation order. + +Comparison operands are evaluated left-to-right, followed by the comparison +node itself. Annotations record the *run-time* evaluation order, so this file +self-validates under CPython (``python3 test_comparison.py``). + +Python short-circuits *chained* comparisons (in ``a < b < c`` the operand ``c`` +is skipped once ``a < b`` is false), whereas the control-flow graph +conservatively evaluates every operand. That divergence cannot be expressed in +a single annotation, so the affected CFG queries report it (captured in the +ConsecutiveTimestamps and NewCfgConsecutivePredecessorTimestamps .expected +files). +""" + +from timer import test, dead + + +@test +def test_two(t): + # Both operands, then the comparison. + (1 @ t[0] < 0 @ t[1]) @ t[2] + + +@test +def test_three(t): + # Chained comparison, all comparisons hold: operands left-to-right, then + # the comparison. Run time and CFG agree. + (1 @ t[0] < 2 @ t[1] < 3 @ t[2]) @ t[3] + + +@test +def test_three_short_circuit(t): + # ``1 > 2`` is false, so at run time Python short-circuits and never + # evaluates ``3``; the comparison is reached at timestamp 2. The CFG still + # evaluates ``3`` (dead at run time), which is why the CFG queries report a + # gap around this comparison. + (1 @ t[0] > 2 @ t[1] < 3 @ t[dead(2)]) @ t[2] diff --git a/python/ql/test/library-tests/ControlFlow/evaluation-order/test_if.py b/python/ql/test/library-tests/ControlFlow/evaluation-order/test_if.py index 8880aaaef34..a6eb6c7d5ca 100644 --- a/python/ql/test/library-tests/ControlFlow/evaluation-order/test_if.py +++ b/python/ql/test/library-tests/ControlFlow/evaluation-order/test_if.py @@ -85,7 +85,7 @@ def test_nested_if_else(t): else: z = 2 @ t[dead(4)] else: - z = 3 @ t[dead(4)] + z = 3 @ t[dead(3), dead(4)] w = 0 @ t[5] diff --git a/python/ql/test/library-tests/ControlFlow/store-load/StoreLoadTest.expected b/python/ql/test/library-tests/ControlFlow/store-load/StoreLoadTest.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/ql/test/library-tests/ControlFlow/store-load/StoreLoadTest.ql b/python/ql/test/library-tests/ControlFlow/store-load/StoreLoadTest.ql new file mode 100644 index 00000000000..4ab2ef5be8f --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/store-load/StoreLoadTest.ql @@ -0,0 +1,41 @@ +/** + * Inline-expectations test for the store/load/delete/parameter + * classification predicates on the new-CFG facade. + * + * Each tag fires when the corresponding predicate (`isLoad`, + * `isStore`, `isDelete`, `isParameter`, `isAugLoad`, `isAugStore`) + * holds on the canonical CFG node wrapping a `Py::Name` with the + * given identifier. Subscript and attribute stores are not covered + * by these tags — only the `Name`-typed targets/loads they involve. + */ + +import python +import semmle.python.controlflow.internal.Cfg as Cfg +import utils.test.InlineExpectationsTest + +module StoreLoadTest implements TestSig { + string getARelevantTag() { result = ["load", "store", "delete", "param", "augload", "augstore"] } + + predicate hasActualResult(Location location, string element, string tag, string value) { + exists(Cfg::NameNode n | + location = n.getLocation() and + element = n.toString() and + value = n.getId() and + ( + n.isLoad() and not n.isAugLoad() and tag = "load" + or + n.isStore() and not n.isAugStore() and tag = "store" + or + n.isDelete() and tag = "delete" + or + n.isParameter() and tag = "param" + or + n.isAugLoad() and tag = "augload" + or + n.isAugStore() and tag = "augstore" + ) + ) + } +} + +import MakeTest diff --git a/python/ql/test/library-tests/ControlFlow/store-load/test.py b/python/ql/test/library-tests/ControlFlow/store-load/test.py new file mode 100644 index 00000000000..dfca45a0b47 --- /dev/null +++ b/python/ql/test/library-tests/ControlFlow/store-load/test.py @@ -0,0 +1,56 @@ +# Store/load/delete/parameter classification on the new-CFG facade. +# +# Each annotated location carries the (sorted, deduplicated) set of +# kinds the CFG facade reports there. Comparing against the legacy +# 'semmle.python.Flow' classification is done by the comparison query +# 'StoreLoadParity.ql' — annotations here are only the positive +# assertions for the new facade. +# +# Tags: +# load= -- isLoad() fires on the Name +# store= -- isStore() fires +# delete= -- isDelete() fires +# param= -- isParameter() fires +# augload= -- isAugLoad() fires (the LHS of x += ... when read) +# augstore= -- isAugStore() fires (the LHS of x += ... when written) + + +# --- plain load / store / delete --- + +x = 1 # $ store=x +y = x + 1 # $ store=y load=x +print(y) # $ load=print load=y +del x # $ delete=x + + +# --- function definitions (parameters) --- + +def f(a, b=2, *args, c, **kwargs): # $ store=f param=a param=b param=args param=c param=kwargs + return a + b + c # $ load=a load=b load=c + + +# --- augmented assignment splits one Name into load + store halves --- + +def aug(): # $ store=aug + n = 0 # $ store=n + n += 1 # $ augload=n augstore=n + return n # $ load=n + + +# --- subscript / attribute stores --- + +class C: # $ store=C + pass + + +def stores(obj, container, idx): # $ store=stores param=obj param=container param=idx + obj.attr = 1 # $ load=obj + container[idx] = 2 # $ load=container load=idx + return obj # $ load=obj + + +# --- tuple unpacking --- + +def unpack(pair): # $ store=unpack param=pair + a, b = pair # $ store=a store=b load=pair + return a + b # $ load=a load=b diff --git a/ql/Cargo.lock b/ql/Cargo.lock index 5c65d88de6a..d6ea9ea343e 100644 Binary files a/ql/Cargo.lock and b/ql/Cargo.lock differ diff --git a/ql/extractor/src/extractor.rs b/ql/extractor/src/extractor.rs index 66096442e85..8383d8424ee 100644 --- a/ql/extractor/src/extractor.rs +++ b/ql/extractor/src/extractor.rs @@ -29,28 +29,24 @@ pub fn run(options: Options) -> std::io::Result<()> { prefix: "ql", ts_language: tree_sitter_ql::LANGUAGE.into(), node_types: tree_sitter_ql::NODE_TYPES, - desugar: None, file_globs: vec!["*.ql".into(), "*.qll".into()], }, simple::LanguageSpec { prefix: "dbscheme", ts_language: tree_sitter_ql_dbscheme::LANGUAGE.into(), node_types: tree_sitter_ql_dbscheme::NODE_TYPES, - desugar: None, file_globs: vec!["*.dbscheme".into()], }, simple::LanguageSpec { prefix: "json", ts_language: tree_sitter_json::LANGUAGE.into(), node_types: tree_sitter_json::NODE_TYPES, - desugar: None, file_globs: vec!["*.json".into(), "*.jsonl".into(), "*.jsonc".into()], }, simple::LanguageSpec { prefix: "blame", ts_language: tree_sitter_blame::LANGUAGE.into(), node_types: tree_sitter_blame::NODE_TYPES, - desugar: None, file_globs: vec!["*.blame".into()], }, ], diff --git a/ruby/extractor/src/extractor.rs b/ruby/extractor/src/extractor.rs index d418c144bfc..3749609e788 100644 --- a/ruby/extractor/src/extractor.rs +++ b/ruby/extractor/src/extractor.rs @@ -94,7 +94,9 @@ pub fn run(options: Options) -> std::io::Result<()> { node_types::read_node_types_str("erb", tree_sitter_embedded_template::NODE_TYPES)?; let lines: std::io::Result> = std::io::BufReader::new(file_list).lines().collect(); let lines = lines?; - let source_root = std::env::current_dir().ok().and_then(|d| d.canonicalize().ok()); + let source_root = std::env::current_dir() + .ok() + .and_then(|d| d.canonicalize().ok()); lines .par_iter() .try_for_each(|line| { @@ -126,7 +128,6 @@ pub fn run(options: Options) -> std::io::Result<()> { &path, &source, &[], - None, ); let (ranges, line_breaks) = scan_erb( @@ -215,7 +216,6 @@ pub fn run(options: Options) -> std::io::Result<()> { &path, &source, &code_ranges, - None, ); std::fs::create_dir_all(src_archive_file.parent().unwrap())?; if needs_conversion { diff --git a/ruby/ql/integration-tests/subst/file.expected b/ruby/ql/integration-tests/subst/file.expected index 47b324fea9e..2962c283b48 100644 --- a/ruby/ql/integration-tests/subst/file.expected +++ b/ruby/ql/integration-tests/subst/file.expected @@ -1,2 +1,2 @@ -| code/test.rb:0:0:0:0 | code/test.rb | | +| code/test.rb:0:0:0:0 | code/test.rb | relative | | file://:0:0:0:0 | | | diff --git a/rust/ql/integration-tests/subst/file.expected b/rust/ql/integration-tests/subst/file.expected index c1ff73337f2..407f50b5c0e 100644 --- a/rust/ql/integration-tests/subst/file.expected +++ b/rust/ql/integration-tests/subst/file.expected @@ -1,2 +1,2 @@ -| code/test.rs:0:0:0:0 | code/test.rs | | +| code/test.rs:0:0:0:0 | code/test.rs | relative | | file://:0:0:0:0 | | | diff --git a/shared/tree-sitter-extractor/src/extractor/desugaring.rs b/shared/tree-sitter-extractor/src/extractor/desugaring.rs new file mode 100644 index 00000000000..c52659750c7 --- /dev/null +++ b/shared/tree-sitter-extractor/src/extractor/desugaring.rs @@ -0,0 +1,104 @@ +//! Extraction for languages that rewrite their syntax tree before extraction. +//! +//! Unlike [`crate::extractor::simple`] (direct tree-sitter extraction), a +//! desugaring language parses source into a [`ParsedTree`] — a `yeast::Ast` +//! plus side-channel `extra` tokens (comments and similar) — and rewrites the +//! AST through a [`yeast::Desugarer`] before emitting TRAP. The parser is a +//! closure, so both tree-sitter grammars (via +//! [`crate::extractor::tree_sitter_parser`]) and fully custom parsers plug in +//! the same way. + +use crate::trap; +use std::path::PathBuf; + +use crate::diagnostics; +use crate::extractor::ParsedTree; +use crate::extractor::driver::{self, LanguageExtractor}; +use crate::node_types::{self, NodeTypeMap}; + +/// A parser turns source bytes into a [`ParsedTree`]. Tree-sitter grammars plug +/// in via [`crate::extractor::tree_sitter_parser`]; custom (non-tree-sitter) +/// parsers supply their own closure. +pub type Parser = Box Result + Send + Sync>; + +pub struct LanguageSpec { + pub prefix: &'static str, + /// The parser: source -> `yeast::Ast` + `extra` tokens (see [`Parser`]). + pub parser: Parser, + /// Fallback TRAP schema, used only when `desugarer` does not supply its own + /// output schema (via [`yeast::Desugarer::output_node_types_yaml`]). May be + /// empty for a custom parser whose desugarer always provides the schema. + pub node_types: &'static str, + /// The desugarer applied to the parsed AST before extraction. Its + /// `output_node_types_yaml()` (when set) provides the TRAP schema. + /// + /// `Box` so the shared extractor is agnostic to the + /// user-defined context type the desugarer uses internally. + pub desugarer: Box, + pub file_globs: Vec, +} + +impl LanguageExtractor for LanguageSpec { + fn file_globs(&self) -> &[String] { + &self.file_globs + } + + fn build_schema(&self) -> std::io::Result { + let effective_node_types: String = match self.desugarer.output_node_types_yaml() { + Some(yaml) => yeast::node_types_yaml::convert(yaml).map_err(|e| { + std::io::Error::other(format!( + "Failed to convert YAML node-types to JSON for {}: {e}", + self.prefix + )) + })?, + None => self.node_types.to_string(), + }; + node_types::read_node_types_str(self.prefix, &effective_node_types) + } + + fn extract_file( + &self, + schema: &NodeTypeMap, + diagnostics_writer: &mut diagnostics::LogWriter, + trap_writer: &mut trap::Writer, + path: &std::path::Path, + source: &[u8], + ) { + crate::extractor::extract_parsed( + self.parser.as_ref(), + self.prefix, + schema, + diagnostics_writer, + trap_writer, + None, + path, + source, + self.desugarer.as_ref(), + ); + } +} + +pub struct Extractor { + pub prefix: String, + pub languages: Vec, + pub trap_dir: PathBuf, + pub source_archive_dir: PathBuf, + pub file_lists: Vec, + // Typically constructed via `trap::Compression::from_env`. + // This allow us to report the error using our diagnostics system + // without exposing it to consumers. + pub trap_compression: Result, +} + +impl Extractor { + pub fn run(&self) -> std::io::Result<()> { + driver::run_extractor( + &self.prefix, + &self.languages, + &self.trap_dir, + &self.source_archive_dir, + &self.file_lists, + &self.trap_compression, + ) + } +} diff --git a/shared/tree-sitter-extractor/src/extractor/driver.rs b/shared/tree-sitter-extractor/src/extractor/driver.rs new file mode 100644 index 00000000000..d97d8f4c75c --- /dev/null +++ b/shared/tree-sitter-extractor/src/extractor/driver.rs @@ -0,0 +1,210 @@ +//! Shared multi-file extraction driver. +//! +//! The `simple` (direct tree-sitter) and `desugaring` (parse + desugar) +//! extractors differ only in how a language's schema is built and how a single +//! file is extracted. Everything else — threading, matching files to languages +//! by glob, writing TRAP, and copying into the source archive — is identical +//! and lives here, parameterised over the [`LanguageExtractor`] trait. + +use globset::{GlobBuilder, GlobSetBuilder}; +use rayon::prelude::*; +use std::fs::File; +use std::io::BufRead; +use std::path::{Path, PathBuf}; + +use crate::diagnostics; +use crate::file_paths; +use crate::node_types::NodeTypeMap; +use crate::trap; + +/// A language that [`run_extractor`] can process: it knows its file globs, its +/// TRAP schema, and how to extract a single file. Implemented by +/// [`super::simple::LanguageSpec`] (direct tree-sitter extraction) and +/// [`super::desugaring::LanguageSpec`] (parse into an AST and desugar it). +pub(crate) trait LanguageExtractor: Sync { + /// The file-name globs that select files for this language. + fn file_globs(&self) -> &[String]; + /// Build the TRAP node-type schema used to validate emitted tuples. + fn build_schema(&self) -> std::io::Result; + /// Extract a single file's `source` into `trap_writer`. + fn extract_file( + &self, + schema: &NodeTypeMap, + diagnostics_writer: &mut diagnostics::LogWriter, + trap_writer: &mut trap::Writer, + path: &Path, + source: &[u8], + ); +} + +/// Drive extraction over `languages` for every file listed in `file_lists`. +/// +/// Sets up the thread pool, builds a combined glob set, and for each input file +/// dispatches to the matching language's [`LanguageExtractor::extract_file`], +/// writing the resulting TRAP and a source-archive copy. +pub(crate) fn run_extractor( + prefix: &str, + languages: &[L], + trap_dir: &Path, + source_archive_dir: &Path, + file_lists: &[PathBuf], + trap_compression: &Result, +) -> std::io::Result<()> { + tracing::info!("Extraction started"); + let diagnostics = diagnostics::DiagnosticLoggers::new(prefix); + let mut main_thread_logger = diagnostics.logger(); + let num_threads = match crate::options::num_threads() { + Ok(num) => num, + Err(e) => { + main_thread_logger.write( + main_thread_logger + .new_entry("configuration-error", "Configuration error") + .message( + "{}; defaulting to 1 thread.", + &[diagnostics::MessageArg::Code(&e)], + ) + .severity(diagnostics::Severity::Warning), + ); + 1 + } + }; + tracing::info!( + "Using {} {}", + num_threads, + if num_threads == 1 { + "thread" + } else { + "threads" + } + ); + let trap_compression = match trap_compression { + Ok(x) => *x, + Err(e) => { + main_thread_logger.write( + main_thread_logger + .new_entry("configuration-error", "Configuration error") + .message("{}; using gzip.", &[diagnostics::MessageArg::Code(e)]) + .severity(diagnostics::Severity::Warning), + ); + trap::Compression::Gzip + } + }; + drop(main_thread_logger); + + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build_global() + .unwrap(); + + let file_lists: Vec = file_lists + .iter() + .map(|file_list| { + File::open(file_list) + .unwrap_or_else(|_| panic!("Unable to open file list at {file_list:?}")) + }) + .collect(); + + let mut schemas = vec![]; + for lang in languages { + schemas.push(lang.build_schema()?); + } + + // Construct a single globset containing all language globs, + // and a mapping from glob index to language index. + let (globset, glob_language_mapping) = { + let mut builder = GlobSetBuilder::new(); + let mut glob_lang_mapping = vec![]; + for (i, lang) in languages.iter().enumerate() { + for glob_str in lang.file_globs() { + let glob = GlobBuilder::new(glob_str) + .literal_separator(true) + .build() + .expect("invalid glob"); + builder.add(glob); + glob_lang_mapping.push(i); + } + } + ( + builder.build().expect("failed to build globset"), + glob_lang_mapping, + ) + }; + + let path_transformer = file_paths::load_path_transformer()?; + + let lines: std::io::Result> = file_lists + .iter() + .flat_map(|file_list| std::io::BufReader::new(file_list).lines()) + .collect(); + let lines = lines?; + + lines + .par_iter() + .try_for_each(|line| { + let mut diagnostics_writer = diagnostics.logger(); + let path = PathBuf::from(line).canonicalize()?; + let src_archive_file = crate::file_paths::path_for( + source_archive_dir, + &path, + "", + path_transformer.as_ref(), + ); + let source = std::fs::read(&path)?; + let mut trap_writer = trap::Writer::new(); + + match path.file_name() { + None => { + tracing::error!(?path, "No file name found, skipping file."); + } + Some(filename) => { + let matches = globset.matches(filename); + if matches.is_empty() { + tracing::error!(?path, "No matching language found, skipping file."); + } else { + let mut languages_processed = vec![false; languages.len()]; + + for m in matches { + let i = glob_language_mapping[m]; + if languages_processed[i] { + continue; + } + languages_processed[i] = true; + let lang = &languages[i]; + + lang.extract_file( + &schemas[i], + &mut diagnostics_writer, + &mut trap_writer, + &path, + &source, + ); + std::fs::create_dir_all(src_archive_file.parent().unwrap())?; + std::fs::copy(&path, &src_archive_file)?; + write_trap(trap_dir, &path, &trap_writer, trap_compression)?; + } + } + } + } + Ok(()) as std::io::Result<()> + }) + .expect("failed to extract files"); + + let path = PathBuf::from("extras"); + let mut trap_writer = trap::Writer::new(); + crate::extractor::populate_empty_location(&mut trap_writer); + + let res = write_trap(trap_dir, &path, &trap_writer, trap_compression); + tracing::info!("Extraction complete"); + res +} + +fn write_trap( + trap_dir: &Path, + path: &Path, + trap_writer: &trap::Writer, + trap_compression: trap::Compression, +) -> std::io::Result<()> { + let trap_file = crate::file_paths::path_for(trap_dir, path, trap_compression.extension(), None); + std::fs::create_dir_all(trap_file.parent().unwrap())?; + trap_writer.write_to_file(&trap_file, trap_compression) +} diff --git a/shared/tree-sitter-extractor/src/extractor/mod.rs b/shared/tree-sitter-extractor/src/extractor/mod.rs index 474dc096a2a..1a8b9e82016 100644 --- a/shared/tree-sitter-extractor/src/extractor/mod.rs +++ b/shared/tree-sitter-extractor/src/extractor/mod.rs @@ -16,6 +16,8 @@ use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tree_sitter::{Language, Node, Parser, Range, Tree}; +pub mod desugaring; +mod driver; pub mod simple; /// Trait abstracting over tree-sitter and yeast node types for extraction. @@ -294,6 +296,10 @@ pub fn location_label(writer: &mut trap::Writer, location: trap::Location) -> tr /// caller's responsibility, allowing it to be done once and shared across /// files. #[allow(clippy::too_many_arguments)] +/// Extract a file with a tree-sitter grammar, walking the parse tree directly. +/// Comments and other `extra` nodes are emitted inline as tokens. This is the +/// path for languages that don't desugar their syntax tree (and hence have no +/// `extra` side channel); desugaring languages use [`extract_parsed`] instead. pub fn extract( language: &Language, language_prefix: &str, @@ -304,7 +310,6 @@ pub fn extract( path: &Path, source: &[u8], ranges: &[Range], - desugarer: Option<&dyn yeast::Desugarer>, ) { let path_str = file_paths::normalize_and_transform_path(path, transformer); let source_root = std::env::current_dir() @@ -337,21 +342,173 @@ pub fn extract( schema, ); - if let Some(desugarer) = desugarer { - let ast = desugarer - .run_from_tree(&tree, source) - .unwrap_or_else(|e| panic!("Desugaring failed for {path_str}: {e}")); - traverse_yeast(&ast, &mut visitor); - // Comments and other `extra` nodes are not represented in the desugared - // AST, so recover them directly from the original parse tree. - traverse_extras(&tree, &mut visitor); - } else { - traverse(&tree, &mut visitor); - } + traverse(&tree, &mut visitor); parser.reset(); } +/// A source tree produced by a parser: the raw `yeast::Ast` (pre-desugaring) +/// plus side-channel `extra` tokens (comments and similar) that are not +/// attached to the AST proper. +pub struct ParsedTree { + pub ast: yeast::Ast, + pub extras: Vec, +} + +/// A piece of side-channel `extra` content (a comment or unexpected text) +/// recovered by a parser. `kind` is a language-defined id written verbatim into +/// the `_trivia_tokeninfo` table (the tree-sitter parser uses the +/// grammar's node kind id here; a custom parser supplies its own stable +/// enumeration). +pub struct ExtraToken { + pub kind: usize, + pub range: yeast::Range, + pub text: String, +} + +/// Build a parser closure that parses `source` with a tree-sitter `language`, +/// producing a [`ParsedTree`]: a `yeast::Ast` (via [`yeast::Ast::from_tree`]) +/// plus the `extra` nodes (comments and similar) recovered as side-channel +/// tokens. This lets a tree-sitter language plug into the same +/// `ParsedTree`-producing interface as a custom parser. +pub fn tree_sitter_parser( + language: tree_sitter::Language, +) -> impl Fn(&[u8]) -> Result + Send + Sync { + move |source: &[u8]| { + let mut parser = Parser::new(); + parser + .set_language(&language) + .map_err(|e| format!("failed to set tree-sitter language: {e}"))?; + let tree = parser + .parse(source, None) + .ok_or_else(|| "tree-sitter failed to parse".to_string())?; + let ast = yeast::Ast::from_tree_with_schema_and_source( + yeast::schema::from_language(&language), + &tree, + &language, + source.to_vec(), + ); + let mut extras = Vec::new(); + collect_extras(tree.root_node(), source, &mut extras); + Ok(ParsedTree { ast, extras }) + } +} + +/// Collect `extra` nodes (comments and similar) under `node` into `out` as +/// [`ExtraToken`]s, keyed by the grammar's node kind id, for a desugaring +/// language whose rewritten AST won't retain them. +fn collect_extras(node: Node<'_>, source: &[u8], out: &mut Vec) { + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + if child.is_extra() { + let text = String::from_utf8_lossy(&source[child.byte_range()]).into_owned(); + out.push(ExtraToken { + kind: child.kind_id() as usize, + range: child.range().into(), + text, + }); + } else { + collect_extras(child, source, out); + } + } +} + +/// Extract a file from a [`ParsedTree`]-producing parser that desugars. The +/// parser yields a `yeast::Ast` plus side-channel `extra` tokens; the AST is +/// rewritten through `desugarer` ([`yeast::Desugarer::run_from_ast`]) before +/// TRAP extraction, and the `extra` tokens (comments and similar, which the +/// desugared AST does not carry) are emitted from the side channel. Both +/// tree-sitter grammars (via [`tree_sitter_parser`]) and custom parsers plug in +/// here; languages that don't desugar use [`extract`] instead. +#[allow(clippy::too_many_arguments)] +pub fn extract_parsed( + parse: &(dyn Fn(&[u8]) -> Result + Send + Sync), + language_prefix: &str, + schema: &NodeTypeMap, + diagnostics_writer: &mut diagnostics::LogWriter, + trap_writer: &mut trap::Writer, + transformer: Option<&file_paths::PathTransformer>, + path: &Path, + source: &[u8], + desugarer: &dyn yeast::Desugarer, +) { + let path_str = file_paths::normalize_and_transform_path(path, transformer); + let source_root = std::env::current_dir() + .ok() + .and_then(|d| d.canonicalize().ok()); + let diagnostics_path = file_paths::relativize_for_diagnostic(path, source_root.as_deref()); + let span = tracing::span!(tracing::Level::TRACE, "extract", file = %path_str); + let _enter = span.enter(); + tracing::debug!("extracting: {}", path_str); + + trap_writer.comment(format!("Auto-generated TRAP file for {path_str}")); + let file_label = populate_file(trap_writer, path, transformer); + let mut visitor = Visitor::new( + source, + diagnostics_writer, + trap_writer, + &diagnostics_path, + file_label, + language_prefix, + schema, + ); + + let parsed = parse(source).unwrap_or_else(|e| panic!("Parsing failed for {path_str}: {e}")); + let ast = desugarer + .run_from_ast(parsed.ast) + .unwrap_or_else(|e| panic!("Desugaring failed for {path_str}: {e}")); + traverse_yeast(&ast, &mut visitor); + // Comments and other `extra` tokens are not part of the desugared AST; emit + // them directly from the parser's side channel. + for extra in &parsed.extras { + visitor.emit_extra(extra); + } +} + +/// A lightweight [`AstNode`] over a piece of side-channel `extra` content +/// recovered by a custom parser, so it can reuse the tree-sitter location +/// machinery. +struct ExtraNode { + range: yeast::Range, + text: String, +} + +impl AstNode for ExtraNode { + fn kind(&self) -> &str { + "extra" + } + fn is_named(&self) -> bool { + false + } + fn is_missing(&self) -> bool { + false + } + fn is_error(&self) -> bool { + false + } + fn is_extra(&self) -> bool { + true + } + fn start_position(&self) -> tree_sitter::Point { + tree_sitter::Point { + row: self.range.start_point.row, + column: self.range.start_point.column, + } + } + fn end_position(&self) -> tree_sitter::Point { + tree_sitter::Point { + row: self.range.end_point.row, + column: self.range.end_point.column, + } + } + fn byte_range(&self) -> std::ops::Range { + self.range.start_byte..self.range.end_byte + } + fn opt_string_content(&self) -> Option { + Some(self.text.clone()) + } +} + struct ChildNode { field_name: Option<&'static str>, label: trap::Label, @@ -415,10 +572,11 @@ impl<'a> Visitor<'a> { } } - /// Emits a `TriviaToken` for the given `extra` node (e.g. a comment) from - /// the original parse tree. Trivia tokens carry a location and their source - /// text, but are not attached to a parent in the (possibly desugared) AST. - fn emit_trivia_token(&mut self, node: &Node) { + /// Emit an `extra` token (a location row plus a `trivia_tokeninfo` row) for + /// any [`AstNode`], with an explicit `kind` id. Shared by the tree-sitter + /// path (kind = the grammar's node kind id) and the custom-parser path + /// (kind = a language-defined `extra` kind id). + fn emit_extra_from(&mut self, node: &N, kind: usize) { let id = self.trap_writer.fresh_id(); let loc = location_for(self, self.file_label, node); let loc_label = location_label(self.trap_writer, loc); @@ -430,12 +588,21 @@ impl<'a> Visitor<'a> { &self.trivia_tokeninfo_table_name, vec![ trap::Arg::Label(id), - trap::Arg::Int(node.kind_id() as usize), + trap::Arg::Int(kind), sliced_source_arg(self.source, node), ], ); } + /// Emit an `extra` token recovered from a parser's side channel. + fn emit_extra(&mut self, extra: &ExtraToken) { + let node = ExtraNode { + range: extra.range, + text: extra.text.clone(), + }; + self.emit_extra_from(&node, extra.kind); + } + fn record_parse_error(&mut self, loc: trap::Label, mesg: &diagnostics::DiagnosticMessage) { self.diagnostics_writer.write(mesg); let id = self.trap_writer.fresh_id(); @@ -871,24 +1038,6 @@ fn traverse(tree: &Tree, visitor: &mut Visitor) { } } -/// Walks the original tree-sitter tree and emits a `TriviaToken` for every -/// `extra` node (e.g. a comment). Used to preserve comments that would -/// otherwise be lost after a desugaring pass rewrites the tree. -fn traverse_extras(tree: &Tree, visitor: &mut Visitor) { - emit_extras_in(visitor, tree.root_node()); -} - -fn emit_extras_in(visitor: &mut Visitor, node: Node<'_>) { - let mut cursor = node.walk(); - for child in node.children(&mut cursor) { - if child.is_extra() { - visitor.emit_trivia_token(&child); - } else { - emit_extras_in(visitor, child); - } - } -} - fn traverse_yeast(tree: &yeast::Ast, visitor: &mut Visitor) { let mut cursor = tree.walk(); visitor.enter_node(cursor.node()); diff --git a/shared/tree-sitter-extractor/src/extractor/simple.rs b/shared/tree-sitter-extractor/src/extractor/simple.rs index 9ba6f21778c..1c6691fa8cf 100644 --- a/shared/tree-sitter-extractor/src/extractor/simple.rs +++ b/shared/tree-sitter-extractor/src/extractor/simple.rs @@ -1,29 +1,52 @@ -use crate::{file_paths, trap}; -use globset::{GlobBuilder, GlobSetBuilder}; -use rayon::prelude::*; -use std::fs::File; -use std::io::BufRead; -use std::path::{Path, PathBuf}; +use crate::trap; +use std::path::PathBuf; use crate::diagnostics; -use crate::node_types; -use yeast; +use crate::extractor::driver::{self, LanguageExtractor}; +use crate::node_types::{self, NodeTypeMap}; +/// A tree-sitter language extracted directly from its parse tree, with no +/// desugaring. Comments and other `extra` nodes are emitted inline as tokens. +/// Languages that rewrite their syntax tree use +/// [`crate::extractor::desugaring`] instead. pub struct LanguageSpec { pub prefix: &'static str, pub ts_language: tree_sitter::Language, pub node_types: &'static str, - /// Optional desugarer. When set, the parsed tree is rewritten through - /// the desugarer before TRAP extraction. The desugarer's - /// `output_node_types_yaml()` (if set) provides the schema used both - /// at runtime (for the rewriter) and for TRAP validation. - /// - /// `Box` so the shared extractor is agnostic to - /// the user-defined context type the desugarer uses internally. - pub desugar: Option>, pub file_globs: Vec, } +impl LanguageExtractor for LanguageSpec { + fn file_globs(&self) -> &[String] { + &self.file_globs + } + + fn build_schema(&self) -> std::io::Result { + node_types::read_node_types_str(self.prefix, self.node_types) + } + + fn extract_file( + &self, + schema: &NodeTypeMap, + diagnostics_writer: &mut diagnostics::LogWriter, + trap_writer: &mut trap::Writer, + path: &std::path::Path, + source: &[u8], + ) { + crate::extractor::extract( + &self.ts_language, + self.prefix, + schema, + diagnostics_writer, + trap_writer, + None, + path, + source, + &[], + ); + } +} + pub struct Extractor { pub prefix: String, pub languages: Vec, @@ -38,182 +61,13 @@ pub struct Extractor { impl Extractor { pub fn run(&self) -> std::io::Result<()> { - tracing::info!("Extraction started"); - let diagnostics = diagnostics::DiagnosticLoggers::new(&self.prefix); - let mut main_thread_logger = diagnostics.logger(); - let num_threads = match crate::options::num_threads() { - Ok(num) => num, - Err(e) => { - main_thread_logger.write( - main_thread_logger - .new_entry("configuration-error", "Configuration error") - .message( - "{}; defaulting to 1 thread.", - &[diagnostics::MessageArg::Code(&e)], - ) - .severity(diagnostics::Severity::Warning), - ); - 1 - } - }; - tracing::info!( - "Using {} {}", - num_threads, - if num_threads == 1 { - "thread" - } else { - "threads" - } - ); - let trap_compression = match &self.trap_compression { - Ok(x) => *x, - Err(e) => { - main_thread_logger.write( - main_thread_logger - .new_entry("configuration-error", "Configuration error") - .message("{}; using gzip.", &[diagnostics::MessageArg::Code(e)]) - .severity(diagnostics::Severity::Warning), - ); - trap::Compression::Gzip - } - }; - drop(main_thread_logger); - - rayon::ThreadPoolBuilder::new() - .num_threads(num_threads) - .build_global() - .unwrap(); - - let file_lists: Vec = self - .file_lists - .iter() - .map(|file_list| { - File::open(file_list) - .unwrap_or_else(|_| panic!("Unable to open file list at {file_list:?}")) - }) - .collect(); - - let mut schemas = vec![]; - for lang in &self.languages { - let effective_node_types: String = match lang - .desugar - .as_ref() - .and_then(|d| d.output_node_types_yaml()) - { - Some(yaml) => yeast::node_types_yaml::convert(yaml).map_err(|e| { - std::io::Error::other(format!( - "Failed to convert YAML node-types to JSON for {}: {e}", - lang.prefix - )) - })?, - None => lang.node_types.to_string(), - }; - let schema = node_types::read_node_types_str(lang.prefix, &effective_node_types)?; - schemas.push(schema); - } - - // Construct a single globset containing all language globs, - // and a mapping from glob index to language index. - let (globset, glob_language_mapping) = { - let mut builder = GlobSetBuilder::new(); - let mut glob_lang_mapping = vec![]; - for (i, lang) in self.languages.iter().enumerate() { - for glob_str in &lang.file_globs { - let glob = GlobBuilder::new(glob_str) - .literal_separator(true) - .build() - .expect("invalid glob"); - builder.add(glob); - glob_lang_mapping.push(i); - } - } - ( - builder.build().expect("failed to build globset"), - glob_lang_mapping, - ) - }; - - let path_transformer = file_paths::load_path_transformer()?; - - let lines: std::io::Result> = file_lists - .iter() - .flat_map(|file_list| std::io::BufReader::new(file_list).lines()) - .collect(); - let lines = lines?; - - lines - .par_iter() - .try_for_each(|line| { - let mut diagnostics_writer = diagnostics.logger(); - let path = PathBuf::from(line).canonicalize()?; - let src_archive_file = crate::file_paths::path_for( - &self.source_archive_dir, - &path, - "", - path_transformer.as_ref(), - ); - let source = std::fs::read(&path)?; - let mut trap_writer = trap::Writer::new(); - - match path.file_name() { - None => { - tracing::error!(?path, "No file name found, skipping file."); - } - Some(filename) => { - let matches = globset.matches(filename); - if matches.is_empty() { - tracing::error!(?path, "No matching language found, skipping file."); - } else { - let mut languages_processed = vec![false; self.languages.len()]; - - for m in matches { - let i = glob_language_mapping[m]; - if languages_processed[i] { - continue; - } - languages_processed[i] = true; - let lang = &self.languages[i]; - - crate::extractor::extract( - &lang.ts_language, - lang.prefix, - &schemas[i], - &mut diagnostics_writer, - &mut trap_writer, - None, - &path, - &source, - &[], - lang.desugar.as_deref(), - ); - std::fs::create_dir_all(src_archive_file.parent().unwrap())?; - std::fs::copy(&path, &src_archive_file)?; - write_trap(&self.trap_dir, &path, &trap_writer, trap_compression)?; - } - } - } - } - Ok(()) as std::io::Result<()> - }) - .expect("failed to extract files"); - - let path = PathBuf::from("extras"); - let mut trap_writer = trap::Writer::new(); - crate::extractor::populate_empty_location(&mut trap_writer); - - let res = write_trap(&self.trap_dir, &path, &trap_writer, trap_compression); - tracing::info!("Extraction complete"); - res + driver::run_extractor( + &self.prefix, + &self.languages, + &self.trap_dir, + &self.source_archive_dir, + &self.file_lists, + &self.trap_compression, + ) } } - -fn write_trap( - trap_dir: &Path, - path: &Path, - trap_writer: &trap::Writer, - trap_compression: trap::Compression, -) -> std::io::Result<()> { - let trap_file = crate::file_paths::path_for(trap_dir, path, trap_compression.extension(), None); - std::fs::create_dir_all(trap_file.parent().unwrap())?; - trap_writer.write_to_file(&trap_file, trap_compression) -} diff --git a/shared/tree-sitter-extractor/tests/integration_test.rs b/shared/tree-sitter-extractor/tests/integration_test.rs index 694eb526f39..2b243ff7945 100644 --- a/shared/tree-sitter-extractor/tests/integration_test.rs +++ b/shared/tree-sitter-extractor/tests/integration_test.rs @@ -13,7 +13,6 @@ fn simple_extractor() { prefix: "ql", ts_language: tree_sitter_ql::LANGUAGE.into(), node_types: tree_sitter_ql::NODE_TYPES, - desugar: None, file_globs: vec!["*.qll".into()], }; diff --git a/shared/tree-sitter-extractor/tests/multiple_languages.rs b/shared/tree-sitter-extractor/tests/multiple_languages.rs index e345eec5828..2e45e56754a 100644 --- a/shared/tree-sitter-extractor/tests/multiple_languages.rs +++ b/shared/tree-sitter-extractor/tests/multiple_languages.rs @@ -13,14 +13,12 @@ fn multiple_language_extractor() { prefix: "ql", ts_language: tree_sitter_ql::LANGUAGE.into(), node_types: tree_sitter_ql::NODE_TYPES, - desugar: None, file_globs: vec!["*.qll".into()], }; let lang_json = simple::LanguageSpec { prefix: "json", ts_language: tree_sitter_json::LANGUAGE.into(), node_types: tree_sitter_json::NODE_TYPES, - desugar: None, file_globs: vec!["*.json".into(), "*Jsonfile".into()], }; diff --git a/shared/yeast-schema/src/node_types_yaml.rs b/shared/yeast-schema/src/node_types_yaml.rs index 5f6a3906f7c..97052f9a835 100644 --- a/shared/yeast-schema/src/node_types_yaml.rs +++ b/shared/yeast-schema/src/node_types_yaml.rs @@ -252,6 +252,41 @@ pub fn extend_schema_from_yaml( let yaml: YamlNodeTypes = serde_yaml::from_str(yaml_input).map_err(|e| format!("Failed to parse YAML: {e}"))?; apply_yaml_to_schema(&yaml, schema); + // The typed `YamlNodeTypes` stores each node's fields in a `BTreeMap` + // (alphabetical), losing the authored order. Re-parse as an ordered value to + // record the declared field order for presentation (see the AST dump). + record_field_order(schema, yaml_input)?; + Ok(()) +} + +/// Record each node kind's declared (named) field order from the source YAML, +/// which `serde`'s `BTreeMap`-based deserialization does not preserve. +/// `serde_yaml::Value` mappings keep insertion (source) order. +fn record_field_order(schema: &mut crate::schema::Schema, yaml_input: &str) -> Result<(), String> { + let value: serde_yaml::Value = serde_yaml::from_str(yaml_input) + .map_err(|e| format!("Failed to parse YAML for field order: {e}"))?; + let Some(named) = value.get("named").and_then(|v| v.as_mapping()) else { + return Ok(()); + }; + for (node_name, fields) in named { + let Some(node_name) = node_name.as_str() else { + continue; + }; + let Some(fields) = fields.as_mapping() else { + continue; // node with no fields (null) + }; + let mut order = Vec::new(); + for (raw_field_name, _) in fields { + let Some(raw) = raw_field_name.as_str() else { + continue; + }; + // Skip the unnamed/`child` slot; the dump handles it separately. + if let Some(name) = parse_field_name(raw).name { + order.push(schema.register_field(&name)); + } + } + schema.set_field_order(node_name, order); + } Ok(()) } diff --git a/shared/yeast-schema/src/schema.rs b/shared/yeast-schema/src/schema.rs index 9c438d7c22e..0675d891342 100644 --- a/shared/yeast-schema/src/schema.rs +++ b/shared/yeast-schema/src/schema.rs @@ -44,6 +44,11 @@ pub struct Schema { field_types: BTreeMap<(String, FieldId), Vec>, field_cardinalities: BTreeMap<(String, FieldId), FieldCardinality>, supertypes: BTreeMap>, + /// Per-node-kind declared field order (named fields only), as written in + /// the source node-types YAML. Field ids are not a stable ordering key + /// across front-ends, so this preserves the authored order for + /// presentation (see the AST dump). + field_order: BTreeMap>, } impl Default for Schema { @@ -65,6 +70,7 @@ impl Schema { field_types: BTreeMap::new(), field_cardinalities: BTreeMap::new(), supertypes: BTreeMap::new(), + field_order: BTreeMap::new(), } } @@ -269,6 +275,17 @@ impl Schema { .get(&(parent_kind.to_string(), field_id)) } + /// Record the declared (named) field order for a node kind, as authored in + /// the source node-types YAML. + pub fn set_field_order(&mut self, kind: &str, field_ids: Vec) { + self.field_order.insert(kind.to_string(), field_ids); + } + + /// The declared (named) field order for a node kind, if known. + pub fn field_order(&self, kind: &str) -> Option<&Vec> { + self.field_order.get(kind) + } + pub fn set_field_cardinality( &mut self, parent_kind: &str, diff --git a/shared/yeast/src/build.rs b/shared/yeast/src/build.rs index 508a1e73134..f4f5ae5d18f 100644 --- a/shared/yeast/src/build.rs +++ b/shared/yeast/src/build.rs @@ -108,6 +108,17 @@ impl<'a, C> BuildCtx<'a, C> { self.captures.get_all(name) } + /// Read the source text of a captured (or any other) node. + /// + /// Convenience for the common `ctx.ast.source_text(id)` reach-through used + /// by Rust-block rules that branch on, or embed, a raw capture's spelling + /// (e.g. an operator or keyword token). Resolves against the stored source + /// bytes, so it works for both parsed nodes and synthesized ones (via their + /// inherited source range). + pub fn source_text(&self, id: Id) -> String { + self.ast.source_text(id) + } + /// Create a named AST node with the given kind and fields. pub fn node(&mut self, kind: &str, fields: Vec<(&str, Vec)>) -> Id { let kind_id = self diff --git a/shared/yeast/src/dump.rs b/shared/yeast/src/dump.rs index 34b61432360..dc348f8789b 100644 --- a/shared/yeast/src/dump.rs +++ b/shared/yeast/src/dump.rs @@ -162,8 +162,12 @@ fn type_error_for_node( fn expected_for_field<'a>( schema: &'a Schema, parent_kind: &str, - field_id: u16, + field_name: &str, ) -> Option<&'a [crate::schema::NodeType]> { + // Resolve the field NAME in the validation schema's own id space, so the + // AST being dumped and the validation schema stay completely independent: + // they need not share field ids, only field names. + let field_id = schema.field_id_for_name(field_name)?; schema .field_types(parent_kind, field_id) .map(|v| v.as_slice()) @@ -223,15 +227,49 @@ fn dump_node( writeln!(out).unwrap(); - // Named fields first - for (&field_id, children) in &node.fields { - if field_id == CHILD_FIELD { - continue; // Handle unnamed children last + // Named fields first, in the schema's declared order when available + // (front-end-independent), else in field-id order. Any present fields not + // covered by the declared order are appended in field-id order. + // + // The declared order lives in the validation schema, keyed by *its* field + // ids; the AST being dumped may key the same field names under different + // ids. So map the declared order through field NAMES into this AST's own id + // space, keeping the two schemas independent (they share names, not ids). + let named_field_ids: Vec = { + let present: Vec = node + .fields + .keys() + .copied() + .filter(|&f| f != CHILD_FIELD) + .collect(); + match type_check.and_then(|(schema, _, _)| { + schema + .field_order(node.kind_name()) + .map(|order| (schema, order)) + }) { + Some((schema, order)) => { + let mut result: Vec = order + .iter() + .filter_map(|&f| schema.field_name_for_id(f)) + .filter_map(|name| ast.field_id_for_name(name)) + .filter(|&f| f != CHILD_FIELD && node.fields.contains_key(&f)) + .collect(); + for &f in &present { + if !result.contains(&f) { + result.push(f); + } + } + result + } + None => present, } + }; + for field_id in named_field_ids { + let children = &node.fields[&field_id]; let field_name = ast.field_name_for_id(field_id).unwrap_or("?"); let child_type_check = type_check.map(|(schema, _, _)| { - let expected = - expected_for_field(schema, node.kind_name(), field_id).or(Some(EMPTY_NODE_TYPES)); + let expected = expected_for_field(schema, node.kind_name(), field_name) + .or(Some(EMPTY_NODE_TYPES)); let parent_field = Some((node.kind_name(), field_name)); (schema, expected, parent_field) }); @@ -273,8 +311,14 @@ fn dump_node( // Check for required fields that are absent if let Some((schema, _, _)) = type_check { - for (field_id, field_name) in schema.required_fields_for_kind(node.kind_name()) { - if !node.fields.contains_key(&field_id) { + for (_field_id, field_name) in schema.required_fields_for_kind(node.kind_name()) { + let present = match field_name { + Some(n) => ast + .field_id_for_name(n) + .is_some_and(|fid| node.fields.contains_key(&fid)), + None => node.fields.contains_key(&CHILD_FIELD), + }; + if !present { let name = field_name.unwrap_or("child"); writeln!(out, "{prefix} <-- ERROR: missing required field '{name}'").unwrap(); } @@ -284,7 +328,9 @@ fn dump_node( // Unnamed children — skip unnamed tokens (keywords, punctuation) if let Some(children) = node.fields.get(&CHILD_FIELD) { let child_type_check = type_check.map(|(schema, _, _)| { - let expected = expected_for_field(schema, node.kind_name(), CHILD_FIELD) + let expected = schema + .field_types(node.kind_name(), CHILD_FIELD) + .map(|v| v.as_slice()) .or(Some(EMPTY_NODE_TYPES)); let parent_field = Some((node.kind_name(), "children")); (schema, expected, parent_field) diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index 6719a5498e2..14a0ab05576 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -1329,10 +1329,27 @@ impl DesugaringConfig { None => Ok(schema::from_language(language)), } } + + /// Build the yeast `Schema` from the output YAML alone, with no input + /// tree-sitter grammar. Requires `output_node_types_yaml` to be set (there + /// is no grammar to fall back to). Used by custom (non-tree-sitter) + /// front-ends whose input schema is supplied by their own parser/adapter. + pub fn build_schema_no_language(&self) -> Result { + match self.output_node_types_yaml { + Some(yaml) => node_types_yaml::schema_from_yaml(yaml), + None => Err( + "a language-free desugarer requires output_node_types_yaml to be set".to_string(), + ), + } + } } pub struct Runner<'a, C = ()> { - language: tree_sitter::Language, + /// The input tree-sitter language, used only when parsing (`run_from_tree` + /// / `run`). `None` for pipelines that only ever run over an + /// externally-built AST (`run_from_ast`), so no tree-sitter grammar is + /// required. + language: Option, schema: schema::Schema, phases: &'a [Phase], } @@ -1342,7 +1359,7 @@ impl<'a, C> Runner<'a, C> { pub fn new(language: tree_sitter::Language, phases: &'a [Phase]) -> Self { let schema = schema::from_language(&language); Self { - language, + language: Some(language), schema, phases, } @@ -1355,7 +1372,18 @@ impl<'a, C> Runner<'a, C> { phases: &'a [Phase], ) -> Self { Self { - language, + language: Some(language), + schema: schema.clone(), + phases, + } + } + + /// Create a runner with no input tree-sitter language, for pipelines that + /// only run over an externally-built AST (`run_from_ast`). The parsing + /// entry points (`run_from_tree` / `run`) will error if called. + pub fn with_schema_no_language(schema: &schema::Schema, phases: &'a [Phase]) -> Self { + Self { + language: None, schema: schema.clone(), phases, } @@ -1368,7 +1396,7 @@ impl<'a, C> Runner<'a, C> { ) -> Result { let schema = config.build_schema(&language)?; Ok(Self { - language, + language: Some(language), schema, phases: &config.phases, }) @@ -1388,7 +1416,9 @@ impl<'a, C: Clone> Runner<'a, C> { let mut ast = Ast::from_tree_with_schema_and_source( self.schema.clone(), tree, - &self.language, + self.language + .as_ref() + .ok_or("run_from_tree requires a tree-sitter language")?, source.to_vec(), ); self.run_phases(&mut ast, user_ctx)?; @@ -1398,9 +1428,13 @@ impl<'a, C: Clone> Runner<'a, C> { /// Parse `input` and run all phases, threading `user_ctx` through /// every rule transform. The caller owns the initial context state. pub fn run_with_ctx(&self, input: &str, user_ctx: &mut C) -> Result { + let language = self + .language + .as_ref() + .ok_or("run requires a tree-sitter language")?; let mut parser = tree_sitter::Parser::new(); parser - .set_language(&self.language) + .set_language(language) .map_err(|e| format!("Failed to set language: {e}"))?; let tree = parser .parse(input, None) @@ -1408,7 +1442,7 @@ impl<'a, C: Clone> Runner<'a, C> { let mut ast = Ast::from_tree_with_schema_and_source( self.schema.clone(), &tree, - &self.language, + language, input.as_bytes().to_vec(), ); self.run_phases(&mut ast, user_ctx)?; @@ -1506,7 +1540,10 @@ pub trait Desugarer: Send + Sync { /// schema so that per-call cost is bounded to constructing a transient /// [`Runner`] and cloning the schema (no YAML re-parsing). pub struct ConcreteDesugarer { - language: tree_sitter::Language, + /// The input tree-sitter language, or `None` for a custom (non-tree-sitter) + /// front-end that only ever desugars an externally-built AST + /// (`run_from_ast`). + language: Option, schema: schema::Schema, config: DesugaringConfig, } @@ -1520,7 +1557,20 @@ impl ConcreteDesugarer { ) -> Result { let schema = config.build_schema(&language)?; Ok(Self { - language, + language: Some(language), + schema, + config, + }) + } + + /// Build a desugarer with no input tree-sitter language, for a custom + /// front-end whose parser supplies the input AST directly. Only + /// [`run_from_ast`](Desugarer::run_from_ast) is supported; the schema comes + /// from the config's `output_node_types_yaml` (which must be set). + pub fn without_language(config: DesugaringConfig) -> Result { + let schema = config.build_schema_no_language()?; + Ok(Self { + language: None, schema, config, }) @@ -1533,7 +1583,11 @@ impl Desugarer for ConcreteDesugarer } fn run_from_tree(&self, tree: &tree_sitter::Tree, source: &[u8]) -> Result { - let runner = Runner::with_schema(self.language.clone(), &self.schema, &self.config.phases); + let language = self + .language + .clone() + .ok_or("run_from_tree requires a tree-sitter language")?; + let runner = Runner::with_schema(language, &self.schema, &self.config.phases); runner.run_from_tree(tree, source) } @@ -1541,7 +1595,8 @@ impl Desugarer for ConcreteDesugarer // The AST was built against its own (external) schema; make sure the // output kind/field names the rules build are resolvable in it. ast.register_names_from_schema(&self.schema); - let runner = Runner::with_schema(self.language.clone(), &self.schema, &self.config.phases); + // `run_from_ast` never parses, so no tree-sitter language is needed. + let runner = Runner::with_schema_no_language(&self.schema, &self.config.phases); runner.run_from_ast(ast) } } diff --git a/unified/BUILD.bazel b/unified/BUILD.bazel index e702fd25159..405500d7d40 100644 --- a/unified/BUILD.bazel +++ b/unified/BUILD.bazel @@ -1,5 +1,6 @@ load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup") load("//misc/bazel:pkg.bzl", "codeql_pack", "codeql_pkg_files") +load("//misc/bazel:utils.bzl", "select_os") package(default_visibility = ["//visibility:public"]) @@ -46,12 +47,26 @@ codeql_pkg_files( prefix = "tools/{CODEQL_PLATFORM}", ) +# The Swift front-end parser (wrapper + real binary + bundled Swift runtime), +# shipped next to the extractor. Only on platforms where swift-syntax builds +# (Linux/macOS); elsewhere the group is empty so the pack still builds (Swift +# extraction is simply unavailable there). +pkg_filegroup( + name = "swift-syntax-parse-arch", + srcs = select_os( + otherwise = [], + posix = ["//unified/swift-syntax-rs:swift-syntax-parse-pkg"], + ), + prefix = "tools/{CODEQL_PLATFORM}", +) + codeql_pack( name = "unified", srcs = [ ":codeql-extractor-yml", ":dbscheme-group", ":extractor-arch", + ":swift-syntax-parse-arch", "//unified/tools", ], ) diff --git a/unified/extractor/BUILD.bazel b/unified/extractor/BUILD.bazel index 0ef7f1b68b0..13a3b6b287b 100644 --- a/unified/extractor/BUILD.bazel +++ b/unified/extractor/BUILD.bazel @@ -7,7 +7,10 @@ codeql_rust_binary( name = "extractor", srcs = glob(["src/**/*.rs"]), aliases = aliases(), - compile_data = ["ast_types.yml"], + compile_data = [ + "ast_types.yml", + "swift_node_types.yml", + ], proc_macro_deps = all_crate_deps( proc_macro = True, ), diff --git a/unified/extractor/ast_types.yml b/unified/extractor/ast_types.yml index 418772aa268..4fa1ff16942 100644 --- a/unified/extractor/ast_types.yml +++ b/unified/extractor/ast_types.yml @@ -31,6 +31,7 @@ supertypes: - throw_expr - try_expr - switch_expr + - unresolved_operator_sequence - unsupported_node expr_or_pattern: - expr @@ -38,6 +39,11 @@ supertypes: expr_or_type: - expr - type_expr + # An element of an `unresolved_operator_sequence`: either an operand (`expr`) + # or one of the infix operators separating the operands. + expr_or_operator: + - expr + - infix_operator pattern: - name_pattern - tuple_pattern @@ -137,6 +143,19 @@ named: operand: expr operator: operator + # A flat, unresolved operator sequence such as `a <+> b <+> c`. + # + # Swift's grammar doesn't encode operator precedence, so an operator chain is + # first parsed as a flat list of operands and operators. The parser front-end + # resolves this into structured `binary_expr` trees when it knows the + # operators' precedence (standard-library operators, and operators declared in + # the same file). When it encounters an operator whose precedence it can't + # determine (e.g. one imported from another module), it leaves that chain + # unresolved and emits it here rather than guessing a (possibly wrong) + # structure. The `element`s alternate operands (`expr`) and infix operators. + unresolved_operator_sequence: + element*: expr_or_operator + # Plain assignment assign_expr: target: expr_or_pattern diff --git a/unified/extractor/src/extractor.rs b/unified/extractor/src/extractor.rs index 301c6cf533f..82bfe81219b 100644 --- a/unified/extractor/src/extractor.rs +++ b/unified/extractor/src/extractor.rs @@ -2,7 +2,7 @@ use clap::Args; use std::path::PathBuf; use crate::languages; -use codeql_extractor::extractor::simple; +use codeql_extractor::extractor::desugaring; use codeql_extractor::trap; #[derive(Args)] @@ -31,7 +31,7 @@ pub fn run(options: Options) -> std::io::Result<()> { lang.prefix = "unified"; } - let extractor = simple::Extractor { + let extractor = desugaring::Extractor { prefix: "unified".to_string(), languages, trap_dir: options.output_dir, diff --git a/unified/extractor/src/languages/mod.rs b/unified/extractor/src/languages/mod.rs index 52a1bd40ffc..c79806dc9d4 100644 --- a/unified/extractor/src/languages/mod.rs +++ b/unified/extractor/src/languages/mod.rs @@ -1,20 +1,21 @@ -use codeql_extractor::extractor::simple; +use codeql_extractor::extractor::desugaring; #[path = "swift/swift.rs"] mod swift; /// swift-syntax JSON -> `yeast::Ast` adapter for the Swift front-end. -/// -/// Currently exercised by tests and the forthcoming runtime extraction path; -/// `allow(dead_code)` because this is a binary crate, so its public API isn't -/// counted as used until the binary itself calls it. #[path = "swift/adapter.rs"] -#[allow(dead_code)] pub mod swift_adapter; +/// Swift front-end parser: shells out to `swift-syntax-parse` and adapts its +/// JSON output via [`swift_adapter`]. This is the live Swift front-end used by +/// [`all_language_specs`]. +#[path = "swift/parse.rs"] +pub mod swift_parse; + /// Shared YEAST output AST schema for all languages. pub(crate) const OUTPUT_AST_SCHEMA: &str = include_str!("../../ast_types.yml"); -pub fn all_language_specs() -> Vec { +pub fn all_language_specs() -> Vec { vec![swift::language_spec(OUTPUT_AST_SCHEMA)] } diff --git a/unified/extractor/src/languages/swift/adapter.rs b/unified/extractor/src/languages/swift/adapter.rs index 37d5aac00d2..4ae4040d043 100644 --- a/unified/extractor/src/languages/swift/adapter.rs +++ b/unified/extractor/src/languages/swift/adapter.rs @@ -2,9 +2,8 @@ //! in-memory format the CodeQL desugaring rules operate on. //! //! The JSON tree is produced by the `swift-syntax-rs` crate's Swift FFI shim -//! (`parse_to_json`). This module is pure Rust (only `yeast` + `serde_json`), -//! so the extractor consumes swift-syntax output without pulling in the Swift -//! toolchain (the JSON is produced out-of-process). +//! (`parse_to_json`). This module needs no Swift toolchain, so the extractor +//! consumes swift-syntax output out-of-process. //! //! The mapping mirrors tree-sitter's node model, which is what yeast (and the //! extractor's rewrite rules) expect: @@ -24,31 +23,19 @@ use std::collections::BTreeMap; +use codeql_extractor::extractor::ExtraToken; use serde_json::Value; -use yeast::schema::Schema; use yeast::{Ast, Id, NodeContent, Point, Range}; -/// A comment (or `unexpectedText`) recovered from the syntax tree's trivia. -/// -/// These are collected into a side channel rather than embedded in the -/// [`yeast::Ast`], mirroring how the extractor treats tree-sitter `extra` -/// nodes: they carry a location and text but are not attached to a parent. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct TriviaToken { - /// The trivia kind (e.g. `lineComment`, `blockComment`, `docLineComment`, - /// `docBlockComment`, `unexpectedText`). - pub kind: String, - /// The verbatim source text of the piece (e.g. `// comment`). - pub text: String, - /// The source range the piece occupies. - pub range: Range, -} - /// The result of adapting a swift-syntax JSON tree: the [`yeast::Ast`] plus the -/// comment/`unexpectedText` trivia harvested from it (in source order). +/// comment/`unexpectedText` [`ExtraToken`]s harvested from it (in source order). +/// +/// The extra tokens are collected into a side channel rather than embedded in +/// the [`yeast::Ast`], mirroring how the extractor treats tree-sitter `extra` +/// nodes: they carry a location and text but are not attached to a parent. pub struct AdaptedTree { pub ast: Ast, - pub trivia: Vec, + pub extras: Vec, } /// swift-syntax `TokenKind` cases whose text is *not* determined by the kind @@ -170,17 +157,18 @@ fn children_of(value: &Value) -> Vec<&Value> { /// in the schema on the fly, immediately before the node is created. Children /// are built first so a parent's field lists reference existing ids. Any /// comment/`unexpectedText` trivia carried by a token is harvested into -/// `trivia` during the same pass rather than embedded in the tree. -fn build(node: &Value, ast: &mut Ast, trivia: &mut Vec) -> Result { +/// `extras` (as [`ExtraToken`]s) during the same pass rather than embedded in +/// the tree. +fn build(node: &Value, ast: &mut Ast, extras: &mut Vec) -> Result { let info = classify(node)?; - collect_trivia(node, trivia); + collect_extras(node, extras); let mut fields: BTreeMap> = BTreeMap::new(); for (field, value) in field_entries(node) { let field_id = ast.register_field(field); let mut ids = Vec::new(); for child in children_of(value) { - ids.push(build(child, ast, trivia)?); + ids.push(build(child, ast, extras)?); } fields.insert(field_id, ids); } @@ -201,9 +189,10 @@ fn build(node: &Value, ast: &mut Ast, trivia: &mut Vec) -> Result) { +/// filtered to comments/`unexpectedText` upstream) into `out` as +/// [`ExtraToken`]s. Non-token nodes have no trivia keys, so this is a no-op for +/// them. +fn collect_extras(node: &Value, out: &mut Vec) { for key in ["leadingTrivia", "trailingTrivia"] { let Some(Value::Array(pieces)) = node.get(key) else { continue; @@ -220,8 +209,8 @@ fn collect_trivia(node: &Value, out: &mut Vec) { .and_then(Value::as_str) .unwrap_or("") .to_string(); - out.push(TriviaToken { - kind: kind.to_string(), + out.push(ExtraToken { + kind: trivia_kind_id(kind), text, range, }); @@ -229,6 +218,21 @@ fn collect_trivia(node: &Value, out: &mut Vec) { } } +/// Map a swift-syntax trivia kind name to the stable integer id stored in an +/// [`ExtraToken`]'s `kind` (and written to the `unified_trivia_tokeninfo` +/// table). The value is opaque to the QL library (which reads only the text), +/// but is kept stable and meaningful. +fn trivia_kind_id(kind: &str) -> usize { + match kind { + "lineComment" => 1, + "blockComment" => 2, + "docLineComment" => 3, + "docBlockComment" => 4, + "unexpectedText" => 5, + _ => 0, + } +} + /// Parse a node's `range` into a [`yeast::Range`]. /// /// The JSON carries, for `start` and `end`, a 0-based UTF-8 file byte `offset`, @@ -258,21 +262,29 @@ fn parse_range(node: &Value) -> Option { }) } +/// The authoritative swift-syntax input node-types schema. +/// [`json_to_ast`] seeds every parse with the schema built from this, +/// pre-registering every input kind and field so rule matching never references +/// a name absent from a given file's tree. +const SWIFT_NODE_TYPES: &str = include_str!("../../../swift_node_types.yml"); + /// Convert a swift-syntax JSON tree (as produced by [`crate::parse_to_json`]) /// into a [`yeast::Ast`] plus the comment/`unexpectedText` trivia harvested -/// from it. Both are produced in a single traversal. +/// from it. Both are produced in a single traversal. The AST is seeded with the +/// authoritative swift-syntax schema ([`SWIFT_NODE_TYPES`]); the adapter only +/// ever consumes swift-syntax input, so the schema is not a parameter. pub fn json_to_ast(json: &str) -> Result { let root: Value = serde_json::from_str(json).map_err(|e| format!("invalid JSON: {e}"))?; - let mut ast = Ast::with_schema(Schema::new()); - let mut trivia = Vec::new(); - let root_id = build(&root, &mut ast, &mut trivia)?; + let mut ast = Ast::with_schema(yeast::node_types_yaml::schema_from_yaml(SWIFT_NODE_TYPES)?); + let mut extras = Vec::new(); + let root_id = build(&root, &mut ast, &mut extras)?; ast.set_root(root_id); - // Emit trivia in source order (the traversal visits nodes bottom-up). - trivia.sort_by_key(|t| t.range.start_byte); + // Emit extras in source order (the traversal visits nodes bottom-up). + extras.sort_by_key(|t| t.range.start_byte); - Ok(AdaptedTree { ast, trivia }) + Ok(AdaptedTree { ast, extras }) } #[cfg(test)] @@ -373,7 +385,7 @@ mod tests { } #[test] - fn collects_trivia_into_side_channel() { + fn collects_extras_into_side_channel() { // A token carrying a trailing line comment in its trivia. let json = r#"{ "kind": "sourceFile", @@ -395,9 +407,10 @@ mod tests { let adapted = json_to_ast(json).expect("adapter should succeed"); // The comment is in the side channel, with its text and location. - assert_eq!(adapted.trivia.len(), 1); - let comment = &adapted.trivia[0]; - assert_eq!(comment.kind, "lineComment"); + assert_eq!(adapted.extras.len(), 1); + let comment = &adapted.extras[0]; + // `lineComment` maps to extra kind id 1. + assert_eq!(comment.kind, 1); assert_eq!(comment.text, "// c"); assert_eq!(comment.range.start_byte, 2); assert_eq!(comment.range.end_byte, 6); diff --git a/unified/extractor/src/languages/swift/parse.rs b/unified/extractor/src/languages/swift/parse.rs new file mode 100644 index 00000000000..c8633f1a27a --- /dev/null +++ b/unified/extractor/src/languages/swift/parse.rs @@ -0,0 +1,121 @@ +//! Swift front-end parser: shells out to the separate `swift-syntax-parse` +//! binary (which links swift-syntax) to obtain a JSON syntax tree, then adapts +//! that JSON into a `yeast::Ast` via the pure-Rust [`swift_adapter`] module. +//! +//! Running the parser in a separate process keeps the Swift toolchain out of +//! the extractor's own build: the extractor never links Swift, so working on +//! other (e.g. tree-sitter based) languages needs no Swift toolchain. Each call +//! spawns the parser afresh; a longer-lived parser process could be swapped in +//! behind this same seam later without touching the extraction pipeline. + +use std::io::Write; +use std::process::{Command, Stdio}; + +use codeql_extractor::extractor::ParsedTree; + +use super::swift_adapter; + +/// Environment variable naming the `swift-syntax-parse` executable. When unset, +/// the parser is resolved next to the extractor executable, then on `PATH`. +const PARSE_BIN_ENV: &str = "CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE"; + +/// Base name of the `swift-syntax-parse` executable as shipped / looked up. +const PARSE_BIN_NAME: &str = "swift-syntax-parse"; + +/// Parse Swift `source` into a [`ParsedTree`] (a raw `yeast::Ast` plus +/// side-channel `extra` tokens), ready to be desugared via `run_from_ast`. +pub fn parse(source: &[u8]) -> Result { + let source = + std::str::from_utf8(source).map_err(|e| format!("Swift source is not valid UTF-8: {e}"))?; + let json = run_parser(source)?; + let mut adapted = swift_adapter::json_to_ast(&json)?; + adapted.ast.set_source(source.as_bytes().to_vec()); + Ok(ParsedTree { + ast: adapted.ast, + extras: adapted.extras, + }) +} + +/// The `swift-syntax-parse` executable to invoke, resolved in priority order: +/// +/// 1. the `CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE` override, if set; +/// 2. a copy shipped next to the extractor executable — this is how the CodeQL +/// extractor pack lays it out (`tools//{extractor, +/// swift-syntax-parse}`), so a packaged extractor is self-contained with no +/// environment setup; +/// 3. a bare `swift-syntax-parse`, looked up on `PATH`. +fn parse_bin() -> String { + if let Ok(bin) = std::env::var(PARSE_BIN_ENV) { + if !bin.is_empty() { + return bin; + } + } + if let Ok(exe) = std::env::current_exe() { + if let Some(sibling) = exe.parent().map(|dir| dir.join(PARSE_BIN_NAME)) { + if sibling.is_file() { + return sibling.to_string_lossy().into_owned(); + } + } + } + PARSE_BIN_NAME.to_string() +} + +/// Whether the `swift-syntax-parse` executable can be launched at all. +/// +/// This reports availability of the *executable*, deliberately not whether +/// parsing succeeds: a binary that launches but then crashes or emits invalid +/// JSON is still "available", so callers run and surface the failure rather +/// than silently skipping. Only a genuinely missing/unlaunchable binary (e.g. +/// no Swift toolchain is installed) reports `false`. +pub fn binary_available() -> bool { + match Command::new(parse_bin()) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + { + Ok(mut child) => { + let _ = child.wait(); + true + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => false, + // Any other spawn failure (e.g. a permissions problem) is a genuine + // issue worth surfacing, so treat the parser as available and let the + // caller fail rather than masking it as "unavailable". + Err(_) => true, + } +} + +/// Run the external parser, feeding `source` on stdin and returning its JSON +/// stdout. +fn run_parser(source: &str) -> Result { + let bin = parse_bin(); + let mut child = Command::new(&bin) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("failed to spawn Swift parser `{bin}`: {e}"))?; + + // The parser reads all of stdin before writing any stdout, so writing the + // whole source and then closing stdin (by dropping it) cannot deadlock. + child + .stdin + .take() + .expect("child stdin was piped") + .write_all(source.as_bytes()) + .map_err(|e| format!("failed to write source to Swift parser `{bin}`: {e}"))?; + + let output = child + .wait_with_output() + .map_err(|e| format!("failed to run Swift parser `{bin}`: {e}"))?; + if !output.status.success() { + return Err(format!( + "Swift parser `{bin}` failed ({}): {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + )); + } + String::from_utf8(output.stdout) + .map_err(|e| format!("Swift parser produced non-UTF-8 output: {e}")) +} diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index 13f1a6beadf..ef3672e1b91 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -1,4 +1,4 @@ -use codeql_extractor::extractor::simple; +use codeql_extractor::extractor::desugaring; use yeast::{ConcreteDesugarer, DesugaringConfig, PhaseKind, Rule, rule, tree}; /// User context propagated from outer rules down to the inner rules that @@ -8,50 +8,42 @@ use yeast::{ConcreteDesugarer, DesugaringConfig, PhaseKind, Rule, rule, tree}; /// post-hoc mutation. #[derive(Clone, Default)] struct SwiftContext { - /// Identifier node for the property name. Set by the outer - /// `property_binding` (computed accessors / willSet-didSet) and - /// `protocol_property_declaration` rules before translating accessor - /// children; read by the accessor inner rules - /// (`computed_getter`/`computed_setter`/`computed_modify`/ - /// `willset_clause`/`didset_clause`/`getter_specifier`/ - /// `setter_specifier`). + /// Identifier node for the property name. Set by the accessor-bearing + /// `variableDecl` rule before translating the accessor block; read by the + /// inner `accessorDecl` rules to name each `accessor_declaration`. property_name: Option, - /// Translated type node for the property type. Set by the outer - /// `property_binding` rule (computed accessors variant) and - /// `protocol_property_declaration` when present; read by the - /// accessor inner rules. + /// Translated type node for the property type. Set (for computed + /// properties) by the accessor-bearing `variableDecl` rule; read by the + /// inner `accessorDecl` rules. Left `None` for stored properties with + /// observers, so their `willSet`/`didSet` accessors carry no type. property_type: Option, - /// Default-value expression for the next translated `parameter`. Set - /// by the outer `function_parameter` rule; read by the `parameter` - /// rules. - default_value: Option, /// Translated outer modifiers to attach to each child of a flattening - /// outer rule. Set by `property_declaration`, `binding_pattern`, - /// `enum_entry`, and `protocol_property_declaration`. For `let`/`var` - /// declarations and `binding_pattern`s the list is led by the binding - /// modifier, which also serves as the "this is a binding" signal for - /// pattern translation (see `in_binding_pattern`). + /// outer rule — e.g. the `let`/`var` binding modifier on each + /// `patternBinding` of a `variableDecl`, or the binding modifier on each + /// accessor of a property. outer_modifiers: Vec, /// True when the current child of a flattening outer rule is not /// the first one — its inner rule should emit a /// `chained_declaration` modifier so the original grouping can be /// recovered downstream. is_chained: bool, + /// True while translating the parameters of a `functionType`. swift-syntax + /// models a function type's parameters with the same `tupleTypeElement` + /// kind as a tuple type's elements, so the shared `tupleTypeElement` rule + /// reads this to emit a `parameter` (function-type param) rather than a + /// `tuple_type_element` (tuple-type element). The `tupleType` / + /// `functionType` rules each set it for their direct children, so nested + /// types are translated in the correct context. + in_function_type: bool, + /// True while translating the argument list of an enum-case + /// `constructor_pattern` (e.g. `case .foo(let x, 3)`). Read by the + /// `labeledExpr` rules so a bare expression argument becomes an + /// `expr_equality_pattern` (wrapped in a `pattern_element`) rather than a + /// call `argument`. + in_pattern: bool, } impl SwiftContext { - /// Whether the pattern currently being translated is a binding - /// (the LHS of a `let`/`var` declaration or a `binding_pattern`). - /// - /// True exactly when an enclosing binding has published its modifier into - /// `outer_modifiers`. This is reliable because non-binding subtrees - /// (bodies, initializer values, ...) are translated after resetting the - /// context (see `reset`), so a bare identifier only sees a - /// non-empty `outer_modifiers` when it really is a binding. - fn in_binding_pattern(&self) -> bool { - !self.outer_modifiers.is_empty() - } - /// Clear the context fields that must not propagate into an /// expression / statement / body subtree. /// @@ -77,7 +69,7 @@ impl SwiftContext { /// rule. Returns `Option` so it splices via `{…}` to 0 or 1 ids. fn chained_modifier(ctx: &mut yeast::build::BuildCtx<'_, SwiftContext>) -> Option { if ctx.is_chained { - Some(ctx.literal("modifier", "chained_declaration")) + Some(tree!((modifier "chained_declaration"))) } else { None } @@ -121,28 +113,19 @@ fn member_chain( ) } +/// Compound-assignment operator spellings (`+=`, `<<=`, ...). Used to tell a +/// compound assignment from an ordinary binary application, both of which +/// arrive as a `binaryOperator`-based `infixOperatorExpr`. +const COMPOUND_ASSIGN_OPS: &[&str] = &[ + "+=", "-=", "*=", "/=", "%=", "<<=", ">>=", "&=", "|=", "^=", "&+=", "&-=", "&*=", +]; + fn translation_rules() -> Vec> { vec![ // ---- Top-level ---- - // Capture all top-level statements, including unnamed tokens like `nil`. - rule!( - (source_file statement: _* @children) - => - (top_level - body: (block stmt: {children}) - ) - ), - // Declarations may be wrapped in local/global wrapper nodes. - rule!((global_declaration _ @inner) => stmt { inner }), - rule!((local_declaration _ @inner) => stmt { inner }), - // ---- swift-syntax front-end (minimal hook-up) ---- - // These rules target the swift-syntax AST (camelCase kind names), - // produced by the sibling `adapter` module. They coexist with the - // tree-sitter rules (snake_case names): rules are dispatched by exact - // kind name, and the two name spaces never collide, so these are inert - // on the tree-sitter path. Only the minimal top-level mapping lives here - // to demonstrate the pipeline end-to-end; the full translation is added - // separately. Unmatched swift-syntax nodes fall through to the + // These rules translate the swift-syntax AST (camelCase kind names), + // produced by the sibling `adapter` module from the `swift-syntax-parse` + // binary's JSON. Anything unmatched falls through to the // `unsupported_node` fallback at the end. // // `sourceFile` holds its top-level statements in an (elided) @@ -153,199 +136,221 @@ fn translation_rules() -> Vec> { => (top_level body: (block stmt: {items})) ), - rule!((codeBlockItem item: @item) => stmt { item }), + // `codeBlockItem` wraps a top-level statement. It is a simple unwrapper, + // but a single wrapped `variableDecl` can translate to *several* + // declarations (`let x = 1, y = 2`), so the wrapped node is captured with + // `_*` and the result annotated `stmt*` to splice all of them. + rule!((codeBlockItem item: _* @item) => stmt* { item }), // ---- Literals ---- - rule!((integer_literal) => (int_literal)), - rule!((hex_literal) => (int_literal)), - rule!((bin_literal) => (int_literal)), - rule!((oct_literal) => (int_literal)), - rule!((real_literal) => (float_literal)), - rule!((boolean_literal) => (boolean_literal)), - rule!("nil" => (builtin_expr)), - rule!((special_literal) => (builtin_expr)), - rule!((line_string_literal) => (string_literal)), - rule!((multi_line_string_literal) => (string_literal)), - rule!((raw_string_literal) => (string_literal)), - rule!((regex_literal) => (regex_literal)), + // swift-syntax does not distinguish the lexical integer/string forms + // (hex/binary/octal, single- vs multi-line, raw): each is a single + // `*LiteralExpr` kind, so the tree-sitter variants collapse to one rule. + rule!((integerLiteralExpr) => (int_literal)), + rule!((floatLiteralExpr) => (float_literal)), + rule!((booleanLiteralExpr) => (boolean_literal)), + rule!((nilLiteralExpr) => (builtin_expr)), + rule!((stringLiteralExpr) => (string_literal)), + rule!((regexLiteralExpr) => (regex_literal)), // ---- Names ---- - rule!((simple_identifier) @id => (name_expr identifier: (identifier #{id}))), - // A referenceable_operator (e.g. `+` used as a value, as in `reduce(0, +)`) - // is treated as a name reference to the operator symbol. - rule!((referenceable_operator) @op => (name_expr identifier: (identifier #{op}))), + // A function reference spelled with argument labels (`f(x:y:z:)`) is a + // `declReferenceExpr` carrying `argumentNames`. Mark it unsupported for + // now (rather than let the bare-name rule below treat it as a plain + // reference), so downstream QL isn't handed a malformed reference. In + // the future this should become a lambda expression. Matched before the + // bare-name rule. + rule!( + (declReferenceExpr argumentNames: (declNameArguments)) + => + (unsupported_node) + ), + // A bare name reference (`x`), and an operator used as a value (`+` in + // `reduce(0, +)`), are both `declReferenceExpr`; its `baseName` is the + // referenced identifier / operator symbol. + rule!((declReferenceExpr baseName: @name) => (name_expr identifier: (identifier #{name}))), + // A discard `_` used as an expression — e.g. the target of a discarding + // assignment `_ = x`. swift-syntax models it as a `discardAssignmentExpr`; + // the tree-sitter path treated the bare `_` as a name, so map it to a + // `name_expr` too. + rule!((discardAssignmentExpr wildcard: @@w) => (name_expr identifier: (identifier #{w}))), // ---- Operators ---- - // All binary operators share the lhs/op/rhs shape. - rule!((additive_expression lhs: @l op: @op rhs: @r) => (binary_expr left: {l} operator: (infix_operator #{op}) right: {r})), - rule!((multiplicative_expression lhs: @l op: @op rhs: @r) => (binary_expr left: {l} operator: (infix_operator #{op}) right: {r})), - rule!((comparison_expression lhs: @l op: @op rhs: @r) => (binary_expr left: {l} operator: (infix_operator #{op}) right: {r})), - rule!((equality_expression lhs: @l op: @op rhs: @r) => (binary_expr left: {l} operator: (infix_operator #{op}) right: {r})), - rule!((conjunction_expression lhs: @l op: @op rhs: @r) => (binary_expr left: {l} operator: (infix_operator #{op}) right: {r})), - rule!((disjunction_expression lhs: @l op: @op rhs: @r) => (binary_expr left: {l} operator: (infix_operator #{op}) right: {r})), - rule!((infix_expression lhs: @l op: @op rhs: @r) => (binary_expr left: {l} operator: (infix_operator #{op}) right: {r})), - // Range expression `a.. (binary_expr left: {l} operator: (infix_operator #{op}) right: {r})), - // Open-ended ranges `a...` / `...b` - rule!((open_end_range_expression start: @l) => (unary_expr operator: (postfix_operator "...") operand: {l})), - rule!((open_start_range_expression end: @r) => (unary_expr operator: (prefix_operator "...") operand: {r})), - // Custom operator declaration: `[prefix|infix|postfix] operator OP [: PrecedenceGroup]`. - // The fixity keyword is an anonymous child of `operator_declaration`, so we - // dispatch on it with one rule per keyword. - rule!( - (operator_declaration "prefix" (referenceable_operator _ @op) (simple_identifier)? @prec) - => - (operator_syntax_declaration name: (identifier #{op}) fixity: (fixity "prefix") precedence: {prec}) - ), - rule!( - (operator_declaration "postfix" (referenceable_operator _ @op) (simple_identifier)? @prec) - => - (operator_syntax_declaration name: (identifier #{op}) fixity: (fixity "postfix") precedence: {prec}) - ), - rule!( - (operator_declaration "infix" (referenceable_operator _ @op) (simple_identifier)? @prec) - => - (operator_syntax_declaration - name: (identifier #{op}) - fixity: (fixity "infix") - precedence: {prec}) - ), - rule!((bitwise_operation lhs: @l op: @op rhs: @r) => (binary_expr left: {l} operator: (infix_operator #{op}) right: {r})), - rule!((nil_coalescing_expression value: @l if_nil: @r) => (binary_expr left: {l} operator: (infix_operator "??") right: {r})), - // Leading-dot member shorthand (e.g. `.some`, `.foo`) means member access - // on a contextually inferred type. - rule!((prefix_expression operation: "." target: @member) => (member_access_expr base: (inferred_type_expr) member: (identifier #{member}))), - // Prefix unary operators - rule!((prefix_expression operation: @op target: @operand) => (unary_expr operator: (prefix_operator #{op}) operand: {operand})), - // Postfix unary operators - rule!((postfix_expression operation: @op target: @operand) => (unary_expr operator: (postfix_operator #{op}) operand: {operand})), - // TODO: Parenthesised single-value tuple is a grouping expression and should pass through. - // Multi-value tuples become tuple_expr. - rule!((tuple_expression value: _* @v) => (tuple_expr element: {v})), - // Blocks contain statement* directly. - rule!((block statement: _+ @stmts) => (block stmt: {stmts})), - rule!((block) => (block)), - // ---- Variables ---- - // property_binding rules — these produce variable_declaration and/or accessor_declaration - // nodes for individual declarators. The outer property_declaration rule splices these out - // and attaches binding/modifiers from the parent. - - // Computed property with explicit accessors (get/set/modify) → a - // sequence of `accessor_declaration` nodes. The outer rule - // publishes the property's name and type into `ctx` so that each - // inner accessor rule - // (`computed_getter`/`computed_setter`/`computed_modify`) builds - // its `accessor_declaration` with `name` and `type` set from the - // start — no schema-invalid intermediate state. + // The parser front-end folds operator chains into nested + // `infixOperatorExpr`s by precedence (see swift-syntax-rs), so + // `1 + 2 * 3` arrives here already structured. // - // Toggles `ctx.is_chained` per accessor iteration: the first - // accessor inherits the outer rule's chained state (i.e. whether - // this whole property_binding is itself a non-first declarator - // of a containing property_declaration); subsequent accessors - // always emit `chained_declaration`. + // A `binaryOperatorExpr` wraps the operator token; unwrap it to the + // operator leaf. Used by `infixOperatorExpr` (folded) and `sequenceExpr` + // (unresolved). + rule!((binaryOperatorExpr operator: @op) => (infix_operator #{op})), + // Compound assignment (`x += y`) vs. an ordinary binary application + // (`a + b`): both are `binaryOperator`-based `infixOperatorExpr`s, + // distinguishable only by the operator's spelling. The query engine + // can't match on token text, so a small Rust block reads the spelling + // and routes to `compound_assign_expr` or `binary_expr`. The operator + // is captured raw (`@@op`) to read its spelling. rule!( - (property_binding - name: @pattern - type: _? @ty - computed_value: (computed_property accessor: _+ @@accessors)) + (infixOperatorExpr leftOperand: @l operator: (binaryOperatorExpr) @@op rightOperand: @r) => - accessor_declaration* { - ctx.property_name = Some(tree!((identifier #{pattern}))); - ctx.property_type = ty; - + expr { + if COMPOUND_ASSIGN_OPS.contains(&ctx.source_text(op).as_str()) { + tree!((compound_assign_expr target: {l} operator: (infix_operator #{op}) value: {r})) + } else { + tree!((binary_expr left: {l} operator: (infix_operator #{op}) right: {r})) + } + } + ), + // Plain assignment (`x = y`). In a folded chain the `=` is an + // `assignmentExpr` node (distinct from other operators), matched by kind. + rule!( + (infixOperatorExpr leftOperand: @l operator: (assignmentExpr) rightOperand: @r) + => + (assign_expr target: {l} value: {r}) + ), + // Escape hatch: an operator chain the front-end could not resolve + // (because it uses an operator of unknown precedence, e.g. imported from + // another module) stays a flat `sequenceExpr`. Preserve it as an + // `unresolved_operator_sequence` whose elements alternate operands and + // infix operators, rather than guessing a structure. + rule!((sequenceExpr elements: _* @els) => (unresolved_operator_sequence element: {els})), + // Prefix unary operators (`!a`, `-x`). + rule!((prefixOperatorExpr operator: @op expression: @operand) => (unary_expr operator: (prefix_operator #{op}) operand: {operand})), + // A `tupleExpr` is a tuple literal (`(a, b)`) or a parenthesised + // expression (`(x)`). For now it is kept as an opaque `tuple_expr` leaf + // (its source text); its elements are not descended into. + // + // TODO: a parenthesised single-element `tupleExpr` is really a grouping + // expression and should be elided (unwrapped to its inner expression) + // rather than modelled as a tuple. + rule!((tupleExpr) => (tuple_expr)), + // A code block contains its statements directly. + rule!((codeBlock statements: _* @stmts) => (block stmt: {stmts})), + // ---- Properties with accessors ---- + // A computed property with an implicit getter (`var a: T { }`) + // becomes a single `accessor_declaration` of kind `get`. This form is + // self-contained (no context threading). It must precede the plain + // `variableDecl` rules, which would otherwise match and drop the accessor + // block. + rule!( + (variableDecl + bindingSpecifier: @@spec + bindings: (patternBinding + pattern: (identifierPattern identifier: @@name) + typeAnnotation: (typeAnnotation type: @ty) + accessorBlock: (accessorBlock accessors: (codeBlockItem)+ @body))) + => + (accessor_declaration + modifier: (modifier #{spec}) + name: (identifier #{name}) + type: {ty} + accessor_kind: (accessor_kind "get") + body: (block stmt: {body})) + ), + // A property with an explicit accessor block. The two shapes differ only + // by the presence of an initializer (tree-sitter split them into distinct + // `willset_didset_block` vs computed-accessor node types; swift-syntax + // makes both plain `accessorDecl`s): + // + // * With an initializer (`var x: T = e { willSet {…} didSet {…} }`) it is + // a *stored* property with observers: emit the backing + // `variable_declaration` first, then one `accessor_declaration` per + // observer (observers carry no type). + // * Without an initializer (`var v: T { get set }`, incl. protocol + // requirements) it is a *computed* property: no backing variable; the + // type is published so the get/set accessors carry it. + // + // In both cases the first emitted declaration is unchained and every + // subsequent one is tagged `chained_declaration` (the `!result.is_empty()` + // test). Must precede the plain `variableDecl` rules. + rule!( + (variableDecl + bindingSpecifier: @@spec + bindings: (patternBinding + pattern: (identifierPattern identifier: @@name) + typeAnnotation: (typeAnnotation type: @ty) + initializer: (initializerClause value: @@val)? + accessorBlock: (accessorBlock accessors: (accessorDecl)+ @@accessors))) + => + member* { + ctx.outer_modifiers = vec![tree!((modifier #{spec}))]; + ctx.property_name = Some(tree!((identifier #{name}))); let mut result = Vec::new(); - for (i, acc) in accessors.into_iter().enumerate() { - if i > 0 { - ctx.is_chained = true; - } + if let Some(val) = val { + // Stored property with observers: the initializer is not part + // of the binding, so translate it in a reset scope. + let val = ctx.scoped(|ctx| { + ctx.reset(); + ctx.translate(val) + })?; + result.push(tree!( + (variable_declaration + modifier: {ctx.outer_modifiers.clone()} + pattern: (name_pattern identifier: (identifier #{name})) + type: {ty} + value: {val}) + )); + } else { + // Computed property: the accessors carry the type. + ctx.property_type = Some(ty); + } + for acc in accessors.into_iter() { + ctx.is_chained = !result.is_empty(); result.extend(ctx.translate(acc)?); } result } ), - // Computed property: shorthand getter (no explicit get/set, just - // statements) → a single accessor_declaration with kind "get". - // Reads outer modifiers / chained tag from `ctx` (set by the - // outer `property_declaration` rule). + // Each `accessorDecl` becomes an `accessor_declaration`, reading the + // property name/type and the binding/chained modifiers from `ctx`. The + // accessor kind comes straight from the specifier keyword + // (`get`/`set`/`willSet`/`didSet`). The body is optional: a body-bearing + // accessor (a computed getter/setter, or a `willSet`/`didSet` observer) + // carries the binding modifier and a translated body, whereas a bodyless + // one (a protocol requirement) carries neither. The property context is + // read out *before* translating the body, which resets `ctx` so the + // accessor's context does not leak into the body subtree. rule!( - (property_binding - name: (pattern bound_identifier: @name) - type: _? @ty - computed_value: (computed_property statement: _* @@body)) + (accessorDecl accessorSpecifier: @@spec body: _? @@body) => - (accessor_declaration - modifier: {ctx.outer_modifiers.clone()} - modifier: {chained_modifier(&mut ctx)} - name: (identifier #{name}) - type: {ty} - accessor_kind: (accessor_kind "get") - body: (block stmt: {ctx.reset(); ctx.translate(body)?})) - ), - // Stored property with willSet/didSet observers (initializer - // optional) → a `variable_declaration` followed by one - // `accessor_declaration` per observer, each born with the - // property name set. Manual rule: we publish the property name - // into `ctx` before translating the observer children so the - // inner `willset_clause` / `didset_clause` rules construct - // valid `accessor_declaration` nodes from the start. - // - // The `variable_declaration` itself inherits the outer rule's - // chained state; observers always get `chained_declaration` - // because they're subsequent outputs of this flattening rule. - rule!( - (property_binding - name: (pattern bound_identifier: @name) - type: _? @ty - value: _? @@val - observers: (willset_didset_block willset: _? @@ws didset: _? @@ds)) - => - member* { - // The initializer value must not inherit the binding - // context (it may contain patterns, e.g. a switch - // expression), so translate it inside a `ctx.scoped` - // block — the block receives a temporary `ctx` whose - // `user_ctx` is a clone; mutations to it are discarded - // when the block returns, so the outer `ctx` is intact - // for the observer loop below. The observers keep the - // outer context: each willSet/didSet accessor emits - // the binding modifier and, in turn, resets the - // context for its own body. - let val = ctx.scoped(|ctx| { - ctx.reset(); - ctx.translate(val) - })?; - - let var_decl = tree!( - (variable_declaration - modifier: {ctx.outer_modifiers.clone()} - modifier: {chained_modifier(&mut ctx)} - pattern: (name_pattern identifier: (identifier #{name})) + accessor_declaration { + let binding = if body.is_some() { + ctx.outer_modifiers.clone() + } else { + Vec::new() + }; + let chained = chained_modifier(&mut ctx); + let name = ctx + .property_name + .ok_or("accessor outside property context")?; + let ty = ctx.property_type; + let body = match body { + Some(block) => { + ctx.reset(); + ctx.translate(block)?.into_iter().next() + } + None => None, + }; + tree!( + (accessor_declaration + modifier: {binding} + modifier: {chained} + name: {name} type: {ty} - value: {val}) - ); - - // Publish the property name for the observer rules. - ctx.property_name = Some(tree!((identifier #{name}))); - // Observers are subsequent outputs of this flattening - // rule, so they always get `chained_declaration`. - ctx.is_chained = true; - - let mut result = vec![var_decl]; - for obs in ws.into_iter().chain(ds) { - result.extend(ctx.translate(obs)?); - } - result + accessor_kind: (accessor_kind #{spec}) + body: {body}) + ) } ), - // property_binding with any pattern name (identifier or - // destructuring). Reads outer modifiers / chained tag from `ctx`. - // - // The enclosing `property_declaration` leads `ctx.outer_modifiers` - // with the `let`/`var` binding modifier, so the auto-translated name - // pattern (the LHS) becomes a binding, while the initializer value is - // translated with a reset context (see `SwiftContext::reset`). + // ---- Variables ---- + // The individual bindings of a `variableDecl`. The binding modifier and + // chained tag come from `ctx` (set by the `variableDecl` rule below). The + // type annotation and initializer are both optional (one combined rule + // covers `let x`, `let x = e`, `let x: T`, and `let x: T = e`); the + // initializer value is translated in a reset scope so it is not treated + // as a binding. rule!( - (property_binding - name: @pattern - type: _? @ty - value: _? @@val) + (patternBinding + pattern: @pattern + typeAnnotation: (typeAnnotation type: @ty)? + initializer: (initializerClause value: @@val)?) => (variable_declaration modifier: {ctx.outer_modifiers.clone()} @@ -354,60 +359,49 @@ fn translation_rules() -> Vec> { type: {ty} value: {ctx.reset(); ctx.translate(val)?}) ), - // property_declaration: flatten declarators (each may translate - // to multiple nodes — variable_declaration and/or - // accessor_declaration) and attach the binding modifier - // (let/var), outer modifiers, and `chained_declaration` for - // non-first declarations. Manual rule: publishes - // binding/outer modifiers into `ctx` and translates each - // declarator with `ctx.is_chained` toggled per iteration. The - // inner declaration rules (`property_binding` variants, - // accessor inner rules) read these fields and emit complete - // `modifier:` lists from the start. + // A `let`/`var` declaration binds one or more comma-separated patterns + // (`let x = 1, y = 2`). The `bindingSpecifier` (`let`/`var`) is published + // as the binding modifier, followed by any attributes and modifiers + // (`@objc`, `public`, `static`, …); each `patternBinding` becomes its own + // `variable_declaration`, with non-first ones tagged `chained_declaration`. + // Accessor/observer forms are handled by the earlier rules. rule!( - (property_declaration - binding: (value_binding_pattern mutability: @@binding_kind) - declarator: _* @@decls - (modifiers)* @mods) + (variableDecl + attributes: _* @attrs + modifiers: _* @mods + bindingSpecifier: @@spec + bindings: _* @@bindings) => - member* { - let binding_text = ctx.ast.source_text(binding_kind); - let binding = ctx.literal("modifier", &binding_text); - // The `let`/`var` binding modifier leads the declaration's - // modifier list and doubles as the "this is a binding" signal - // for pattern translation (see `in_binding_pattern`). - ctx.outer_modifiers = std::iter::once(binding).chain(mods).collect(); - + stmt* { + let binding = tree!((modifier #{spec})); + // The binding (`let`/`var`) leads, then attributes then modifiers + // in source order (Swift writes attributes before modifiers). + ctx.outer_modifiers = std::iter::once(binding).chain(attrs).chain(mods).collect(); let mut result = Vec::new(); - for (i, decl) in decls.into_iter().enumerate() { + for (i, b) in bindings.into_iter().enumerate() { ctx.is_chained = i > 0; - result.extend(ctx.translate(decl)?); + result.extend(ctx.translate(b)?); } result } ), // ---- Enums ---- - // enum_type_parameter → parameter (with optional name as pattern). + // An enum-case payload parameter (`radius: Double`, or just `Double`). + // The label (`firstName`) is optional. rule!( - (enum_type_parameter name: @name type: @ty) + (enumCaseParameter firstName: _? @@name type: @ty) => - (parameter - pattern: (name_pattern identifier: (identifier #{name})) - type: {ty}) + (parameter pattern: {name.map(|name| tree!((name_pattern identifier: (identifier #{name}))))} type: {ty}) ), + // An enum element with associated values (`case circle(radius: Double)`) + // becomes a nested `class_like_declaration` whose constructor carries the + // payload parameters; an element with a raw value (`case a = 1`) or a + // plain element (`case north`) becomes a `variable_declaration`. All + // carry the shared case modifiers / chained tag from `ctx` (set by the + // `enumCaseDecl` rule below) and are tagged `enum_case` (after any + // `chained_declaration` tag, matching the tree-sitter modifier order). rule!( - (enum_type_parameter type: @ty) - => - (parameter type: {ty}) - ), - // enum_case_entry with associated values → class_like_declaration - // containing a constructor whose parameters are the data - // parameters. Reads outer modifiers / chained tag from `ctx` - // (set by the outer `enum_entry` rule). - rule!( - (enum_case_entry - name: @name - data_contents: (enum_type_parameters parameter: _* @params)) + (enumCaseElement name: @name parameterClause: (enumCaseParameterClause parameters: _* @params)) => (class_like_declaration modifier: {ctx.outer_modifiers.clone()} @@ -416,9 +410,8 @@ fn translation_rules() -> Vec> { name: (identifier #{name}) member: (constructor_declaration parameter: {params} body: (block))) ), - // enum_case_entry with explicit raw value → variable_declaration with that value. rule!( - (enum_case_entry name: @name raw_value: @val) + (enumCaseElement name: @name rawValue: (initializerClause value: @val)) => (variable_declaration modifier: {ctx.outer_modifiers.clone()} @@ -427,9 +420,8 @@ fn translation_rules() -> Vec> { pattern: (name_pattern identifier: (identifier #{name})) value: {val}) ), - // enum_case_entry without associated values → variable_declaration tagged enum_case. rule!( - (enum_case_entry name: @name) + (enumCaseElement name: @name) => (variable_declaration modifier: {ctx.outer_modifiers.clone()} @@ -437,12 +429,14 @@ fn translation_rules() -> Vec> { modifier: (modifier "enum_case") pattern: (name_pattern identifier: (identifier #{name}))) ), - // enum_entry: flatten case entries; publish outer modifiers - // into `ctx` and translate each case with `ctx.is_chained` - // toggled per iteration so the inner `enum_case_entry` rules - // emit complete `modifier:` lists from the start. + // Enum cases. A single `case` declaration may carry modifiers + // (e.g. `indirect`) and list several comma-separated elements; each + // becomes its own declaration carrying those shared modifiers, and + // non-first ones are tagged `chained_declaration` (mirroring the + // tree-sitter `enum_entry` rule). The modifiers are published into `ctx` + // for the element rules above, which build the actual declaration. rule!( - (enum_entry case: _+ @@cases (modifiers)* @mods) + (enumCaseDecl modifiers: _* @mods elements: _* @@cases) => member* { ctx.outer_modifiers = mods; @@ -455,198 +449,224 @@ fn translation_rules() -> Vec> { result } ), - // Plain assignment: `x = expr` + // `identifierPattern` wraps a single identifier token. rule!( - (assignment operator: "=" target: (directly_assignable_expression expr: @target) result: @value) - => - (assign_expr target: {target} value: {value}) - ), - // Compound assignment: `x += expr` etc. - rule!( - (assignment operator: @op target: (directly_assignable_expression expr: @target) result: @value) - => - (compound_assign_expr target: {target} operator: (infix_operator #{op}) value: {value}) - ), - // Unwrap `type` wrapper node - rule!((type name: @inner) => type_expr { inner }), - // `directly_assignable_expression` is just a wrapper; unwrap it - rule!((directly_assignable_expression expr: @inner) => expr { inner }), - // Pattern with bound_identifier → name_pattern. - rule!( - (pattern bound_identifier: @name) + (identifierPattern identifier: @name) => (name_pattern identifier: (identifier #{name})) ), - // Pattern with 'let' or 'var' binding: publish the binding modifier - // into `ctx` and translate the inner pattern under it. + // A `let`/`var` value-binding pattern (`let x`) inside a case or `if case` + // introduces a new binding; it unwraps to its inner pattern (a + // `name_pattern`). + rule!((valueBindingPattern pattern: @p) => pattern { p }), + // An enum-case pattern with associated values (`case .foo(let x)`, + // `case Color.foo(let x)`) is an expression pattern wrapping a call of a + // member access. It becomes a `constructor_pattern`; its arguments are + // translated as pattern elements (see the `labeledExpr` rules, gated by + // `ctx.in_pattern`). Matched before the generic `expressionPattern` rule. + // The base is optional: a leading-dot form (`.foo`) has none, so the + // constructor's base is an `inferred_type_expr`. rule!( - (pattern kind: (binding_pattern binding: (value_binding_pattern mutability: @@binding_kind) pattern: @@pattern)) + (expressionPattern expression: (functionCallExpr + calledExpression: (memberAccessExpr base: _? @base period: @dot declName: (declReferenceExpr baseName: @name)) + arguments: _* @@args)) => - pattern* { - let binding_text = ctx.ast.source_text(binding_kind); - let binding = ctx.literal("modifier", &binding_text); - ctx.outer_modifiers = vec![binding]; - ctx.translate(pattern)? + constructor_pattern { + ctx.in_pattern = true; + let elements = ctx.translate(args)?; + let base = base.unwrap_or_else(|| tree!((inferred_type_expr #{dot}))); + tree!((constructor_pattern + constructor: (member_access_expr + base: {base} + member: (identifier #{name})) + element: {elements})) } ), - // case T.foo(x,y) pattern + // A tuple destructuring pattern (`let (a, b) = …`). A labelled element + // (`let (x: a) = …`) carries its label through as the `pattern_element` + // key; unlabelled elements have no key. + rule!((tuplePattern elements: _* @els) => (tuple_pattern element: {els})), rule!( - (pattern kind: (case_pattern type: @typ name: @name arguments: (tuple_pattern item: (tuple_pattern_item)* @items)? )) + (tuplePatternElement label: _? @label pattern: @p) => - (constructor_pattern - constructor: (member_access_expr base: {typ} member: (identifier #{name})) - element: {items}) + (pattern_element key: {label.map(|l| tree!((identifier #{l})))} pattern: {p}) ), - // case .foo(x,y) pattern + // A type-casting pattern (`case is T`). Not yet supported, so it is + // mapped to `unsupported_node` — an explicit reminder that this needs + // handling in the future. (Redundant with the catch-all fallback, but + // kept as a signpost.) + rule!((isTypePattern) => (unsupported_node)), + // A standalone wildcard pattern (`case _:`, `if case _`): swift-syntax + // models the bare `_` as an `expressionPattern` wrapping a + // `discardAssignmentExpr`. Matched before the generic `expressionPattern` + // rule so `_` becomes an `ignore_pattern` rather than an equality match. + // (Wildcards *inside* an enum-case argument list are handled by the + // `labeledExpr`/`discardAssignmentExpr` rules.) + rule!((expressionPattern expression: (discardAssignmentExpr)) => (ignore_pattern)), + // A wildcard *binding* pattern (`let _ = x`, `for _ in xs`). swift-syntax + // models this as a `wildcardPattern` — distinct from the `_` *match* + // pattern above, which is an `expressionPattern` over a + // `discardAssignmentExpr`. + rule!((wildcardPattern) => (ignore_pattern)), + // A tuple pattern in a match position (`case (let a, 3):`) is parsed by + // swift-syntax as an `expressionPattern` wrapping a `tupleExpr` — unlike a + // binding tuple (`let (a, b)`), which is a real `tuplePattern`. Recognise + // it as a `tuple_pattern`; its `labeledExpr` elements translate to + // `pattern_element`s under `ctx.in_pattern` (a binding element becomes a + // `name_pattern`, any other expression an `expr_equality_pattern`). rule!( - (pattern kind: (case_pattern dot: @dot name: @name arguments: (tuple_pattern item: (tuple_pattern_item)* @items)? )) + (expressionPattern expression: (tupleExpr elements: _* @@els)) => - (constructor_pattern - constructor: (member_access_expr base: (inferred_type_expr #{dot}) member: (identifier #{name})) - element: {items}) - ), - // Tuple pattern and its (optionally named) items - rule!((pattern kind: (tuple_pattern item: _* @elems)) => (tuple_pattern element: {elems})), - rule!((tuple_pattern_item name: @key pattern: @pat) => (pattern_element key: (identifier #{key}) pattern: {pat})), - rule!((tuple_pattern_item pattern: @pat) => (pattern_element pattern: {pat})), - // Type casting pattern (TODO) - rule!((pattern kind: (type_casting_pattern)) => (unsupported_node)), - // Wildcard pattern - rule!((pattern kind: (wildcard_pattern)) => (ignore_pattern)), - // A bare identifier used as an expression-pattern. Under a `var`/`let` - // binding it introduces a new variable and becomes a `name_pattern`; - // otherwise it matches by equality and is left as an `expr_equality_pattern` - // over the name expression. - rule!( - (pattern kind: (simple_identifier) @name) - => - pattern { - if ctx.in_binding_pattern() { - tree!((name_pattern identifier: (identifier #{name}))) - } else { - let expr = tree!((name_expr identifier: (identifier #{name}))); - tree!((expr_equality_pattern expr: {expr})) - } + tuple_pattern { + ctx.in_pattern = true; + let elements = ctx.translate(els)?; + tree!((tuple_pattern element: {elements})) } ), - // Expression pattern - // We lack a way to check if 'expr' is actually an expression, but due to rule ordering - // the 'expression' case is the only remaining possibility when this rule tries to match. - rule!((pattern kind: @expr) => (expr_equality_pattern expr: {expr})), + // A bare expression pattern (`case 1:`, `case someConstant:`) matches by + // equality. + rule!((expressionPattern expression: @e) => (expr_equality_pattern expr: {e})), // ---- Functions ---- - // Function declaration - // Function declaration (return type optional, body statements optional). + // A function declaration (parameters/return type/body optional). The + // parameters and return type nest under `signature`; the body is a + // `codeBlock`. A bodyless function (a protocol requirement) still emits + // an empty `block`, matching the tree-sitter path. rule!( - (function_declaration + (functionDecl name: @name - parameter: _* @params - return_type: _? @ret - body: (block statement: _* @body_stmts)) + signature: (functionSignature + parameterClause: (functionParameterClause parameters: _* @params) + returnClause: (returnClause type: @ret)?) + body: (codeBlock statements: _* @body)) => (function_declaration name: (identifier #{name}) parameter: {params} return_type: {ret} - body: (block stmt: {body_stmts})) + body: (block stmt: {body})) ), - // Parameters are wrapped in function_parameter, which also carries - // optional default values. Publishes the default value into `ctx` - // before translating the inner `parameter` so the `parameter` - // rules can include it as a `default:` field directly. rule!( - (function_parameter parameter: @@p default_value: _? @def) + (functionDecl + name: @name + signature: (functionSignature + parameterClause: (functionParameterClause parameters: _* @params) + returnClause: (returnClause type: @ret)?)) => - parameter* { - ctx.default_value = def; - ctx.translate(p)? + (function_declaration + name: (identifier #{name}) + parameter: {params} + return_type: {ret} + body: (block)) + ), + // A function parameter. With two names (`firstName`+`secondName`) the + // first is the external argument label and the second the internal name; + // with one name it is just the internal name. The default value is + // optional. + // + // PARITY: the declared type is intentionally dropped. In the tree-sitter + // path the untyped-parameter rule was ordered before the typed one and + // shadowed it (first match wins), so the baseline emits no parameter + // type; emitting one here would diverge from it. + rule!( + (functionParameter + firstName: @@first + secondName: _? @@second + defaultValue: (initializerClause value: @val)?) + => + parameter { + let (external, name) = match second { + Some(second) => (Some(tree!((identifier #{first}))), second), + None => (None, first), + }; + tree!((parameter + external_name: {external} + pattern: (name_pattern identifier: (identifier #{name})) + default: {val})) } ), - // Parameter with external name and type + // A function/method call (`foo(1, 2)`). `calledExpression` is the callee + // and `arguments` is an (elided) list of `labeledExpr`, each translated + // to an `argument` below. A trailing closure (`xs.map { … }`) becomes a + // final unlabelled argument; that variant is matched first. rule!( - (parameter external_name: @ext name: @name) + (functionCallExpr calledExpression: @callee arguments: _* @args trailingClosure: @tc) => - (parameter - external_name: (identifier #{ext}) - pattern: (name_pattern identifier: (identifier #{name})) - default: {ctx.default_value}) + (call_expr callee: {callee} argument: {args} argument: (argument value: {tc})) ), rule!( - (parameter external_name: @ext name: @name type: @ty) + (functionCallExpr calledExpression: @callee arguments: _* @args) => - (parameter - external_name: (identifier #{ext}) - pattern: (name_pattern identifier: (identifier #{name})) - type: {ty} - default: {ctx.default_value}) + (call_expr callee: {callee} argument: {args}) ), - // Parameter with just name and type (no external name) + // A call argument or an enum-case pattern argument. When translating an + // enum-case `constructor_pattern`'s arguments (`ctx.in_pattern`), a + // `patternExpr` argument (`let x`) becomes a bound `name_pattern`, a + // wildcard (`_`) becomes an `ignore_pattern`, and any other expression + // becomes an `expr_equality_pattern`; each is wrapped in a + // `pattern_element` carrying the optional argument label as its `key`. + // Otherwise the argument keeps its label as the `name` and its value. + // The pattern-only shapes (`patternExpr`, `discardAssignmentExpr`) are + // matched first; they never occur as ordinary call arguments. rule!( - (parameter name: @name) + (labeledExpr label: _? @lbl expression: (patternExpr pattern: @p)) => - (parameter - pattern: (name_pattern identifier: (identifier #{name})) - default: {ctx.default_value}) + (pattern_element key: {lbl.map(|l| tree!((identifier #{l})))} pattern: {p}) ), rule!( - (parameter name: @name type: @ty) + (labeledExpr label: _? @lbl expression: (discardAssignmentExpr) @@wildcard) => - (parameter - pattern: (name_pattern identifier: (identifier #{name})) - type: {ty} - default: {ctx.default_value}) + (pattern_element key: {lbl.map(|l| tree!((identifier #{l})))} pattern: (ignore_pattern #{wildcard})) ), - // Reference to a function, f(x:y:z:). This is parsed as a call with a single argument with multiple reference_specifier labels. - // We don't want downstream QL to try to handle this as a call_expr with a weird argument, so explicitly mark it as unsupported for now. - // In the future we probably want to translate this to a lambda expression. rule!( - (call_expression suffix: (call_suffix arguments: (value_arguments argument: (value_argument reference_specifier: _+) @ref_arg))) + (labeledExpr label: _? @lbl expression: @val) => - (unsupported_node) + argument { + let key = lbl.map(|l| tree!((identifier #{l}))); + if ctx.in_pattern { + tree!((pattern_element + key: {key} + pattern: (expr_equality_pattern expr: {val}))) + } else { + tree!((argument name: {key} value: {val})) + } + } ), - // Call expression: function(args...) + // Member access (`list.append`). The `declName` is itself a + // `declReferenceExpr`; pull its `baseName` out as the member identifier. + // A leading-dot access (`.foo`) has no explicit base — the base is an + // `inferred_type_expr`. The base-ful form is matched first. rule!( - (call_expression function: @func suffix: (call_suffix arguments: (value_arguments argument: (value_argument)* @args))) + (memberAccessExpr base: @base declName: (declReferenceExpr baseName: @member)) => - (call_expr callee: {func} argument: {args}) + (member_access_expr base: {base} member: (identifier #{member})) ), - // Value argument with label (value: _ matches both named nodes and anonymous tokens like nil) rule!( - (value_argument name: (value_argument_label name: @label) value: @val) + (memberAccessExpr declName: (declReferenceExpr baseName: @member)) => - (argument name: (identifier #{label}) value: {val}) + (member_access_expr base: (inferred_type_expr) member: (identifier #{member})) ), - // Value argument without label - rule!( - (value_argument value: @val) - => - (argument value: {val}) - ), - // Navigation expression → member_access_expr - rule!( - (navigation_expression target: @target suffix: (navigation_suffix suffix: @member)) - => - (member_access_expr base: {target} member: (identifier #{member})) - ), - // Return / break / continue, one rule per keyword. - // The anonymous "return"/"break"/"continue" keywords are matched as - // string literals. - rule!((control_transfer_statement kind: "return" result: _? @val) => (return_expr value: {val})), - rule!((control_transfer_statement kind: "break" result: @lbl) => (break_expr label: (identifier #{lbl}))), - rule!((control_transfer_statement kind: "break") => (break_expr)), - rule!((control_transfer_statement kind: "continue" result: @lbl) => (continue_expr label: (identifier #{lbl}))), - rule!((control_transfer_statement kind: "continue") => (continue_expr)), - rule!((control_transfer_statement kind: (throw_keyword) result: @val) => (throw_expr value: {val})), + // Control transfer, one rule per keyword. `return` carries an optional + // value; `break` / `continue` an optional target label; `throw` its + // thrown expression. + rule!((returnStmt expression: _? @val) => (return_expr value: {val})), + rule!((breakStmt label: _? @@lbl) => (break_expr label: {lbl.map(|l| tree!((identifier #{l})))})), + rule!((continueStmt label: _? @@lbl) => (continue_expr label: {lbl.map(|l| tree!((identifier #{l})))})), + rule!((throwStmt expression: @val) => (throw_expr value: {val})), // ---- Closures ---- - // Lambda literal with optional type header (parameters + optional return type). - // The return_type capture is optional, so this rule covers both cases. + // A closure (`{ (x: Int) -> Int in … }`) becomes a `function_expr`. The + // whole signature is optional, as are its capture list, parameter + // clause, and return clause, so one rule covers everything from a bare + // `{ … }` to `{ [weak self] (x) -> T in … }`. Shorthand `$0` closures + // have no signature and their `$0` references are ordinary name + // expressions. rule!( - (lambda_literal - attribute: _* @attrs - captures: (capture_list item: _* @captures)? - type: (lambda_function_type - params: (lambda_function_type_parameters parameter: _* @params) - return_type: _? @ret)? - statement: _* @body) + (closureExpr + signature: (closureSignature + attributes: _* @attrs + capture: (closureCaptureClause items: _* @captures)? + parameterClause: _* @params + returnClause: (returnClause type: @ret)?)? + statements: _* @body) => (function_expr modifier: {attrs} @@ -655,114 +675,109 @@ fn translation_rules() -> Vec> { return_type: {ret} body: (block stmt: {body})) ), - // capture_list_item with ownership modifier (e.g. [weak self], [unowned x]) + // A closure capture (`[weak self]`, `[x]`, `[y = expr]`). The optional + // ownership specifier (`weak`/`unowned`) becomes a modifier; the + // captured name becomes the bound `name_pattern`; an explicit capture + // initializer (`[y = expr]`) becomes the bound value. rule!( - (capture_list_item ownership: _? @ownership name: @name value: _? @val) + (closureCapture + specifier: (closureCaptureSpecifier specifier: @@spec)? + name: @@name + initializer: (initializerClause value: @val)?) => (variable_declaration - modifier: {ownership} + modifier: {spec.map(|s| tree!((modifier #{s})))} pattern: (name_pattern identifier: (identifier #{name})) value: {val}) ), - // Lambda parameter with type and optional external name + // A closure parameter clause (`(x: Int, y)`) unwraps to its parameters. + rule!((closureParameterClause parameters: _* @params) => parameter* { params }), + // A closure parameter (`x: Int`, or just `x`). Unlike a function + // parameter it has no external label; the type is optional. rule!( - (lambda_parameter external_name: @ext name: @name type: @ty) + (closureParameter firstName: @name type: _? @ty) => - (parameter - external_name: (identifier #{ext}) - pattern: (name_pattern identifier: (identifier #{name})) - type: {ty}) + (parameter pattern: (name_pattern identifier: (identifier #{name})) type: {ty}) ), + // A shorthand closure parameter (`x` in `{ x, y in … }`): a bare name + // with no parentheses and no type. rule!( - (lambda_parameter name: @name type: @ty) - => - (parameter - pattern: (name_pattern identifier: (identifier #{name})) - type: {ty}) - ), - rule!( - (lambda_parameter external_name: @ext name: @name) - => - (parameter - external_name: (identifier #{ext}) - pattern: (name_pattern identifier: (identifier #{name}))) - ), - rule!( - (lambda_parameter name: @name) + (closureShorthandParameter name: @name) => (parameter pattern: (name_pattern identifier: (identifier #{name}))) ), - // Call expression with trailing closure (no value_arguments) - rule!( - (call_expression function: @func suffix: (call_suffix lambda: (lambda_literal) @closure)) - => - (call_expr - callee: {func} - argument: (argument value: {closure})) - ), // ---- Control flow ---- - // If statement + // An `if`/`else` expression. Conditions are joined via `and_chain`; the + // then-body and optional else-body (another block, or an `ifExpr` for an + // else-if chain) are translated recursively. rule!( - (if_statement condition: _* @cond body: @then_body else_branch: _? @else_stmts) + (ifExpr conditions: _* @cond body: @then_body elseBody: _? @else_stmts) => (if_expr condition: {and_chain(&mut ctx, cond)} then: {then_body} else: {else_stmts}) ), - // Guard statement + // A `guard … else { }` statement. The `body` is the else block. rule!( - (guard_statement condition: _* @cond body: (block statement: _* @else_stmts)) + (guardStmt conditions: _* @cond body: @else_stmts) => (guard_if_stmt condition: {and_chain(&mut ctx, cond)} - else: (block stmt: {else_stmts})) + else: {else_stmts}) ), - // Ternary expression → if_expr + // Ternary (`c ? a : b`) desugars to an `if_expr`, as in the tree-sitter + // path. rule!( - (ternary_expression condition: @cond if_true: @then_val if_false: @else_val) + (ternaryExpr condition: @cond thenExpression: @then_val elseExpression: @else_val) => (if_expr condition: {cond} then: {then_val} else: {else_val}) ), - // Switch statement + // A `switch` statement. Each `switchCase` becomes a `switch_case` with a + // pattern (or an `or_pattern` for comma-separated `case a, b:`) and a + // body; a `default:` case has a body but no pattern. The case items and + // body are auto-translated; the Rust block only picks the pattern shape + // by arity (the query engine can't branch on list length). rule!( - (switch_statement expr: @val entry: (switch_entry)* @cases) + (switchExpr subject: @val cases: _* @cases) => (switch_expr value: {val} case: {cases}) ), - // Switch entry with multiple patterns and body rule!( - (switch_entry - pattern: (switch_pattern pattern: @first) - pattern: (switch_pattern pattern: @rest)+ - statement: _* @body) + (switchCase label: (switchCaseLabel caseItems: _* @items) statements: _* @body) => - (switch_case pattern: (or_pattern pattern: {first} pattern: {rest}) body: (block stmt: {body})) + switch_case { + let pattern = if items.len() == 1 { + items[0] + } else { + tree!((or_pattern pattern: {items})) + }; + tree!((switch_case pattern: {pattern} body: (block stmt: {body}))) + } ), - // Switch entry with exactly one pattern and body rule!( - (switch_entry pattern: (switch_pattern pattern: @pat) statement: _* @body) - => - (switch_case pattern: {pat} body: (block stmt: {body})) - ), - // Switch entry: default case (no patterns) - rule!( - (switch_entry default: (default_keyword) statement: _* @body) + (switchCase label: (switchDefaultLabel) statements: _* @body) => (switch_case body: (block stmt: {body})) ), - // if case PATTERN = expr — preserve the pattern directly (no Optional wrapping) + // A single case item unwraps to its pattern (used as an `or_pattern` + // element). + rule!((switchCaseItem pattern: @p) => pattern { p }), + // A pattern-matching condition (`if case let x = e`, `if case .foo(let x) + // = e`) becomes a `pattern_guard_expr`: the matched pattern and the + // scrutinee value are translated recursively. rule!( - (if_let_binding "case" pattern: @pat value: @val) + (matchingPatternCondition pattern: @pat initializer: (initializerClause value: @val)) => - (pattern_guard_expr - value: {val} - pattern: {pat}) + (pattern_guard_expr pattern: {pat} value: {val}) ), + // Optional binding (`if let x = foo`, or shorthand `if let x`) desugars + // to a `pattern_guard_expr` matching `Optional.some(x)`, exactly as the + // tree-sitter path does. The initialized form is matched first. rule!( - (if_let_binding - pattern: (pattern binding: (value_binding_pattern) bound_identifier: @name) - value: @val) + (optionalBindingCondition + pattern: (identifierPattern identifier: @name) + initializer: (initializerClause value: @val)) => (pattern_guard_expr value: {val} @@ -770,10 +785,8 @@ fn translation_rules() -> Vec> { constructor: (member_access_expr base: (named_type_expr name: (identifier "Optional")) member: (identifier "some")) element: (pattern_element pattern: (name_pattern identifier: (identifier #{name}))))) ), - // Shorthand if let x (Swift 5.7+) — also semantically .some(x) rule!( - (if_let_binding - pattern: (pattern binding: (value_binding_pattern) bound_identifier: @name)) + (optionalBindingCondition pattern: (identifierPattern identifier: @name)) => (pattern_guard_expr value: (name_expr identifier: (identifier #{name})) @@ -781,292 +794,320 @@ fn translation_rules() -> Vec> { constructor: (member_access_expr base: (named_type_expr name: (identifier "Optional")) member: (identifier "some")) element: (pattern_element pattern: (name_pattern identifier: (identifier #{name}))))) ), - // If-condition — unwrap (pass through the inner expression/pattern) - rule!((if_condition kind: @inner) => expr_or_pattern { inner }), + // A single condition in an `if`/`while`/`guard` condition list unwraps to + // its inner expression; `and_chain` joins multiple with `&&`. + rule!((conditionElement condition: @c) => expr { c }), + // `if`/`switch`/`do` are expressions in Swift; when used as a statement + // swift-syntax wraps them in an `expressionStmt`. Unwrap to the inner + // expression (a plain expression statement, e.g. a call, is not wrapped). + rule!((expressionStmt expression: @e) => expr { e }), // ---- Loops ---- - // For-in loop with optional where-clause guard. + // A `for`-`in` loop. The optional `where` clause becomes the `guard`. rule!( - (for_statement - item: @pat - collection: @iter - where: (where_clause expr: @guard)? - body: (block statement: _* @body)) + (forStmt + pattern: @pat + sequence: @iter + whereClause: (whereClause condition: @guard)? + body: @body) => (for_each_stmt pattern: {pat} iterable: {iter} guard: {guard} - body: (block stmt: {body})) + body: {body}) ), - // While loop + // A `while` loop. rule!( - (while_statement condition: _* @cond body: (block statement: _* @body)) + (whileStmt conditions: _* @cond body: @body) => (while_stmt condition: {and_chain(&mut ctx, cond)} - body: (block stmt: {body})) + body: {body}) ), - // Repeat-while loop + // A `repeat { } while c` loop desugars to a `do_while_stmt`. rule!( - (repeat_while_statement condition: _* @cond body: (block statement: _* @body)) + (repeatStmt body: @body condition: @cond) => - (do_while_stmt - condition: {and_chain(&mut ctx, cond)} - body: (block stmt: {body})) + (do_while_stmt condition: {cond} body: {body}) + ), + // A labeled statement (`outer: for … { }`). swift-syntax stores the + // label and colon as separate tokens, so the label token is already the + // bare name (no trailing `:` to strip). + rule!( + (labeledStmt label: @@lbl statement: @stmt) + => + (labeled_stmt label: (identifier #{lbl}) stmt: {stmt}) ), - // Labeled statement (e.g. `outer: for ...`). Strip the trailing ':' from the label token. - rule!((labeled_statement label: (statement_label) @lbl statement: @stmt) => labeled_stmt { - let text = ctx.ast.source_text(lbl); - let name = &text[..text.len() - 1]; - tree!((labeled_stmt label: (identifier #{name}) stmt: {stmt})) - }), // ---- Collections ---- - // Array literal - rule!((array_literal element: _* @elems) => (array_literal element: {elems})), - // Empty array literal - rule!((array_literal) => (array_literal)), - // Dictionary literal — zip keys and values into key_value_pairs + // An array literal (`[1, 2, 3]`). Each `arrayElement` unwraps to its + // contained expression. rule!( - (dictionary_literal key: _* @keys value: _* @vals) + (arrayExpr elements: _* @els) => - (map_literal element: {keys.into_iter().zip(vals).map(|(k, v)| - tree!((key_value_pair key: {k} value: {v})) - )}) + (array_literal element: {els}) + ), + rule!((arrayElement expression: @e) => expr { e }), + // A dictionary literal (`["a": 1]`) is kept as an opaque `map_literal` + // leaf (its source span), matching the tree-sitter path. + rule!((dictionaryExpr) => (map_literal)), + // A subscript access (`xs[0]`) is modelled as a call, exactly as the + // tree-sitter grammar does (it parses `xs[0]` like `xs(0)`). + rule!( + (subscriptCallExpr calledExpression: @callee arguments: _* @args) + => + (call_expr callee: {callee} argument: {args}) ), - rule!((dictionary_literal element: _* @elems) => (map_literal element: {elems})), - rule!((dictionary_literal_item key: @k value: @v) => (key_value_pair key: {k} value: {v})), // ---- Optionals and errors ---- // Optional chaining — unwrap the marker - rule!((optional_chain_marker expr: @inner) => expr { inner }), + rule!((optionalChainingExpr expression: @inner) => expr { inner }), // try/try?/try! expr → unary_expr with operator "try", "try?" or "try!" - rule!((try_expression (try_operator) @op expr: @inner) => (unary_expr operator: (prefix_operator #{op}) operand: {inner})), - rule!((try_expression operator: (try_operator) @op expr: @inner) => (unary_expr operator: (prefix_operator #{op}) operand: {inner})), + rule!( + (tryExpr questionOrExclamationMark: _? @@m expression: @e) + => + expr { + let op = format!("try{}", m.map(|m| ctx.source_text(m)).unwrap_or_default()); + tree!((unary_expr operator: (prefix_operator #{op}) operand: {e})) + } + ), // Do-catch → try_expr rule!( - (do_statement body: (block statement: _* @body) catch: (catch_block)* @catches) + (doStmt body: @body catchClauses: _* @catches) => (try_expr - body: (block stmt: {body}) + body: {body} catch_clause: {catches}) ), // Catch block with bound identifier; optional where-clause guard. rule!( - (catch_block - keyword: (catch_keyword) - error: @pattern - where: (where_clause expr: @guard)? - body: (block statement: _* @body)) + (catchClause + catchItems: (catchItem + pattern: @pattern + whereClause: (whereClause condition: @guard)?) + body: @body) => (catch_clause pattern: {pattern} guard: {guard} - body: (block stmt: {body})) + body: {body}) ), // Catch block without error binding - rule!( - (catch_block keyword: (catch_keyword) body: (block statement: _* @body)) - => - (catch_clause body: (block stmt: {body})) - ), - // Empty catch block: catch {} - rule!( - (catch_block (catch_keyword)) - => - (catch_clause body: (block)) - ), - // Catch block with unhandled pattern — preserve pattern; optional body. - rule!( - (catch_block keyword: (catch_keyword) error: @pat body: (block statement: _* @body)) - => - (catch_clause - pattern: {pat} - body: (block stmt: {body})) - ), + rule!((catchClause body: @body) => (catch_clause body: {body})), // As expression (type cast) — as?, as! - rule!((as_expression (as_operator) @op expr: @val type: @ty) => (type_cast_expr expr: {val} operator: (infix_operator #{op}) type: {ty})), + rule!((asExpr expression: @val questionOrExclamationMark: _? @@mark type: @ty) => type_cast_expr { + let op = format!("as{}", mark.map(|m| ctx.source_text(m)).unwrap_or_default()); + tree!((type_cast_expr expr: {val} operator: (infix_operator #{op}) type: {ty})) + }), // Check expression (`x is T`) → type_test_expr - rule!((check_expression op: @op target: @val type: @ty) => (type_test_expr expr: {val} operator: (infix_operator #{op}) type: {ty})), + rule!((isExpr expression: @val type: @ty) => (type_test_expr expr: {val} operator: (infix_operator "is") type: {ty})), // Await expression → unary_expr with operator "await" - rule!((await_expression expr: @val) => (unary_expr operator: (prefix_operator "await") operand: {val})), - // A multi-part identifier (for example `Foo.Bar.Baz`) is translated to - // a member_access_expr chain with a name_expr base. + rule!((awaitExpr expression: @val) => (unary_expr operator: (prefix_operator "await") operand: {val})), + // Force-unwrap (`x!`) → postfix unary_expr. swift-syntax has a dedicated + // `forceUnwrapExpr` node (the tree-sitter path used the generic postfix + // operator rule instead). + rule!((forceUnwrapExpr expression: @e) => (unary_expr operator: (postfix_operator "!") operand: {e})), + // ---- Imports ---- + // An import declaration. The dotted path (a list of + // `importPathComponent`s) becomes a `name_expr`/`member_access_expr` + // chain (via `member_chain`). A scoped import (`import struct Foo.Bar`) + // has an `importKindSpecifier` and binds the last path component as a + // `name_pattern`; a plain import (`import Foundation`) has none and uses + // a `bulk_importing_pattern` spanning the whole declaration. Any leading + // attributes (`@_exported`) and access modifiers (`public`) become + // `modifier`s. rule!( - (identifier part: _+ @parts) + (importDecl + attributes: _* @attrs + modifiers: _* @mods + importKindSpecifier: _? @@kind + path: (importPathComponent name: @@parts)*) => - expr { member_chain(&mut ctx, parts) } + import_declaration { + let pattern = match kind { + Some(_) => { + let last = *parts.last().ok_or("import has no path")?; + tree!((name_pattern identifier: (identifier #{last}))) + } + None => tree!((bulk_importing_pattern)), + }; + tree!((import_declaration + modifier: {kind.map(|k| tree!((modifier #{k})))} + modifier: {attrs} + modifier: {mods} + pattern: {pattern} + imported_expr: {member_chain(&mut ctx, parts)})) + } ), - // Scoped import declaration (for example `import struct Foo.Bar`): - // flatten the identifier parts into a member_access_expr and bind the - // final segment as a name_pattern. - rule!( - (import_declaration scoped_import_kind: @kind name: (identifier part: _+ @parts) @name modifiers: (modifiers)? @mods) - => - (import_declaration - pattern: (name_pattern identifier: (identifier #{parts.last().unwrap()})) - imported_expr: {name} - modifier: (modifier #{kind}) - modifier: {mods}) - ), - // Non-scoped import declaration (for example `import Foundation`): - // flatten the identifier parts into a member_access_expr and use a - // bulk_importing_pattern. - rule!( - (import_declaration name: @name modifiers: (modifiers)? @mods) - => - (import_declaration - pattern: (bulk_importing_pattern) - imported_expr: {name} - modifier: {mods}) - ), - // ---- Types and classes ---- - // Self expression → name_expr - rule!((self_expression) => (name_expr identifier: (identifier "self"))), - // Super expression → super_expr - rule!((super_expression) => (super_expr)), - // Modifiers — unwrap to individual modifier children - rule!((modifiers _* @mods) => modifier* { mods }), + // ---- Types and declarations ---- + // A leading attribute (`@objc`) or access/function/member/mutation/ + // ownership modifier (swift-syntax models each as a single `declModifier`) + // becomes a `modifier`; its source text is the modifier spelling. rule!((attribute) @m => (modifier #{m})), - rule!((visibility_modifier) @m => (modifier #{m})), - rule!((function_modifier) @m => (modifier #{m})), - rule!((member_modifier) @m => (modifier #{m})), - rule!((mutation_modifier) @m => (modifier #{m})), - rule!((ownership_modifier) @m => (modifier #{m})), - rule!((property_modifier) @m => (modifier #{m})), - rule!((parameter_modifier) @m => (modifier #{m})), - rule!((inheritance_modifier) @m => (modifier #{m})), - rule!((property_behavior_modifier) @m => (modifier #{m})), - // Type annotations — unwrap - rule!((type_annotation type: @inner) => type_expr { inner }), - // user_type is split into simple_user_type parts. - // Keep a conservative textual fallback to avoid dropping type information. - rule!((user_type) @ty => (named_type_expr name: (identifier #{ty}))), - // Tuple type → tuple_type_expr - rule!((tuple_type element: _* @elems) => (tuple_type_expr element: {elems})), - rule!((tuple_type_item name: @name type: @ty) => (tuple_type_element name: (identifier #{name}) type: {ty})), - rule!((tuple_type_item type: @ty) => (tuple_type_element type: {ty})), - // Array type `[T]` → generic_type_expr with Array base - rule!((array_type element: @e) => (generic_type_expr - base: (named_type_expr name: (identifier "Array")) - type_argument: {e})), - // Dictionary type `[K: V]` → generic_type_expr with Dictionary base - rule!((dictionary_type key: @k value: @v) => (generic_type_expr - base: (named_type_expr name: (identifier "Dictionary")) - type_argument: {k} - type_argument: {v})), - // Optional type `T?` → generic_type_expr with Optional base - rule!((optional_type wrapped: @w) => (generic_type_expr - base: (named_type_expr name: (identifier "Optional")) - type_argument: {w})), - // Function type `(Params) -> Ret` → function_type_expr. - rule!((function_type parameter: _* @ps return_type: @ret) => (function_type_expr parameter: {ps} return_type: {ret})), - rule!((function_type_parameter name: @name type: @ty) => (parameter external_name: (identifier #{name}) type: {ty})), - rule!((function_type_parameter type: @ty) => (parameter type: {ty})), - // Selector expression: `#selector(inner)` -- not yet supported + rule!((declModifier) @m => (modifier #{m})), + // A `super` expression. (`self` needs no rule: swift-syntax models it as + // an ordinary `declReferenceExpr`, already mapped to a `name_expr`.) + rule!((superExpr) => (super_expr)), + // Type expressions. A generic type applied with explicit arguments + // (`Set`) is represented opaquely, using the whole source text as + // the name (PARITY(tree-sitter): the generic arguments are not + // structured `type_argument`s). Matched before the plain `identifierType` + // rule, which would otherwise drop the arguments. rule!( - (selector_expression _ @inner) + (identifierType genericArgumentClause: (genericArgumentClause)) @@ty => - (unsupported_node) + (named_type_expr name: (identifier #{ty})) ), - // Key path expressions are currently unsupported. - rule!((key_path_expression) => (unsupported_node)), - // Inheritance specifier → base_type - rule!((inheritance_specifier inherits_from: @ty) => (base_type type: {ty})), + // A named type (`Int`). `identifierType.name` is the type-name token. + rule!((identifierType name: @@n) => (named_type_expr name: (identifier #{n}))), + // A qualified type (`Outer.Inner`, `NSString.CompareOptions`). swift-syntax + // nests these as `memberType` nodes; like the old tree-sitter `user_type` + // rule, we keep the whole dotted path as the opaque `named_type_expr` name. + rule!((memberType) @ty => (named_type_expr name: (identifier #{ty}))), + // Sugared types desugar to `generic_type_expr`: `T?` -> Optional, + // `[T]` -> Array, `[K: V]` -> Dictionary. + rule!( + (optionalType wrappedType: @w) + => + (generic_type_expr base: (named_type_expr name: (identifier "Optional")) type_argument: {w}) + ), + rule!( + (arrayType element: @e) + => + (generic_type_expr base: (named_type_expr name: (identifier "Array")) type_argument: {e}) + ), + rule!( + (dictionaryType key: @k value: @v) + => + (generic_type_expr base: (named_type_expr name: (identifier "Dictionary")) type_argument: {k} type_argument: {v}) + ), + // A tuple type (`(Int, String)`) or function type (`(Int) -> Bool`). + // Both hold their contents as `tupleTypeElement`s, but a tuple element + // maps to `tuple_type_element` while a function parameter maps to + // `parameter`. Each container sets `ctx.in_function_type` for its direct + // children (and translates them explicitly, so a nested type is + // translated in the right context) and the shared `tupleTypeElement` + // rule below reads it. An element's label (`firstName`) is optional. + rule!( + (tupleType elements: _* @@elems) + => + tuple_type_expr { + ctx.in_function_type = false; + let mut out = Vec::new(); + for e in elems { + out.extend(ctx.translate(e)?); + } + tree!((tuple_type_expr element: {out})) + } + ), + rule!( + (functionType parameters: _* @@params returnClause: (returnClause type: @ret)) + => + function_type_expr { + ctx.in_function_type = true; + let mut out = Vec::new(); + for p in params { + out.extend(ctx.translate(p)?); + } + tree!((function_type_expr parameter: {out} return_type: {ret})) + } + ), + rule!( + (tupleTypeElement firstName: _? @@name type: @ty) + => + tuple_type_element { + let name = name.map(|n| tree!((identifier #{n}))); + if ctx.in_function_type { + tree!((parameter external_name: {name} type: {ty})) + } else { + tree!((tuple_type_element name: {name} type: {ty})) + } + } + ), + // Selector expression: `#selector(inner)` -- not yet supported + // (swift-syntax represents `#selector`/`#keyPath` and other macro + // expansions uniformly as a `macroExpansionExpr`). + rule!((macroExpansionExpr) => (unsupported_node)), + // PARITY(tree-sitter): a nominal type's `inheritanceClause` (`: Base, + // Proto`) is not emitted as a `base_type` — the tree-sitter path drops + // it (no corpus target has a `base_type`). swift-syntax exposes it + // cleanly, so emitting `base_type` is a correctness improvement to make + // once tree-sitter is retired. Each declaration keyword gets its own + // rule; the bodies are identical but for the keyword. // Class declaration with body containing members rule!( - (class_declaration - declaration_kind: @kind - name: @name - body: (class_body member: _* @members) - (inheritance_specifier)* @bases - (modifiers)* @mods) + (classDecl classKeyword: @kind modifiers: _* @mods name: @name memberBlock: (memberBlock members: _* @members)) => (class_like_declaration modifier: (modifier #{kind}) modifier: {mods} name: (identifier #{name}) - base_type: {bases} member: {members}) ), // Enum class declaration: same as a regular class but with an enum body. rule!( - (class_declaration - declaration_kind: @kind - name: @name - body: (enum_class_body member: _* @members) - (inheritance_specifier)* @bases - (modifiers)* @mods) + (enumDecl enumKeyword: @kind modifiers: _* @mods name: @name memberBlock: (memberBlock members: _* @members)) => (class_like_declaration modifier: (modifier #{kind}) modifier: {mods} name: (identifier #{name}) - base_type: {bases} member: {members}) ), - // Class declaration with empty body + // A `struct` declaration. rule!( - (class_declaration - declaration_kind: @kind - name: @name - body: _ - (inheritance_specifier)* @bases - (modifiers)* @mods) + (structDecl structKeyword: @kind modifiers: _* @mods name: @name memberBlock: (memberBlock members: _* @members)) => (class_like_declaration modifier: (modifier #{kind}) modifier: {mods} name: (identifier #{name}) - base_type: {bases}) + member: {members}) ), // Protocol declaration rule!( - (protocol_declaration - name: @name - body: (protocol_body member: _* @members) - (inheritance_specifier)* @bases - (modifiers)* @mods) + (protocolDecl protocolKeyword: @kind modifiers: _* @mods name: @name memberBlock: (memberBlock members: _* @members)) => (class_like_declaration - modifier: (modifier "protocol") + modifier: (modifier #{kind}) modifier: {mods} name: (identifier #{name}) - base_type: {bases} member: {members}) ), - // Protocol function — return type and body statements both optional. + // An `extension Foo { … }` is likewise a `class_like_declaration`, named + // by the extended type. The extended type is captured opaquely (as its + // source text) so that qualified names (`extension String.Interpolation`, + // a `memberType`) name the declaration just like simple ones, matching the + // old tree-sitter `user_type` behaviour. rule!( - (protocol_function_declaration - name: @name - (parameter)* @params - return_type: _? @ret - body: (block statement: _* @body_stmts)? - (modifiers)* @mods) + (extensionDecl extensionKeyword: @kind modifiers: _* @mods extendedType: @@name memberBlock: (memberBlock members: _* @members)) => - (function_declaration + (class_like_declaration + modifier: (modifier #{kind}) modifier: {mods} name: (identifier #{name}) - parameter: {params} - return_type: {ret} - body: (block stmt: {body_stmts})) + member: {members}) ), + // A member of a type declaration unwraps to the contained declaration. + rule!((memberBlockItem decl: _* @d) => member* { d }), // Init declaration → constructor_declaration. Body statements optional; // body itself is also optional (protocol requirement). + // + // PARITY(tree-sitter): the parameters are not emitted, because the + // tree-sitter path dropped them (its `(parameter)*` capture missed the + // field-attached parameters). Emitting them is a future improvement. rule!( - (init_declaration - (parameter)* @params - body: (block statement: _* @body_stmts)? - (modifiers)* @mods) + (initializerDecl + modifiers: _* @mods + body: (codeBlock statements: _* @body_stmts)?) => (constructor_declaration modifier: {mods} - parameter: {params} body: (block stmt: {body_stmts})) ), // Deinit declaration → destructor_declaration. Body statements optional. rule!( - (deinit_declaration - body: (block statement: _* @body_stmts) - (modifiers)* @mods) + (deinitializerDecl + modifiers: _* @mods + body: (codeBlock statements: _* @body_stmts)) => (destructor_declaration modifier: {mods} @@ -1074,165 +1115,28 @@ fn translation_rules() -> Vec> { ), // Typealias declaration rule!( - (typealias_declaration name: @name value: @val (modifiers)* @mods) + (typeAliasDecl + modifiers: _* @mods + name: @@name + initializer: (typeInitializerClause value: @val)) => (type_alias_declaration modifier: {mods} name: (identifier #{name}) r#type: {val}) ), - // Subscript declaration (not yet supported -- grammar needs to distinguish plain calls from subscript calls) - rule!( - (subscript_declaration (parameter)* @params (modifiers)* @mods) - => - (unsupported_node) - ), // Associated type declaration (with optional bound) rule!( - (associatedtype_declaration name: @name inherits_from: _? @bound (modifiers)* @mods) + (associatedTypeDecl + modifiers: _* @mods + name: @@name + inheritanceClause: (inheritanceClause inheritedTypes: (inheritedType type: @bound))?) => (associated_type_declaration modifier: {mods} name: (identifier #{name}) bound: {bound}) ), - // Protocol property declaration: translate each accessor - // requirement to an `accessor_declaration` carrying the property - // name, type, and outer modifiers. Manual rule: we publish the - // property's name/type/modifiers into `ctx` and translate each - // accessor with `ctx.is_chained` toggled per iteration so the - // inner `getter_specifier`/`setter_specifier` rules emit - // complete nodes from the start (including the - // `chained_declaration` tag for non-first accessors). - rule!( - (protocol_property_declaration - name: (pattern bound_identifier: @name) - requirements: (protocol_property_requirements accessor: _+ @@accessors) - type: _? @ty - (modifiers)* @mods) - => - accessor_declaration* { - ctx.property_name = Some(tree!((identifier #{name}))); - ctx.property_type = ty; - ctx.outer_modifiers = mods; - - let mut result = Vec::new(); - for (i, acc) in accessors.into_iter().enumerate() { - ctx.is_chained = i > 0; - result.extend(ctx.translate(acc)?); - } - result - } - ), - // getter_specifier / setter_specifier → bodyless accessor_declaration - // getter_specifier / setter_specifier → bodyless - // accessor_declaration. Reads property name/type/modifiers from - // `ctx` set by the outer `protocol_property_declaration` rule. - rule!( - (getter_specifier) - => - (accessor_declaration - name: {ctx.property_name.ok_or("getter_specifier outside protocol_property_declaration context")?} - type: {ctx.property_type} - accessor_kind: (accessor_kind "get") - modifier: {ctx.outer_modifiers.clone()} - modifier: {chained_modifier(&mut ctx)}) - ), - rule!( - (setter_specifier) - => - (accessor_declaration - name: {ctx.property_name.ok_or("setter_specifier outside protocol_property_declaration context")?} - type: {ctx.property_type} - accessor_kind: (accessor_kind "set") - modifier: {ctx.outer_modifiers.clone()} - modifier: {chained_modifier(&mut ctx)}) - ), - // protocol_property_requirements wrapper — should be consumed by above; fallback - rule!((protocol_property_requirements accessor: _* @accs) => accessor_declaration* { accs }), - // Computed getter → accessor_declaration (body optional). - // Reads property name/type from the outer property_binding rule - // and binding/outer modifiers + chained tag from the outer - // property_declaration rule. - rule!( - (computed_getter body: (block statement: _* @@body)?) - => - (accessor_declaration - modifier: {ctx.outer_modifiers.clone()} - modifier: {chained_modifier(&mut ctx)} - name: {ctx.property_name.ok_or("computed_getter outside property_binding context")?} - type: {ctx.property_type} - accessor_kind: (accessor_kind "get") - body: (block stmt: {ctx.reset(); ctx.translate(body)?})) - ), - // Computed setter with explicit parameter name. - rule!( - (computed_setter parameter: @param body: (block statement: _* @@body)) - => - (accessor_declaration - modifier: {ctx.outer_modifiers.clone()} - modifier: {chained_modifier(&mut ctx)} - name: {ctx.property_name.ok_or("computed_setter outside property_binding context")?} - type: {ctx.property_type} - accessor_kind: (accessor_kind "set") - parameter: (parameter pattern: (name_pattern identifier: (identifier #{param}))) - body: (block stmt: {ctx.reset(); ctx.translate(body)?})) - ), - // Computed setter without explicit parameter name; body optional. - rule!( - (computed_setter body: (block statement: _* @@body)?) - => - (accessor_declaration - modifier: {ctx.outer_modifiers.clone()} - modifier: {chained_modifier(&mut ctx)} - name: {ctx.property_name.ok_or("computed_setter outside property_binding context")?} - type: {ctx.property_type} - accessor_kind: (accessor_kind "set") - body: (block stmt: {ctx.reset(); ctx.translate(body)?})) - ), - // Computed modify → accessor_declaration - rule!( - (computed_modify body: (block statement: _* @@body)) - => - (accessor_declaration - modifier: {ctx.outer_modifiers.clone()} - modifier: {chained_modifier(&mut ctx)} - name: {ctx.property_name.ok_or("computed_modify outside property_binding context")?} - type: {ctx.property_type} - accessor_kind: (accessor_kind "modify") - body: (block stmt: {ctx.reset(); ctx.translate(body)?})) - ), - // willset/didset block — spread to children (only reachable as a - // fallback; the outer property_binding manual rule normally - // captures the willset/didset clauses directly). - rule!((willset_didset_block _* @clauses) => accessor_declaration* { clauses }), - // willset clause → accessor_declaration (body optional). Reads - // `ctx.property_name` set by the outer property_binding rule and - // binding/outer modifiers + chained tag from the outer - // property_declaration rule. - rule!( - (willset_clause body: (block statement: _* @@body)?) - => - (accessor_declaration - modifier: {ctx.outer_modifiers.clone()} - modifier: {chained_modifier(&mut ctx)} - name: {ctx.property_name.ok_or("willset_clause outside property_binding context")?} - accessor_kind: (accessor_kind "willSet") - body: (block stmt: {ctx.reset(); ctx.translate(body)?})) - ), - // didset clause → accessor_declaration (body optional). - rule!( - (didset_clause body: (block statement: _* @@body)?) - => - (accessor_declaration - modifier: {ctx.outer_modifiers.clone()} - modifier: {chained_modifier(&mut ctx)} - name: {ctx.property_name.ok_or("didset_clause outside property_binding context")?} - accessor_kind: (accessor_kind "didSet") - body: (block stmt: {ctx.reset(); ctx.translate(body)?})) - ), - // Preprocessor conditionals — unsupported - rule!((diagnostic) => (unsupported_node)), // ---- Fallbacks ---- // Bare `_` (rather than `(_)`) so this matches both named nodes // and unnamed tokens. Any unnamed token that escapes the @@ -1249,18 +1153,17 @@ fn translation_rules() -> Vec> { ] } -pub fn language_spec(desugared_ast_schema: &'static str) -> simple::LanguageSpec { - let ts_language: tree_sitter::Language = tree_sitter_swift::LANGUAGE.into(); +pub fn language_spec(desugared_ast_schema: &'static str) -> desugaring::LanguageSpec { let config = DesugaringConfig::::new() .add_phase("translate", PhaseKind::OneShot, translation_rules()) .with_output_node_types_yaml(desugared_ast_schema); - let desugarer = ConcreteDesugarer::new(ts_language.clone(), config) - .expect("failed to build Swift desugarer"); - simple::LanguageSpec { + let desugarer = + ConcreteDesugarer::without_language(config).expect("failed to build Swift desugarer"); + desugaring::LanguageSpec { prefix: "swift", - ts_language, - node_types: tree_sitter_swift::NODE_TYPES, + parser: Box::new(super::swift_parse::parse), + node_types: "", file_globs: vec!["*.swift".into(), "*.swiftinterface".into()], - desugar: Some(Box::new(desugarer)), + desugarer: Box::new(desugarer), } } diff --git a/unified/extractor/swift_node_types.yml b/unified/extractor/swift_node_types.yml new file mode 100644 index 00000000000..c98dd1d33b2 --- /dev/null +++ b/unified/extractor/swift_node_types.yml @@ -0,0 +1,1280 @@ +supertypes: + decl: + - accessorDecl + - actorDecl + - associatedTypeDecl + - classDecl + - deinitializerDecl + - editorPlaceholderDecl + - enumCaseDecl + - enumDecl + - extensionDecl + - functionDecl + - ifConfigDecl + - importDecl + - initializerDecl + - macroDecl + - macroExpansionDecl + - missingDecl + - operatorDecl + - poundSourceLocation + - precedenceGroupDecl + - protocolDecl + - structDecl + - subscriptDecl + - typeAliasDecl + - unexpectedCodeDecl + - usingDecl + - variableDecl + expr: + - _canImportExpr + - _canImportVersionInfo + - arrayExpr + - arrowExpr + - asExpr + - assignmentExpr + - awaitExpr + - binaryOperatorExpr + - booleanLiteralExpr + - borrowExpr + - closureExpr + - consumeExpr + - copyExpr + - declReferenceExpr + - dictionaryExpr + - discardAssignmentExpr + - doExpr + - editorPlaceholderExpr + - floatLiteralExpr + - forceUnwrapExpr + - functionCallExpr + - genericSpecializationExpr + - ifExpr + - inOutExpr + - infixOperatorExpr + - integerLiteralExpr + - isExpr + - keyPathExpr + - macroExpansionExpr + - memberAccessExpr + - missingExpr + - nilLiteralExpr + - optionalChainingExpr + - packElementExpr + - packExpansionExpr + - patternExpr + - postfixIfConfigExpr + - postfixOperatorExpr + - prefixOperatorExpr + - regexLiteralExpr + - sequenceExpr + - simpleStringLiteralExpr + - stringLiteralExpr + - subscriptCallExpr + - superExpr + - switchExpr + - ternaryExpr + - tryExpr + - tupleExpr + - typeExpr + - unresolvedAsExpr + - unresolvedIsExpr + - unresolvedTernaryExpr + - unsafeExpr + pattern: + - expressionPattern + - identifierPattern + - isTypePattern + - missingPattern + - tuplePattern + - valueBindingPattern + - wildcardPattern + stmt: + - breakStmt + - continueStmt + - deferStmt + - discardStmt + - doStmt + - expressionStmt + - fallThroughStmt + - forStmt + - guardStmt + - labeledStmt + - missingStmt + - repeatStmt + - returnStmt + - thenStmt + - throwStmt + - whileStmt + - yieldStmt + syntax: + - abiAttributeArguments + - accessorBlock + - accessorBlockFile + - accessorEffectSpecifiers + - accessorParameters + - arrayElement + - attribute + - attributeClauseFile + - availabilityArgument + - availabilityCondition + - availabilityLabeledArgument + - availabilityMacroDefinitionFile + - backDeployedAttributeArguments + - catchClause + - catchItem + - closureCapture + - closureCaptureClause + - closureCaptureSpecifier + - closureParameter + - closureParameterClause + - closureShorthandParameter + - closureSignature + - codeBlock + - codeBlockFile + - codeBlockItem + - compositionTypeElement + - conditionElement + - conformanceRequirement + - declModifier + - declModifierDetail + - declNameArgument + - declNameArguments + - deinitializerEffectSpecifiers + - derivativeAttributeArguments + - designatedType + - dictionaryElement + - differentiabilityArgument + - differentiabilityArguments + - differentiabilityWithRespectToArgument + - differentiableAttributeArguments + - documentationAttributeArgument + - dynamicReplacementAttributeArguments + - enumCaseElement + - enumCaseParameter + - enumCaseParameterClause + - expressionSegment + - functionEffectSpecifiers + - functionParameter + - functionParameterClause + - functionSignature + - genericArgument + - genericArgumentClause + - genericParameter + - genericParameterClause + - genericRequirement + - genericWhereClause + - ifConfigClause + - implementsAttributeArguments + - importPathComponent + - inheritanceClause + - inheritedType + - initializerClause + - keyPathComponent + - keyPathMethodComponent + - keyPathOptionalComponent + - keyPathPropertyComponent + - keyPathSubscriptComponent + - labeledExpr + - labeledSpecializeArgument + - layoutRequirement + - lifetimeSpecifierArgument + - lifetimeTypeSpecifier + - matchingPatternCondition + - memberBlock + - memberBlockItem + - memberBlockItemListFile + - missing + - moduleSelector + - multipleTrailingClosureElement + - nonisolatedSpecifierArgument + - nonisolatedTypeSpecifier + - objCSelectorPiece + - operatorPrecedenceAndTypes + - optionalBindingCondition + - originallyDefinedInAttributeArguments + - patternBinding + - platformVersion + - platformVersionItem + - poundSourceLocationArguments + - precedenceGroupAssignment + - precedenceGroupAssociativity + - precedenceGroupName + - precedenceGroupRelation + - primaryAssociatedType + - primaryAssociatedTypeClause + - returnClause + - sameTypeRequirement + - simpleTypeSpecifier + - sourceFile + - specializeAvailabilityArgument + - specializeTargetFunctionArgument + - specializedAttributeArgument + - stringSegment + - switchCase + - switchCaseItem + - switchCaseLabel + - switchDefaultLabel + - throwsClause + - tuplePatternElement + - tupleTypeElement + - typeAnnotation + - typeEffectSpecifiers + - typeInitializerClause + - versionComponent + - versionTuple + - whereClause + - yieldedExpression + - yieldedExpressionsClause + type: + - arrayType + - attributedType + - classRestrictionType + - compositionType + - dictionaryType + - functionType + - identifierType + - implicitlyUnwrappedOptionalType + - inlineArrayType + - memberType + - metatypeType + - missingType + - namedOpaqueReturnType + - optionalType + - packElementType + - packExpansionType + - someOrAnyType + - suppressedType + - tupleType +named: + _canImportExpr: + canImportKeyword: _token + leftParen: _token + importPath: _token + versionInfo?: _canImportVersionInfo + rightParen: _token + _canImportVersionInfo: + comma: _token + label: _token + colon: _token + version: versionTuple + abiAttributeArguments: + provider: [associatedTypeDecl, deinitializerDecl, enumCaseDecl, functionDecl, initializerDecl, missingDecl, subscriptDecl, typeAliasDecl, variableDecl] + accessorBlock: + leftBrace: _token + accessors: [accessorDecl, codeBlockItemList] + rightBrace: _token + accessorBlockFile: + leftBrace?: _token + accessors*: accessorDecl + rightBrace?: _token + endOfFileToken: _token + accessorDecl: + attributes*: [attribute, ifConfigDecl] + modifier?: declModifier + accessorSpecifier: _token + parameters?: accessorParameters + effectSpecifiers?: accessorEffectSpecifiers + body?: codeBlock + accessorEffectSpecifiers: + asyncSpecifier?: _token + throwsClause?: throwsClause + accessorParameters: + leftParen: _token + name: _token + rightParen: _token + actorDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + actorKeyword: _token + name: _token + genericParameterClause?: genericParameterClause + inheritanceClause?: inheritanceClause + genericWhereClause?: genericWhereClause + memberBlock: memberBlock + arrayElement: + expression: expr + trailingComma?: _token + arrayExpr: + leftSquare: _token + elements*: arrayElement + rightSquare: _token + arrayType: + leftSquare: _token + element: type + rightSquare: _token + arrowExpr: + effectSpecifiers?: typeEffectSpecifiers + arrow: _token + asExpr: + expression: expr + asKeyword: _token + questionOrExclamationMark?: _token + type: type + assignmentExpr: + equal: _token + associatedTypeDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + associatedtypeKeyword: _token + name: _token + inheritanceClause?: inheritanceClause + initializer?: typeInitializerClause + genericWhereClause?: genericWhereClause + attribute: + atSign: _token + attributeName: type + leftParen?: _token + arguments?: [labeledExprList, availabilityArgumentList, specializeAttributeArgumentList, specializedAttributeArgument, objCSelectorPieceList, implementsAttributeArguments, differentiableAttributeArguments, derivativeAttributeArguments, backDeployedAttributeArguments, originallyDefinedInAttributeArguments, dynamicReplacementAttributeArguments, effectsAttributeArgumentList, documentationAttributeArgumentList, abiAttributeArguments] + rightParen?: _token + attributeClauseFile: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + endOfFileToken: _token + attributedType: + specifiers*: [simpleTypeSpecifier, lifetimeTypeSpecifier, nonisolatedTypeSpecifier] + attributes*: [attribute, ifConfigDecl] + lateSpecifiers*: [simpleTypeSpecifier, lifetimeTypeSpecifier, nonisolatedTypeSpecifier] + baseType: type + availabilityArgument: + argument: [_token, platformVersion, availabilityLabeledArgument] + trailingComma?: _token + availabilityCondition: + availabilityKeyword: _token + leftParen: _token + availabilityArguments*: availabilityArgument + rightParen: _token + availabilityLabeledArgument: + label: _token + colon: _token + value: [simpleStringLiteralExpr, versionTuple] + availabilityMacroDefinitionFile: + platformVersion: platformVersion + colon: _token + specs*: availabilityArgument + endOfFileToken: _token + awaitExpr: + awaitKeyword: _token + expression: expr + backDeployedAttributeArguments: + beforeLabel: _token + colon: _token + platforms*: platformVersionItem + binaryOperatorExpr: + operator: _token + booleanLiteralExpr: + literal: _token + borrowExpr: + borrowKeyword: _token + expression: expr + breakStmt: + breakKeyword: _token + label?: _token + catchClause: + catchKeyword: _token + catchItems*: catchItem + body: codeBlock + catchItem: + pattern?: pattern + whereClause?: whereClause + trailingComma?: _token + classDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + classKeyword: _token + name: _token + genericParameterClause?: genericParameterClause + inheritanceClause?: inheritanceClause + genericWhereClause?: genericWhereClause + memberBlock: memberBlock + classRestrictionType: + classKeyword: _token + closureCapture: + specifier?: closureCaptureSpecifier + name: _token + initializer?: initializerClause + trailingComma?: _token + closureCaptureClause: + leftSquare: _token + items*: closureCapture + rightSquare: _token + closureCaptureSpecifier: + specifier: _token + leftParen?: _token + detail?: _token + rightParen?: _token + closureExpr: + leftBrace: _token + signature?: closureSignature + statements*: codeBlockItem + rightBrace: _token + closureParameter: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + firstName: _token + secondName?: _token + colon?: _token + type?: type + ellipsis?: _token + trailingComma?: _token + closureParameterClause: + leftParen: _token + parameters*: closureParameter + rightParen: _token + closureShorthandParameter: + name: _token + trailingComma?: _token + closureSignature: + attributes*: [attribute, ifConfigDecl] + capture?: closureCaptureClause + parameterClause?: [closureShorthandParameterList, closureParameterClause] + effectSpecifiers?: typeEffectSpecifiers + returnClause?: returnClause + inKeyword: _token + codeBlock: + leftBrace: _token + statements*: codeBlockItem + rightBrace: _token + codeBlockFile: + body: codeBlock + endOfFileToken: _token + codeBlockItem: + item: [decl, stmt, expr] + semicolon?: _token + compositionType: + elements*: compositionTypeElement + compositionTypeElement: + type: type + ampersand?: token + conditionElement: + condition: [expr, availabilityCondition, matchingPatternCondition, optionalBindingCondition] + trailingComma?: _token + conformanceRequirement: + leftType: type + colon: _token + rightType: type + consumeExpr: + consumeKeyword: _token + expression: expr + continueStmt: + continueKeyword: _token + label?: _token + copyExpr: + copyKeyword: _token + expression: expr + declModifier: + name: _token + detail?: declModifierDetail + declModifierDetail: + leftParen: _token + detail: _token + rightParen: _token + declNameArgument: + name: token + colon: _token + declNameArguments: + leftParen: _token + arguments*: declNameArgument + rightParen: _token + declReferenceExpr: + moduleSelector?: moduleSelector + baseName: _token + argumentNames?: declNameArguments + deferStmt: + deferKeyword: _token + body: codeBlock + deinitializerDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + deinitKeyword: _token + effectSpecifiers?: deinitializerEffectSpecifiers + body?: codeBlock + deinitializerEffectSpecifiers: + asyncSpecifier?: _token + derivativeAttributeArguments: + ofLabel: _token + colon: _token + originalDeclName: expr + period?: _token + accessorSpecifier?: _token + comma?: _token + arguments?: differentiabilityWithRespectToArgument + designatedType: + leadingComma: _token + name: token + dictionaryElement: + key: expr + colon: _token + value: expr + trailingComma?: _token + dictionaryExpr: + leftSquare: _token + content: [_token, dictionaryElementList] + rightSquare: _token + dictionaryType: + leftSquare: _token + key: type + colon: _token + value: type + rightSquare: _token + differentiabilityArgument: + argument: _token + trailingComma?: _token + differentiabilityArguments: + leftParen: _token + arguments*: differentiabilityArgument + rightParen: _token + differentiabilityWithRespectToArgument: + wrtLabel: _token + colon: _token + arguments: [differentiabilityArgument, differentiabilityArguments] + differentiableAttributeArguments: + kindSpecifier?: _token + kindSpecifierComma?: _token + arguments?: differentiabilityWithRespectToArgument + argumentsComma?: _token + genericWhereClause?: genericWhereClause + discardAssignmentExpr: + wildcard: _token + discardStmt: + discardKeyword: _token + expression: expr + doExpr: + doKeyword: _token + body: codeBlock + catchClauses*: catchClause + doStmt: + doKeyword: _token + throwsClause?: throwsClause + body: codeBlock + catchClauses*: catchClause + documentationAttributeArgument: + label: _token + colon: _token + value: [_token, stringLiteralExpr] + trailingComma?: _token + dynamicReplacementAttributeArguments: + forLabel: _token + colon: _token + declName: declReferenceExpr + editorPlaceholderDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + placeholder: _token + editorPlaceholderExpr: + placeholder: _token + enumCaseDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + caseKeyword: _token + elements*: enumCaseElement + enumCaseElement: + name: _token + parameterClause?: enumCaseParameterClause + rawValue?: initializerClause + trailingComma?: _token + enumCaseParameter: + modifiers*: declModifier + firstName?: _token + secondName?: _token + colon?: _token + type: type + defaultValue?: initializerClause + trailingComma?: _token + enumCaseParameterClause: + leftParen: _token + parameters*: enumCaseParameter + rightParen: _token + enumDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + enumKeyword: _token + name: _token + genericParameterClause?: genericParameterClause + inheritanceClause?: inheritanceClause + genericWhereClause?: genericWhereClause + memberBlock: memberBlock + expressionPattern: + expression: expr + expressionSegment: + backslash: _token + pounds?: _token + leftParen: _token + expressions*: labeledExpr + rightParen: _token + expressionStmt: + expression: expr + extensionDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + extensionKeyword: _token + extendedType: type + inheritanceClause?: inheritanceClause + genericWhereClause?: genericWhereClause + memberBlock: memberBlock + fallThroughStmt: + fallthroughKeyword: _token + floatLiteralExpr: + literal: _token + forStmt: + forKeyword: _token + tryKeyword?: _token + awaitKeyword?: _token + unsafeKeyword?: _token + caseKeyword?: _token + pattern: pattern + typeAnnotation?: typeAnnotation + inKeyword: _token + sequence: expr + whereClause?: whereClause + body: codeBlock + forceUnwrapExpr: + expression: expr + exclamationMark: _token + functionCallExpr: + calledExpression: expr + leftParen?: _token + arguments*: labeledExpr + rightParen?: _token + trailingClosure?: closureExpr + additionalTrailingClosures*: multipleTrailingClosureElement + functionDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + funcKeyword: _token + name: _token + genericParameterClause?: genericParameterClause + signature: functionSignature + genericWhereClause?: genericWhereClause + body?: codeBlock + functionEffectSpecifiers: + asyncSpecifier?: _token + throwsClause?: throwsClause + functionParameter: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + firstName: _token + secondName?: _token + colon: _token + type: type + ellipsis?: _token + defaultValue?: initializerClause + trailingComma?: _token + functionParameterClause: + leftParen: _token + parameters*: functionParameter + rightParen: _token + functionSignature: + parameterClause: functionParameterClause + effectSpecifiers?: functionEffectSpecifiers + returnClause?: returnClause + functionType: + leftParen: _token + parameters*: tupleTypeElement + rightParen: _token + effectSpecifiers?: typeEffectSpecifiers + returnClause: returnClause + genericArgument: + argument: [type, expr] + trailingComma?: _token + genericArgumentClause: + leftAngle: _token + arguments*: genericArgument + rightAngle: _token + genericParameter: + attributes*: [attribute, ifConfigDecl] + specifier?: _token + name: _token + colon?: _token + inheritedType?: type + trailingComma?: _token + genericParameterClause: + leftAngle: _token + parameters*: genericParameter + genericWhereClause?: genericWhereClause + rightAngle: _token + genericRequirement: + requirement: [sameTypeRequirement, conformanceRequirement, layoutRequirement] + trailingComma?: _token + genericSpecializationExpr: + expression: expr + genericArgumentClause: genericArgumentClause + genericWhereClause: + whereKeyword: _token + requirements*: genericRequirement + guardStmt: + guardKeyword: _token + conditions*: conditionElement + elseKeyword: _token + body: codeBlock + identifierPattern: + identifier: _token + identifierType: + moduleSelector?: moduleSelector + name: _token + genericArgumentClause?: genericArgumentClause + ifConfigClause: + poundKeyword: _token + condition?: expr + elements?: [codeBlockItemList, switchCaseList, memberBlockItemList, expr, attributeList] + ifConfigDecl: + clauses*: ifConfigClause + poundEndif: _token + ifExpr: + ifKeyword: _token + conditions*: conditionElement + body: codeBlock + elseKeyword?: _token + elseBody?: [ifExpr, codeBlock] + implementsAttributeArguments: + type: type + comma: _token + declName: declReferenceExpr + implicitlyUnwrappedOptionalType: + wrappedType: type + exclamationMark: _token + importDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + importKeyword: _token + importKindSpecifier?: _token + path*: importPathComponent + importPathComponent: + name: _token + trailingPeriod?: _token + inOutExpr: + ampersand: _token + expression: expr + infixOperatorExpr: + leftOperand: expr + operator: expr + rightOperand: expr + inheritanceClause: + colon: _token + inheritedTypes*: inheritedType + inheritedType: + type: type + trailingComma?: _token + initializerClause: + equal: _token + value: expr + initializerDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + initKeyword: _token + optionalMark?: _token + genericParameterClause?: genericParameterClause + signature: functionSignature + genericWhereClause?: genericWhereClause + body?: codeBlock + inlineArrayType: + leftSquare: _token + count: genericArgument + separator: _token + element: genericArgument + rightSquare: _token + integerLiteralExpr: + literal: _token + isExpr: + expression: expr + isKeyword: _token + type: type + isTypePattern: + isKeyword: _token + type: type + keyPathComponent: + period?: _token + component: [keyPathPropertyComponent, keyPathMethodComponent, keyPathSubscriptComponent, keyPathOptionalComponent] + keyPathExpr: + backslash: _token + root?: type + components*: keyPathComponent + keyPathMethodComponent: + declName: declReferenceExpr + leftParen: _token + arguments*: labeledExpr + rightParen: _token + keyPathOptionalComponent: + questionOrExclamationMark: _token + keyPathPropertyComponent: + declName: declReferenceExpr + genericArgumentClause?: genericArgumentClause + keyPathSubscriptComponent: + leftSquare: _token + arguments*: labeledExpr + rightSquare: _token + labeledExpr: + label?: _token + colon?: _token + expression: expr + trailingComma?: _token + labeledSpecializeArgument: + label: _token + colon: _token + value: token + trailingComma?: _token + labeledStmt: + label: _token + colon: _token + statement: stmt + layoutRequirement: + type: type + colon: _token + layoutSpecifier: _token + leftParen?: _token + size?: _token + comma?: _token + alignment?: _token + rightParen?: _token + lifetimeSpecifierArgument: + parameter: _token + trailingComma?: _token + lifetimeTypeSpecifier: + dependsOnKeyword: _token + leftParen: _token + scopedKeyword?: _token + arguments*: lifetimeSpecifierArgument + rightParen: _token + macroDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + macroKeyword: _token + name: _token + genericParameterClause?: genericParameterClause + signature: functionSignature + definition?: initializerClause + genericWhereClause?: genericWhereClause + macroExpansionDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + pound: _token + moduleSelector?: moduleSelector + macroName: _token + genericArgumentClause?: genericArgumentClause + leftParen?: _token + arguments*: labeledExpr + rightParen?: _token + trailingClosure?: closureExpr + additionalTrailingClosures*: multipleTrailingClosureElement + macroExpansionExpr: + pound: _token + moduleSelector?: moduleSelector + macroName: _token + genericArgumentClause?: genericArgumentClause + leftParen?: _token + arguments*: labeledExpr + rightParen?: _token + trailingClosure?: closureExpr + additionalTrailingClosures*: multipleTrailingClosureElement + matchingPatternCondition: + caseKeyword: _token + pattern: pattern + typeAnnotation?: typeAnnotation + initializer: initializerClause + memberAccessExpr: + base?: expr + period: _token + declName: declReferenceExpr + memberBlock: + leftBrace: _token + members*: memberBlockItem + rightBrace: _token + memberBlockItem: + decl: decl + semicolon?: _token + memberBlockItemListFile: + members*: memberBlockItem + endOfFileToken: _token + memberType: + baseType: type + period: _token + moduleSelector?: moduleSelector + name: _token + genericArgumentClause?: genericArgumentClause + metatypeType: + baseType: type + period: _token + metatypeSpecifier: _token + missing: + placeholder: _token + missingDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + placeholder: _token + missingExpr: + placeholder: _token + missingPattern: + placeholder: _token + missingStmt: + placeholder: _token + missingType: + placeholder: _token + moduleSelector: + moduleName: _token + colonColon: _token + multipleTrailingClosureElement: + label: _token + colon: _token + closure: closureExpr + namedOpaqueReturnType: + genericParameterClause: genericParameterClause + type: type + nilLiteralExpr: + nilKeyword: _token + nonisolatedSpecifierArgument: + leftParen: _token + nonsendingKeyword: _token + rightParen: _token + nonisolatedTypeSpecifier: + nonisolatedKeyword: _token + argument?: nonisolatedSpecifierArgument + objCSelectorPiece: + name?: token + colon?: _token + operatorDecl: + fixitySpecifier: _token + operatorKeyword: _token + name: _token + operatorPrecedenceAndTypes?: operatorPrecedenceAndTypes + operatorPrecedenceAndTypes: + colon: _token + precedenceGroup: _token + designatedTypes*: designatedType + optionalBindingCondition: + bindingSpecifier: _token + pattern: pattern + typeAnnotation?: typeAnnotation + initializer?: initializerClause + optionalChainingExpr: + expression: expr + questionMark: _token + optionalType: + wrappedType: type + questionMark: _token + originallyDefinedInAttributeArguments: + moduleLabel: _token + colon: _token + moduleName: stringLiteralExpr + comma: _token + platforms*: platformVersionItem + packElementExpr: + eachKeyword: _token + pack: expr + packElementType: + eachKeyword: _token + pack: type + packExpansionExpr: + repeatKeyword: _token + repetitionPattern: expr + packExpansionType: + repeatKeyword: _token + repetitionPattern: type + patternBinding: + pattern: pattern + typeAnnotation?: typeAnnotation + initializer?: initializerClause + accessorBlock?: accessorBlock + trailingComma?: _token + patternExpr: + pattern: pattern + platformVersion: + platform: _token + version?: versionTuple + platformVersionItem: + platformVersion: platformVersion + trailingComma?: _token + postfixIfConfigExpr: + base?: expr + config: ifConfigDecl + postfixOperatorExpr: + expression: expr + operator: _token + poundSourceLocation: + poundSourceLocation: _token + leftParen: _token + arguments?: poundSourceLocationArguments + rightParen: _token + poundSourceLocationArguments: + fileLabel: _token + fileColon: _token + fileName: simpleStringLiteralExpr + comma: _token + lineLabel: _token + lineColon: _token + lineNumber: _token + precedenceGroupAssignment: + assignmentLabel: _token + colon: _token + value: _token + precedenceGroupAssociativity: + associativityLabel: _token + colon: _token + value: _token + precedenceGroupDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + precedencegroupKeyword: _token + name: _token + leftBrace: _token + groupAttributes*: [precedenceGroupRelation, precedenceGroupAssignment, precedenceGroupAssociativity] + rightBrace: _token + precedenceGroupName: + name: _token + trailingComma?: _token + precedenceGroupRelation: + higherThanOrLowerThanLabel: _token + colon: _token + precedenceGroups*: precedenceGroupName + prefixOperatorExpr: + operator: _token + expression: expr + primaryAssociatedType: + name: _token + trailingComma?: _token + primaryAssociatedTypeClause: + leftAngle: _token + primaryAssociatedTypes*: primaryAssociatedType + rightAngle: _token + protocolDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + protocolKeyword: _token + name: _token + primaryAssociatedTypeClause?: primaryAssociatedTypeClause + inheritanceClause?: inheritanceClause + genericWhereClause?: genericWhereClause + memberBlock: memberBlock + regexLiteralExpr: + openingPounds?: _token + openingSlash: _token + regex: _token + closingSlash: _token + closingPounds?: _token + repeatStmt: + repeatKeyword: _token + body: codeBlock + whileKeyword: _token + condition: expr + returnClause: + arrow: _token + type: type + returnStmt: + returnKeyword: _token + expression?: expr + sameTypeRequirement: + leftType: [type, expr] + equal: _token + rightType: [type, expr] + sequenceExpr: + elements*: expr + simpleStringLiteralExpr: + openingQuote: _token + segments*: stringSegment + closingQuote: _token + simpleTypeSpecifier: + specifier: _token + someOrAnyType: + someOrAnySpecifier: _token + constraint: type + sourceFile: + shebang?: _token + statements*: codeBlockItem + endOfFileToken: _token + specializeAvailabilityArgument: + availabilityLabel: _token + colon: _token + availabilityArguments*: availabilityArgument + semicolon: _token + specializeTargetFunctionArgument: + targetLabel: _token + colon: _token + declName: declReferenceExpr + trailingComma?: _token + specializedAttributeArgument: + genericWhereClause: genericWhereClause + stringLiteralExpr: + openingPounds?: _token + openingQuote: _token + segments*: [stringSegment, expressionSegment] + closingQuote: _token + closingPounds?: _token + stringSegment: + content: _token + structDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + structKeyword: _token + name: _token + genericParameterClause?: genericParameterClause + inheritanceClause?: inheritanceClause + genericWhereClause?: genericWhereClause + memberBlock: memberBlock + subscriptCallExpr: + calledExpression: expr + leftSquare: _token + arguments*: labeledExpr + rightSquare: _token + trailingClosure?: closureExpr + additionalTrailingClosures*: multipleTrailingClosureElement + subscriptDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + subscriptKeyword: _token + genericParameterClause?: genericParameterClause + parameterClause: functionParameterClause + returnClause: returnClause + genericWhereClause?: genericWhereClause + accessorBlock?: accessorBlock + superExpr: + superKeyword: _token + suppressedType: + withoutTilde: _token + type: type + switchCase: + attribute?: attribute + label: [switchDefaultLabel, switchCaseLabel] + statements*: codeBlockItem + switchCaseItem: + pattern: pattern + whereClause?: whereClause + trailingComma?: _token + switchCaseLabel: + caseKeyword: _token + caseItems*: switchCaseItem + colon: _token + switchDefaultLabel: + defaultKeyword: _token + colon: _token + switchExpr: + switchKeyword: _token + subject: expr + leftBrace: _token + cases*: [switchCase, ifConfigDecl] + rightBrace: _token + ternaryExpr: + condition: expr + questionMark: _token + thenExpression: expr + colon: _token + elseExpression: expr + thenStmt: + thenKeyword: _token + expression: expr + throwStmt: + throwKeyword: _token + expression: expr + throwsClause: + throwsSpecifier: _token + leftParen?: _token + type?: type + rightParen?: _token + tryExpr: + tryKeyword: _token + questionOrExclamationMark?: _token + expression: expr + tupleExpr: + leftParen: _token + elements*: labeledExpr + rightParen: _token + tuplePattern: + leftParen: _token + elements*: tuplePatternElement + rightParen: _token + tuplePatternElement: + label?: _token + colon?: _token + pattern: pattern + trailingComma?: _token + tupleType: + leftParen: _token + elements*: tupleTypeElement + rightParen: _token + tupleTypeElement: + inoutKeyword?: _token + firstName?: _token + secondName?: _token + colon?: _token + type: type + ellipsis?: _token + trailingComma?: _token + typeAliasDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + typealiasKeyword: _token + name: _token + genericParameterClause?: genericParameterClause + initializer: typeInitializerClause + genericWhereClause?: genericWhereClause + typeAnnotation: + colon: _token + type: type + typeEffectSpecifiers: + asyncSpecifier?: _token + throwsClause?: throwsClause + typeExpr: + type: type + typeInitializerClause: + equal: _token + value: type + unexpectedCodeDecl: + unresolvedAsExpr: + asKeyword: _token + questionOrExclamationMark?: _token + unresolvedIsExpr: + isKeyword: _token + unresolvedTernaryExpr: + questionMark: _token + thenExpression: expr + colon: _token + unsafeExpr: + unsafeKeyword: _token + expression: expr + usingDecl: + usingKeyword: _token + specifier: [attribute, _token] + valueBindingPattern: + bindingSpecifier: _token + pattern: pattern + variableDecl: + attributes*: [attribute, ifConfigDecl] + modifiers*: declModifier + bindingSpecifier: _token + bindings*: patternBinding + versionComponent: + period: _token + number: _token + versionTuple: + major: _token + components*: versionComponent + whereClause: + whereKeyword: _token + condition: expr + whileStmt: + whileKeyword: _token + conditions*: conditionElement + body: codeBlock + wildcardPattern: + wildcard: _token + yieldStmt: + yieldKeyword: _token + yieldedExpressions: [yieldedExpressionsClause, expr] + yieldedExpression: + expression: expr + comma?: _token + yieldedExpressionsClause: + leftParen: _token + elements*: yieldedExpression + rightParen: _token + _token: + binaryOperator: + dollarIdentifier: + floatLiteral: + identifier: + integerLiteral: + postfixOperator: + prefixOperator: + rawStringPoundDelimiter: + regexLiteralPattern: + regexPoundDelimiter: + shebang: + unknown: diff --git a/unified/extractor/tests/corpus/swift/closures/closure-with-capture-list.output b/unified/extractor/tests/corpus/swift/closures/closure-with-capture-list.output index 8f28322b493..6833344de16 100644 --- a/unified/extractor/tests/corpus/swift/closures/closure-with-capture-list.output +++ b/unified/extractor/tests/corpus/swift/closures/closure-with-capture-list.output @@ -2,41 +2,61 @@ let f = { [weak self] in self?.doThing() } --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "f" - value: - lambda_literal - captures: - capture_list - item: - capture_list_item - name: simple_identifier "self" - ownership: - ownership_modifier - statement: - call_expression - function: - navigation_expression - suffix: - navigation_suffix - suffix: simple_identifier "doThing" - target: - optional_chain_marker - expr: - self_expression - suffix: - call_suffix - arguments: - value_arguments +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + closureExpr + leftBrace: { + rightBrace: } + signature: + closureSignature + attributes: + capture: + closureCaptureClause + leftSquare: [ + rightSquare: ] + items: + closureCapture + name: self + specifier: + closureCaptureSpecifier + specifier: weak + inKeyword: in + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + additionalTrailingClosures: + calledExpression: + memberAccessExpr + period: . + declName: + declReferenceExpr + baseName: identifier "doThing" + base: + optionalChainingExpr + expression: + declReferenceExpr + baseName: self + questionMark: ? + pattern: + identifierPattern + identifier: identifier "f" --- @@ -51,6 +71,12 @@ top_level identifier: identifier "f" value: function_expr + capture_declaration: + variable_declaration + modifier: modifier "weak" + pattern: + name_pattern + identifier: identifier "self" body: block stmt: @@ -61,9 +87,3 @@ top_level name_expr identifier: identifier "self" member: identifier "doThing" - capture_declaration: - variable_declaration - modifier: modifier "weak" - pattern: - name_pattern - identifier: identifier "self" diff --git a/unified/extractor/tests/corpus/swift/closures/closure-with-explicit-parameters.output b/unified/extractor/tests/corpus/swift/closures/closure-with-explicit-parameters.output index 95d638118d8..c20da77c72c 100644 --- a/unified/extractor/tests/corpus/swift/closures/closure-with-explicit-parameters.output +++ b/unified/extractor/tests/corpus/swift/closures/closure-with-explicit-parameters.output @@ -2,45 +2,63 @@ let f = { (x: Int) -> Int in x * 2 } --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "f" - value: - lambda_literal - statement: - multiplicative_expression - lhs: simple_identifier "x" - op: * - rhs: integer_literal "2" - type: - lambda_function_type - params: - lambda_function_type_parameters - parameter: - lambda_parameter - name: simple_identifier "x" - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" - return_type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + closureExpr + leftBrace: { + rightBrace: } + signature: + closureSignature + attributes: + inKeyword: in + parameterClause: + closureParameterClause + leftParen: ( + rightParen: ) + parameters: + closureParameter + colon: : + attributes: + modifiers: + type: + identifierType + name: identifier "Int" + firstName: identifier "x" + returnClause: + returnClause + arrow: -> + type: + identifierType + name: identifier "Int" + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "*" + leftOperand: + declReferenceExpr + baseName: identifier "x" + rightOperand: + integerLiteralExpr + literal: integerLiteral "2" + pattern: + identifierPattern + identifier: identifier "f" --- @@ -55,23 +73,23 @@ top_level identifier: identifier "f" value: function_expr + parameter: + parameter + type: + named_type_expr + name: identifier "Int" + pattern: + name_pattern + identifier: identifier "x" + return_type: + named_type_expr + name: identifier "Int" body: block stmt: binary_expr - operator: infix_operator "*" left: name_expr identifier: identifier "x" + operator: infix_operator "*" right: int_literal "2" - parameter: - parameter - pattern: - name_pattern - identifier: identifier "x" - type: - named_type_expr - name: identifier "Int" - return_type: - named_type_expr - name: identifier "Int" diff --git a/unified/extractor/tests/corpus/swift/closures/closure-with-shorthand-parameters.output b/unified/extractor/tests/corpus/swift/closures/closure-with-shorthand-parameters.output index bd286c38579..1b07604e02c 100644 --- a/unified/extractor/tests/corpus/swift/closures/closure-with-shorthand-parameters.output +++ b/unified/extractor/tests/corpus/swift/closures/closure-with-shorthand-parameters.output @@ -2,24 +2,40 @@ let f = { $0 + $1 } --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "f" - value: - lambda_literal - statement: - additive_expression - lhs: simple_identifier "$0" - op: + - rhs: simple_identifier "$1" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + closureExpr + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "+" + leftOperand: + declReferenceExpr + baseName: dollarIdentifier "$0" + rightOperand: + declReferenceExpr + baseName: dollarIdentifier "$1" + pattern: + identifierPattern + identifier: identifier "f" --- @@ -38,10 +54,10 @@ top_level block stmt: binary_expr - operator: infix_operator "+" left: name_expr identifier: identifier "$0" + operator: infix_operator "+" right: name_expr identifier: identifier "$1" diff --git a/unified/extractor/tests/corpus/swift/closures/multi-statement-closure.output b/unified/extractor/tests/corpus/swift/closures/multi-statement-closure.output index 6c9a403f19c..4fbc1c8f71f 100644 --- a/unified/extractor/tests/corpus/swift/closures/multi-statement-closure.output +++ b/unified/extractor/tests/corpus/swift/closures/multi-statement-closure.output @@ -5,62 +5,91 @@ let f = { (x: Int) -> Int in --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "f" - value: - lambda_literal - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "y" - value: - additive_expression - lhs: simple_identifier "x" - op: + - rhs: integer_literal "1" - control_transfer_statement - kind: return - result: - multiplicative_expression - lhs: simple_identifier "y" - op: * - rhs: integer_literal "2" - type: - lambda_function_type - params: - lambda_function_type_parameters - parameter: - lambda_parameter - name: simple_identifier "x" - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" - return_type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + closureExpr + leftBrace: { + rightBrace: } + signature: + closureSignature + attributes: + inKeyword: in + parameterClause: + closureParameterClause + leftParen: ( + rightParen: ) + parameters: + closureParameter + colon: : + attributes: + modifiers: + type: + identifierType + name: identifier "Int" + firstName: identifier "x" + returnClause: + returnClause + arrow: -> + type: + identifierType + name: identifier "Int" + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "+" + leftOperand: + declReferenceExpr + baseName: identifier "x" + rightOperand: + integerLiteralExpr + literal: integerLiteral "1" + pattern: + identifierPattern + identifier: identifier "y" + codeBlockItem + item: + returnStmt + expression: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "*" + leftOperand: + declReferenceExpr + baseName: identifier "y" + rightOperand: + integerLiteralExpr + literal: integerLiteral "2" + returnKeyword: return + pattern: + identifierPattern + identifier: identifier "f" --- @@ -75,6 +104,17 @@ top_level identifier: identifier "f" value: function_expr + parameter: + parameter + type: + named_type_expr + name: identifier "Int" + pattern: + name_pattern + identifier: identifier "x" + return_type: + named_type_expr + name: identifier "Int" body: block stmt: @@ -85,27 +125,16 @@ top_level identifier: identifier "y" value: binary_expr - operator: infix_operator "+" left: name_expr identifier: identifier "x" + operator: infix_operator "+" right: int_literal "1" return_expr value: binary_expr - operator: infix_operator "*" left: name_expr identifier: identifier "y" + operator: infix_operator "*" right: int_literal "2" - parameter: - parameter - pattern: - name_pattern - identifier: identifier "x" - type: - named_type_expr - name: identifier "Int" - return_type: - named_type_expr - name: identifier "Int" diff --git a/unified/extractor/tests/corpus/swift/closures/trailing-closure.output b/unified/extractor/tests/corpus/swift/closures/trailing-closure.output index 56b8bf9a7c2..3e26ff27eb5 100644 --- a/unified/extractor/tests/corpus/swift/closures/trailing-closure.output +++ b/unified/extractor/tests/corpus/swift/closures/trailing-closure.output @@ -2,24 +2,40 @@ xs.map { $0 * 2 } --- -source_file - statement: - call_expression - function: - navigation_expression - suffix: - navigation_suffix - suffix: simple_identifier "map" - target: simple_identifier "xs" - suffix: - call_suffix - lambda: - lambda_literal - statement: - multiplicative_expression - lhs: simple_identifier "$0" - op: * - rhs: integer_literal "2" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + functionCallExpr + arguments: + additionalTrailingClosures: + calledExpression: + memberAccessExpr + period: . + declName: + declReferenceExpr + baseName: identifier "map" + base: + declReferenceExpr + baseName: identifier "xs" + trailingClosure: + closureExpr + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "*" + leftOperand: + declReferenceExpr + baseName: dollarIdentifier "$0" + rightOperand: + integerLiteralExpr + literal: integerLiteral "2" --- @@ -28,6 +44,12 @@ top_level block stmt: call_expr + callee: + member_access_expr + base: + name_expr + identifier: identifier "xs" + member: identifier "map" argument: argument value: @@ -36,14 +58,8 @@ top_level block stmt: binary_expr - operator: infix_operator "*" left: name_expr identifier: identifier "$0" + operator: infix_operator "*" right: int_literal "2" - callee: - member_access_expr - base: - name_expr - identifier: identifier "xs" - member: identifier "map" diff --git a/unified/extractor/tests/corpus/swift/collections/array-literal.output b/unified/extractor/tests/corpus/swift/collections/array-literal.output index f6ee44c1f80..c6ea2c2094f 100644 --- a/unified/extractor/tests/corpus/swift/collections/array-literal.output +++ b/unified/extractor/tests/corpus/swift/collections/array-literal.output @@ -2,23 +2,42 @@ let xs = [1, 2, 3] --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "xs" - value: - array_literal - element: - integer_literal "1" - integer_literal "2" - integer_literal "3" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + arrayExpr + elements: + arrayElement + expression: + integerLiteralExpr + literal: integerLiteral "1" + trailingComma: , + arrayElement + expression: + integerLiteralExpr + literal: integerLiteral "2" + trailingComma: , + arrayElement + expression: + integerLiteralExpr + literal: integerLiteral "3" + leftSquare: [ + rightSquare: ] + pattern: + identifierPattern + identifier: identifier "xs" --- diff --git a/unified/extractor/tests/corpus/swift/collections/dictionary-literal.output b/unified/extractor/tests/corpus/swift/collections/dictionary-literal.output index a19028d3f3b..edd306a69cb 100644 --- a/unified/extractor/tests/corpus/swift/collections/dictionary-literal.output +++ b/unified/extractor/tests/corpus/swift/collections/dictionary-literal.output @@ -2,30 +2,53 @@ let d = ["a": 1, "b": 2] --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "d" - value: - dictionary_literal - element: - dictionary_literal_item - key: - line_string_literal - text: line_str_text "a" - value: integer_literal "1" - dictionary_literal_item - key: - line_string_literal - text: line_str_text "b" - value: integer_literal "2" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + dictionaryExpr + leftSquare: [ + rightSquare: ] + content: + dictionaryElement + colon: : + trailingComma: , + value: + integerLiteralExpr + literal: integerLiteral "1" + key: + stringLiteralExpr + closingQuote: " + openingQuote: " + segments: + stringSegment + content: stringSegment "a" + dictionaryElement + colon: : + value: + integerLiteralExpr + literal: integerLiteral "2" + key: + stringLiteralExpr + closingQuote: " + openingQuote: " + segments: + stringSegment + content: stringSegment "b" + pattern: + identifierPattern + identifier: identifier "d" --- diff --git a/unified/extractor/tests/corpus/swift/collections/dictionary-subscript.output b/unified/extractor/tests/corpus/swift/collections/dictionary-subscript.output index 59afc51867a..e2f939e0683 100644 --- a/unified/extractor/tests/corpus/swift/collections/dictionary-subscript.output +++ b/unified/extractor/tests/corpus/swift/collections/dictionary-subscript.output @@ -4,31 +4,40 @@ let v = d["key"] --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "v" - value: - call_expression - function: simple_identifier "d" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: - line_string_literal - text: line_str_text "key" - comment "// TODO: same parser issue as the array subscript case above —" - comment "// `d[\"key\"]` is parsed as `call_expression(d, (\"key\"))`." +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + subscriptCallExpr + leftSquare: [ + rightSquare: ] + arguments: + labeledExpr + expression: + stringLiteralExpr + closingQuote: " + openingQuote: " + segments: + stringSegment + content: stringSegment "key" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "d" + pattern: + identifierPattern + identifier: identifier "v" --- @@ -43,9 +52,9 @@ top_level identifier: identifier "v" value: call_expr - argument: - argument - value: string_literal "\"key\"" callee: name_expr identifier: identifier "d" + argument: + argument + value: string_literal "\"key\"" diff --git a/unified/extractor/tests/corpus/swift/collections/empty-array-literal-with-type.output b/unified/extractor/tests/corpus/swift/collections/empty-array-literal-with-type.output index 90d1a9dde36..e9a70a43bb4 100644 --- a/unified/extractor/tests/corpus/swift/collections/empty-array-literal-with-type.output +++ b/unified/extractor/tests/corpus/swift/collections/empty-array-literal-with-type.output @@ -2,32 +2,38 @@ let xs: [Int] = [] --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "xs" - type: - type_annotation - type: - type - name: - array_type +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + arrayExpr + elements: + leftSquare: [ + rightSquare: ] + pattern: + identifierPattern + identifier: identifier "xs" + typeAnnotation: + typeAnnotation + colon: : + type: + arrayType + leftSquare: [ + rightSquare: ] element: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" - value: - array_literal + identifierType + name: identifier "Int" --- diff --git a/unified/extractor/tests/corpus/swift/collections/set-literal.output b/unified/extractor/tests/corpus/swift/collections/set-literal.output index a1a1dde75f9..c29492c9bc0 100644 --- a/unified/extractor/tests/corpus/swift/collections/set-literal.output +++ b/unified/extractor/tests/corpus/swift/collections/set-literal.output @@ -2,41 +2,57 @@ let s: Set = [1, 2, 3] --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "s" - type: - type_annotation - type: - type - name: - user_type - part: - simple_user_type +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + arrayExpr + elements: + arrayElement + expression: + integerLiteralExpr + literal: integerLiteral "1" + trailingComma: , + arrayElement + expression: + integerLiteralExpr + literal: integerLiteral "2" + trailingComma: , + arrayElement + expression: + integerLiteralExpr + literal: integerLiteral "3" + leftSquare: [ + rightSquare: ] + pattern: + identifierPattern + identifier: identifier "s" + typeAnnotation: + typeAnnotation + colon: : + type: + identifierType + name: identifier "Set" + genericArgumentClause: + genericArgumentClause arguments: - type_arguments + genericArgument argument: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" - name: type_identifier "Set" - value: - array_literal - element: - integer_literal "1" - integer_literal "2" - integer_literal "3" + identifierType + name: identifier "Int" + leftAngle: < + rightAngle: > --- diff --git a/unified/extractor/tests/corpus/swift/collections/subscript-access.output b/unified/extractor/tests/corpus/swift/collections/subscript-access.output index 481a3e95f77..f7e518b8477 100644 --- a/unified/extractor/tests/corpus/swift/collections/subscript-access.output +++ b/unified/extractor/tests/corpus/swift/collections/subscript-access.output @@ -5,30 +5,36 @@ let first = xs[0] --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "first" - value: - call_expression - function: simple_identifier "xs" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: integer_literal "0" - comment "// TODO: tree-sitter-swift parses `xs[0]` as a call_expression (same shape" - comment "// as `xs(0)`), so the mapping currently produces a call_expr. Update the" - comment "// parser / add a separate subscript_expr node and remap when fixed." +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + subscriptCallExpr + leftSquare: [ + rightSquare: ] + arguments: + labeledExpr + expression: + integerLiteralExpr + literal: integerLiteral "0" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "xs" + pattern: + identifierPattern + identifier: identifier "first" --- @@ -43,9 +49,9 @@ top_level identifier: identifier "first" value: call_expr - argument: - argument - value: int_literal "0" callee: name_expr identifier: identifier "xs" + argument: + argument + value: int_literal "0" diff --git a/unified/extractor/tests/corpus/swift/collections/tuple-literal.output b/unified/extractor/tests/corpus/swift/collections/tuple-literal.output index a0ac8861674..facbc2fcbb4 100644 --- a/unified/extractor/tests/corpus/swift/collections/tuple-literal.output +++ b/unified/extractor/tests/corpus/swift/collections/tuple-literal.output @@ -2,28 +2,46 @@ let t = (1, "two", 3.0) --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "t" - value: - tuple_expression - element: - tuple_expression_item - value: integer_literal "1" - tuple_expression_item +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = value: - line_string_literal - text: line_str_text "two" - tuple_expression_item - value: real_literal "3.0" + tupleExpr + leftParen: ( + rightParen: ) + elements: + labeledExpr + expression: + integerLiteralExpr + literal: integerLiteral "1" + trailingComma: , + labeledExpr + expression: + stringLiteralExpr + closingQuote: " + openingQuote: " + segments: + stringSegment + content: stringSegment "two" + trailingComma: , + labeledExpr + expression: + floatLiteralExpr + literal: floatLiteral "3.0" + pattern: + identifierPattern + identifier: identifier "t" --- diff --git a/unified/extractor/tests/corpus/swift/collections/tuple-member-access.output b/unified/extractor/tests/corpus/swift/collections/tuple-member-access.output index 29b234f30e2..6ecbd22eebe 100644 --- a/unified/extractor/tests/corpus/swift/collections/tuple-member-access.output +++ b/unified/extractor/tests/corpus/swift/collections/tuple-member-access.output @@ -2,23 +2,32 @@ let n = t.0 --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "n" - value: - navigation_expression - suffix: - navigation_suffix - suffix: integer_literal "0" - target: simple_identifier "t" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + memberAccessExpr + period: . + declName: + declReferenceExpr + baseName: integerLiteral "0" + base: + declReferenceExpr + baseName: identifier "t" + pattern: + identifierPattern + identifier: identifier "n" --- diff --git a/unified/extractor/tests/corpus/swift/control-flow/binding-modifier-does-not-leak-to-sibling.output b/unified/extractor/tests/corpus/swift/control-flow/binding-modifier-does-not-leak-to-sibling.output index 3919b875b96..800647eb747 100644 --- a/unified/extractor/tests/corpus/swift/control-flow/binding-modifier-does-not-leak-to-sibling.output +++ b/unified/extractor/tests/corpus/swift/control-flow/binding-modifier-does-not-leak-to-sibling.output @@ -8,44 +8,79 @@ default: --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "x" - value: integer_literal "1" - switch_statement - entry: - switch_entry - pattern: - switch_pattern +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + integerLiteralExpr + literal: integerLiteral "1" pattern: - pattern - kind: simple_identifier "someConstant" - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: - line_string_literal - text: line_str_text "matched" - switch_entry - default: default_keyword "default" - statement: - control_transfer_statement - kind: break - expr: simple_identifier "y" + identifierPattern + identifier: identifier "x" + codeBlockItem + item: + expressionStmt + expression: + switchExpr + leftBrace: { + rightBrace: } + cases: + switchCase + label: + switchCaseLabel + colon: : + caseKeyword: case + caseItems: + switchCaseItem + pattern: + expressionPattern + expression: + declReferenceExpr + baseName: identifier "someConstant" + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + stringLiteralExpr + closingQuote: " + openingQuote: " + segments: + stringSegment + content: stringSegment "matched" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + switchCase + label: + switchDefaultLabel + colon: : + defaultKeyword: default + statements: + codeBlockItem + item: + breakStmt + breakKeyword: break + subject: + declReferenceExpr + baseName: identifier "y" + switchKeyword: switch --- @@ -60,27 +95,27 @@ top_level identifier: identifier "x" value: int_literal "1" switch_expr + value: + name_expr + identifier: identifier "y" case: switch_case - body: - block - stmt: - call_expr - argument: - argument - value: string_literal "\"matched\"" - callee: - name_expr - identifier: identifier "print" pattern: expr_equality_pattern expr: name_expr identifier: identifier "someConstant" + body: + block + stmt: + call_expr + callee: + name_expr + identifier: identifier "print" + argument: + argument + value: string_literal "\"matched\"" switch_case body: block stmt: break_expr "break" - value: - name_expr - identifier: identifier "y" diff --git a/unified/extractor/tests/corpus/swift/control-flow/guard-let.output b/unified/extractor/tests/corpus/swift/control-flow/guard-let.output index a2196234121..d739eca564b 100644 --- a/unified/extractor/tests/corpus/swift/control-flow/guard-let.output +++ b/unified/extractor/tests/corpus/swift/control-flow/guard-let.output @@ -2,25 +2,37 @@ guard let value = optional else { return } --- -source_file - statement: - guard_statement - body: - block - statement: - control_transfer_statement - kind: return - condition: - if_condition - kind: - if_let_binding - pattern: - pattern - binding: - value_binding_pattern - mutability: let - bound_identifier: simple_identifier "value" - value: simple_identifier "optional" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + guardStmt + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + returnStmt + returnKeyword: return + conditions: + conditionElement + condition: + optionalBindingCondition + initializer: + initializerClause + equal: = + value: + declReferenceExpr + baseName: identifier "optional" + pattern: + identifierPattern + identifier: identifier "value" + bindingSpecifier: let + elseKeyword: else + guardKeyword: guard --- @@ -33,17 +45,17 @@ top_level pattern_guard_expr pattern: constructor_pattern - element: - pattern_element - pattern: - name_pattern - identifier: identifier "value" constructor: member_access_expr base: named_type_expr name: identifier "Optional" member: identifier "some" + element: + pattern_element + pattern: + name_pattern + identifier: identifier "value" value: name_expr identifier: identifier "optional" diff --git a/unified/extractor/tests/corpus/swift/control-flow/if-case-let-with-shadowing-in-condition-value.output b/unified/extractor/tests/corpus/swift/control-flow/if-case-let-with-shadowing-in-condition-value.output index 6a53c87d21a..2a0676513f2 100644 --- a/unified/extractor/tests/corpus/swift/control-flow/if-case-let-with-shadowing-in-condition-value.output +++ b/unified/extractor/tests/corpus/swift/control-flow/if-case-let-with-shadowing-in-condition-value.output @@ -4,40 +4,59 @@ if case let x = x + 10 { --- -source_file - statement: - if_statement - body: - block - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: simple_identifier "x" - condition: - if_condition - kind: - if_let_binding - pattern: - pattern - kind: - binding_pattern - binding: - value_binding_pattern - mutability: let +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + expressionStmt + expression: + ifExpr + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + declReferenceExpr + baseName: identifier "x" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + conditions: + conditionElement + condition: + matchingPatternCondition + initializer: + initializerClause + equal: = + value: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "+" + leftOperand: + declReferenceExpr + baseName: identifier "x" + rightOperand: + integerLiteralExpr + literal: integerLiteral "10" pattern: - pattern - bound_identifier: simple_identifier "x" - value: - additive_expression - lhs: simple_identifier "x" - op: + - rhs: integer_literal "10" + valueBindingPattern + pattern: + identifierPattern + identifier: identifier "x" + bindingSpecifier: let + caseKeyword: case + ifKeyword: if --- @@ -53,20 +72,20 @@ top_level identifier: identifier "x" value: binary_expr - operator: infix_operator "+" left: name_expr identifier: identifier "x" + operator: infix_operator "+" right: int_literal "10" then: block stmt: call_expr + callee: + name_expr + identifier: identifier "print" argument: argument value: name_expr identifier: identifier "x" - callee: - name_expr - identifier: identifier "print" diff --git a/unified/extractor/tests/corpus/swift/control-flow/if-else-if-chain.output b/unified/extractor/tests/corpus/swift/control-flow/if-else-if-chain.output index e8f41372646..35f1c2c46d2 100644 --- a/unified/extractor/tests/corpus/swift/control-flow/if-else-if-chain.output +++ b/unified/extractor/tests/corpus/swift/control-flow/if-else-if-chain.output @@ -8,61 +8,103 @@ if x > 0 { --- -source_file - statement: - if_statement - body: - block - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: integer_literal "1" - condition: - if_condition - kind: - comparison_expression - lhs: simple_identifier "x" - op: > - rhs: integer_literal "0" - else_branch: - if_statement - body: - block - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: integer_literal "2" - condition: - if_condition - kind: - comparison_expression - lhs: simple_identifier "x" - op: < - rhs: integer_literal "0" - else_branch: - block - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: integer_literal "3" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + expressionStmt + expression: + ifExpr + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + integerLiteralExpr + literal: integerLiteral "1" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + conditions: + conditionElement + condition: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator ">" + leftOperand: + declReferenceExpr + baseName: identifier "x" + rightOperand: + integerLiteralExpr + literal: integerLiteral "0" + elseKeyword: else + elseBody: + ifExpr + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + integerLiteralExpr + literal: integerLiteral "2" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + conditions: + conditionElement + condition: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "<" + leftOperand: + declReferenceExpr + baseName: identifier "x" + rightOperand: + integerLiteralExpr + literal: integerLiteral "0" + elseKeyword: else + elseBody: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + integerLiteralExpr + literal: integerLiteral "3" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + ifKeyword: if + ifKeyword: if --- @@ -73,47 +115,47 @@ top_level if_expr condition: binary_expr - operator: infix_operator ">" left: name_expr identifier: identifier "x" + operator: infix_operator ">" right: int_literal "0" - else: - if_expr - condition: - binary_expr - operator: infix_operator "<" - left: - name_expr - identifier: identifier "x" - right: int_literal "0" - else: - block - stmt: - call_expr - argument: - argument - value: int_literal "3" - callee: - name_expr - identifier: identifier "print" - then: - block - stmt: - call_expr - argument: - argument - value: int_literal "2" - callee: - name_expr - identifier: identifier "print" then: block stmt: call_expr - argument: - argument - value: int_literal "1" callee: name_expr identifier: identifier "print" + argument: + argument + value: int_literal "1" + else: + if_expr + condition: + binary_expr + left: + name_expr + identifier: identifier "x" + operator: infix_operator "<" + right: int_literal "0" + then: + block + stmt: + call_expr + callee: + name_expr + identifier: identifier "print" + argument: + argument + value: int_literal "2" + else: + block + stmt: + call_expr + callee: + name_expr + identifier: identifier "print" + argument: + argument + value: int_literal "3" diff --git a/unified/extractor/tests/corpus/swift/control-flow/if-else.output b/unified/extractor/tests/corpus/swift/control-flow/if-else.output index 46986121467..744b5c51ab2 100644 --- a/unified/extractor/tests/corpus/swift/control-flow/if-else.output +++ b/unified/extractor/tests/corpus/swift/control-flow/if-else.output @@ -6,43 +6,70 @@ if x > 0 { --- -source_file - statement: - if_statement - body: - block - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: simple_identifier "x" - condition: - if_condition - kind: - comparison_expression - lhs: simple_identifier "x" - op: > - rhs: integer_literal "0" - else_branch: - block - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: - prefix_expression - operation: - - target: simple_identifier "x" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + expressionStmt + expression: + ifExpr + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + declReferenceExpr + baseName: identifier "x" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + conditions: + conditionElement + condition: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator ">" + leftOperand: + declReferenceExpr + baseName: identifier "x" + rightOperand: + integerLiteralExpr + literal: integerLiteral "0" + elseKeyword: else + elseBody: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + prefixOperatorExpr + expression: + declReferenceExpr + baseName: identifier "x" + operator: prefixOperator "-" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + ifKeyword: if --- @@ -53,15 +80,30 @@ top_level if_expr condition: binary_expr - operator: infix_operator ">" left: name_expr identifier: identifier "x" + operator: infix_operator ">" right: int_literal "0" + then: + block + stmt: + call_expr + callee: + name_expr + identifier: identifier "print" + argument: + argument + value: + name_expr + identifier: identifier "x" else: block stmt: call_expr + callee: + name_expr + identifier: identifier "print" argument: argument value: @@ -70,18 +112,3 @@ top_level name_expr identifier: identifier "x" operator: prefix_operator "-" - callee: - name_expr - identifier: identifier "print" - then: - block - stmt: - call_expr - argument: - argument - value: - name_expr - identifier: identifier "x" - callee: - name_expr - identifier: identifier "print" diff --git a/unified/extractor/tests/corpus/swift/control-flow/if-let-optional-binding.output b/unified/extractor/tests/corpus/swift/control-flow/if-let-optional-binding.output index f6b605a7461..d1bcf82b0fa 100644 --- a/unified/extractor/tests/corpus/swift/control-flow/if-let-optional-binding.output +++ b/unified/extractor/tests/corpus/swift/control-flow/if-let-optional-binding.output @@ -4,32 +4,48 @@ if let value = optional { --- -source_file - statement: - if_statement - body: - block - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: simple_identifier "value" - condition: - if_condition - kind: - if_let_binding - pattern: - pattern - binding: - value_binding_pattern - mutability: let - bound_identifier: simple_identifier "value" - value: simple_identifier "optional" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + expressionStmt + expression: + ifExpr + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + declReferenceExpr + baseName: identifier "value" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + conditions: + conditionElement + condition: + optionalBindingCondition + initializer: + initializerClause + equal: = + value: + declReferenceExpr + baseName: identifier "optional" + pattern: + identifierPattern + identifier: identifier "value" + bindingSpecifier: let + ifKeyword: if --- @@ -42,17 +58,17 @@ top_level pattern_guard_expr pattern: constructor_pattern - element: - pattern_element - pattern: - name_pattern - identifier: identifier "value" constructor: member_access_expr base: named_type_expr name: identifier "Optional" member: identifier "some" + element: + pattern_element + pattern: + name_pattern + identifier: identifier "value" value: name_expr identifier: identifier "optional" @@ -60,11 +76,11 @@ top_level block stmt: call_expr + callee: + name_expr + identifier: identifier "print" argument: argument value: name_expr identifier: identifier "value" - callee: - name_expr - identifier: identifier "print" diff --git a/unified/extractor/tests/corpus/swift/control-flow/if-statement.output b/unified/extractor/tests/corpus/swift/control-flow/if-statement.output index 2c29ab1dc69..77bba019ffb 100644 --- a/unified/extractor/tests/corpus/swift/control-flow/if-statement.output +++ b/unified/extractor/tests/corpus/swift/control-flow/if-statement.output @@ -4,28 +4,47 @@ if x > 0 { --- -source_file - statement: - if_statement - body: - block - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: simple_identifier "x" - condition: - if_condition - kind: - comparison_expression - lhs: simple_identifier "x" - op: > - rhs: integer_literal "0" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + expressionStmt + expression: + ifExpr + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + declReferenceExpr + baseName: identifier "x" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + conditions: + conditionElement + condition: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator ">" + leftOperand: + declReferenceExpr + baseName: identifier "x" + rightOperand: + integerLiteralExpr + literal: integerLiteral "0" + ifKeyword: if --- @@ -36,20 +55,20 @@ top_level if_expr condition: binary_expr - operator: infix_operator ">" left: name_expr identifier: identifier "x" + operator: infix_operator ">" right: int_literal "0" then: block stmt: call_expr + callee: + name_expr + identifier: identifier "print" argument: argument value: name_expr identifier: identifier "x" - callee: - name_expr - identifier: identifier "print" diff --git a/unified/extractor/tests/corpus/swift/control-flow/switch-statement.output b/unified/extractor/tests/corpus/swift/control-flow/switch-statement.output index bb90cb60fc5..a8857985294 100644 --- a/unified/extractor/tests/corpus/swift/control-flow/switch-statement.output +++ b/unified/extractor/tests/corpus/swift/control-flow/switch-statement.output @@ -9,65 +9,114 @@ default: --- -source_file - statement: - switch_statement - entry: - switch_entry - pattern: - switch_pattern - pattern: - pattern - kind: integer_literal "1" - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: - line_string_literal - text: line_str_text "one" - switch_entry - pattern: - switch_pattern - pattern: - pattern - kind: integer_literal "2" - switch_pattern - pattern: - pattern - kind: integer_literal "3" - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: - line_string_literal - text: line_str_text "two or three" - switch_entry - default: default_keyword "default" - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: - line_string_literal - text: line_str_text "other" - expr: simple_identifier "x" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + expressionStmt + expression: + switchExpr + leftBrace: { + rightBrace: } + cases: + switchCase + label: + switchCaseLabel + colon: : + caseKeyword: case + caseItems: + switchCaseItem + pattern: + expressionPattern + expression: + integerLiteralExpr + literal: integerLiteral "1" + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + stringLiteralExpr + closingQuote: " + openingQuote: " + segments: + stringSegment + content: stringSegment "one" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + switchCase + label: + switchCaseLabel + colon: : + caseKeyword: case + caseItems: + switchCaseItem + trailingComma: , + pattern: + expressionPattern + expression: + integerLiteralExpr + literal: integerLiteral "2" + switchCaseItem + pattern: + expressionPattern + expression: + integerLiteralExpr + literal: integerLiteral "3" + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + stringLiteralExpr + closingQuote: " + openingQuote: " + segments: + stringSegment + content: stringSegment "two or three" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + switchCase + label: + switchDefaultLabel + colon: : + defaultKeyword: default + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + stringLiteralExpr + closingQuote: " + openingQuote: " + segments: + stringSegment + content: stringSegment "other" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + subject: + declReferenceExpr + baseName: identifier "x" + switchKeyword: switch --- @@ -76,32 +125,25 @@ top_level block stmt: switch_expr + value: + name_expr + identifier: identifier "x" case: switch_case - body: - block - stmt: - call_expr - argument: - argument - value: string_literal "\"one\"" - callee: - name_expr - identifier: identifier "print" pattern: expr_equality_pattern expr: int_literal "1" - switch_case body: block stmt: call_expr - argument: - argument - value: string_literal "\"two or three\"" callee: name_expr identifier: identifier "print" + argument: + argument + value: string_literal "\"one\"" + switch_case pattern: or_pattern pattern: @@ -109,17 +151,24 @@ top_level expr: int_literal "2" expr_equality_pattern expr: int_literal "3" + body: + block + stmt: + call_expr + callee: + name_expr + identifier: identifier "print" + argument: + argument + value: string_literal "\"two or three\"" switch_case body: block stmt: call_expr - argument: - argument - value: string_literal "\"other\"" callee: name_expr identifier: identifier "print" - value: - name_expr - identifier: identifier "x" + argument: + argument + value: string_literal "\"other\"" diff --git a/unified/extractor/tests/corpus/swift/control-flow/switch-with-binding-pattern.output b/unified/extractor/tests/corpus/swift/control-flow/switch-with-binding-pattern.output index 4d98620fe8f..90bb6b44e6e 100644 --- a/unified/extractor/tests/corpus/swift/control-flow/switch-with-binding-pattern.output +++ b/unified/extractor/tests/corpus/swift/control-flow/switch-with-binding-pattern.output @@ -7,77 +7,111 @@ case .square(let s): --- -source_file - statement: - switch_statement - entry: - switch_entry - pattern: - switch_pattern - pattern: - pattern - kind: - case_pattern - arguments: - tuple_pattern - item: - tuple_pattern_item - pattern: - pattern - kind: - binding_pattern - binding: - value_binding_pattern - mutability: let - pattern: - pattern - bound_identifier: simple_identifier "r" - dot: . - name: simple_identifier "circle" - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: simple_identifier "r" - switch_entry - pattern: - switch_pattern - pattern: - pattern - kind: - case_pattern - arguments: - tuple_pattern - item: - tuple_pattern_item - pattern: - pattern - kind: - binding_pattern - binding: - value_binding_pattern - mutability: let - pattern: - pattern - bound_identifier: simple_identifier "s" - dot: . - name: simple_identifier "square" - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: simple_identifier "s" - expr: simple_identifier "shape" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + expressionStmt + expression: + switchExpr + leftBrace: { + rightBrace: } + cases: + switchCase + label: + switchCaseLabel + colon: : + caseKeyword: case + caseItems: + switchCaseItem + pattern: + expressionPattern + expression: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + patternExpr + pattern: + valueBindingPattern + pattern: + identifierPattern + identifier: identifier "r" + bindingSpecifier: let + additionalTrailingClosures: + calledExpression: + memberAccessExpr + period: . + declName: + declReferenceExpr + baseName: identifier "circle" + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + declReferenceExpr + baseName: identifier "r" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + switchCase + label: + switchCaseLabel + colon: : + caseKeyword: case + caseItems: + switchCaseItem + pattern: + expressionPattern + expression: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + patternExpr + pattern: + valueBindingPattern + pattern: + identifierPattern + identifier: identifier "s" + bindingSpecifier: let + additionalTrailingClosures: + calledExpression: + memberAccessExpr + period: . + declName: + declReferenceExpr + baseName: identifier "square" + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + declReferenceExpr + baseName: identifier "s" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + subject: + declReferenceExpr + baseName: identifier "shape" + switchKeyword: switch --- @@ -86,55 +120,55 @@ top_level block stmt: switch_expr + value: + name_expr + identifier: identifier "shape" case: switch_case - body: - block - stmt: - call_expr - argument: - argument - value: - name_expr - identifier: identifier "r" - callee: - name_expr - identifier: identifier "print" pattern: constructor_pattern + constructor: + member_access_expr + base: inferred_type_expr "." + member: identifier "circle" element: pattern_element pattern: name_pattern identifier: identifier "r" - constructor: - member_access_expr - base: inferred_type_expr "." - member: identifier "circle" - switch_case body: block stmt: call_expr + callee: + name_expr + identifier: identifier "print" argument: argument value: name_expr - identifier: identifier "s" - callee: - name_expr - identifier: identifier "print" + identifier: identifier "r" + switch_case pattern: constructor_pattern + constructor: + member_access_expr + base: inferred_type_expr "." + member: identifier "square" element: pattern_element pattern: name_pattern identifier: identifier "s" - constructor: - member_access_expr - base: inferred_type_expr "." - member: identifier "square" - value: - name_expr - identifier: identifier "shape" + body: + block + stmt: + call_expr + callee: + name_expr + identifier: identifier "print" + argument: + argument + value: + name_expr + identifier: identifier "s" diff --git a/unified/extractor/tests/corpus/swift/control-flow/switch-with-labeled-case-pattern-arguments.output b/unified/extractor/tests/corpus/swift/control-flow/switch-with-labeled-case-pattern-arguments.output index aef36c16855..20b17d160e4 100644 --- a/unified/extractor/tests/corpus/swift/control-flow/switch-with-labeled-case-pattern-arguments.output +++ b/unified/extractor/tests/corpus/swift/control-flow/switch-with-labeled-case-pattern-arguments.output @@ -7,79 +7,119 @@ case .thread(threadRowId: _, let rowId): --- -source_file - statement: - switch_statement - entry: - switch_entry - pattern: - switch_pattern - pattern: - pattern - kind: - case_pattern - arguments: - tuple_pattern - item: - tuple_pattern_item - name: simple_identifier "isAcknowledged" - pattern: - pattern - kind: - boolean_literal - dot: . - name: simple_identifier "implicit" - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: - line_string_literal - text: line_str_text "yes" - switch_entry - pattern: - switch_pattern - pattern: - pattern - kind: - case_pattern - arguments: - tuple_pattern - item: - tuple_pattern_item - name: simple_identifier "threadRowId" - pattern: - pattern - kind: wildcard_pattern "_" - tuple_pattern_item - pattern: - pattern - kind: - binding_pattern - binding: - value_binding_pattern - mutability: let - pattern: - pattern - bound_identifier: simple_identifier "rowId" - dot: . - name: simple_identifier "thread" - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: simple_identifier "rowId" - expr: simple_identifier "x" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + expressionStmt + expression: + switchExpr + leftBrace: { + rightBrace: } + cases: + switchCase + label: + switchCaseLabel + colon: : + caseKeyword: case + caseItems: + switchCaseItem + pattern: + expressionPattern + expression: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + colon: : + label: identifier "isAcknowledged" + expression: + booleanLiteralExpr + literal: false + additionalTrailingClosures: + calledExpression: + memberAccessExpr + period: . + declName: + declReferenceExpr + baseName: identifier "implicit" + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + stringLiteralExpr + closingQuote: " + openingQuote: " + segments: + stringSegment + content: stringSegment "yes" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + switchCase + label: + switchCaseLabel + colon: : + caseKeyword: case + caseItems: + switchCaseItem + pattern: + expressionPattern + expression: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + colon: : + label: identifier "threadRowId" + expression: + discardAssignmentExpr + wildcard: _ + trailingComma: , + labeledExpr + expression: + patternExpr + pattern: + valueBindingPattern + pattern: + identifierPattern + identifier: identifier "rowId" + bindingSpecifier: let + additionalTrailingClosures: + calledExpression: + memberAccessExpr + period: . + declName: + declReferenceExpr + baseName: identifier "thread" + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + declReferenceExpr + baseName: identifier "rowId" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + subject: + declReferenceExpr + baseName: identifier "x" + switchKeyword: switch --- @@ -88,45 +128,40 @@ top_level block stmt: switch_expr + value: + name_expr + identifier: identifier "x" case: switch_case - body: - block - stmt: - call_expr - argument: - argument - value: string_literal "\"yes\"" - callee: - name_expr - identifier: identifier "print" pattern: constructor_pattern + constructor: + member_access_expr + base: inferred_type_expr "." + member: identifier "implicit" element: pattern_element key: identifier "isAcknowledged" pattern: expr_equality_pattern expr: boolean_literal "false" - constructor: - member_access_expr - base: inferred_type_expr "." - member: identifier "implicit" - switch_case body: block stmt: call_expr - argument: - argument - value: - name_expr - identifier: identifier "rowId" callee: name_expr identifier: identifier "print" + argument: + argument + value: string_literal "\"yes\"" + switch_case pattern: constructor_pattern + constructor: + member_access_expr + base: inferred_type_expr "." + member: identifier "thread" element: pattern_element key: identifier "threadRowId" @@ -135,10 +170,15 @@ top_level pattern: name_pattern identifier: identifier "rowId" - constructor: - member_access_expr - base: inferred_type_expr "." - member: identifier "thread" - value: - name_expr - identifier: identifier "x" + body: + block + stmt: + call_expr + callee: + name_expr + identifier: identifier "print" + argument: + argument + value: + name_expr + identifier: identifier "rowId" diff --git a/unified/extractor/tests/corpus/swift/control-flow/ternary-expression.output b/unified/extractor/tests/corpus/swift/control-flow/ternary-expression.output index 7f9da80d34a..88c7bc1b886 100644 --- a/unified/extractor/tests/corpus/swift/control-flow/ternary-expression.output +++ b/unified/extractor/tests/corpus/swift/control-flow/ternary-expression.output @@ -2,29 +2,47 @@ let y = x > 0 ? 1 : -1 --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "y" - value: - ternary_expression - condition: - comparison_expression - lhs: simple_identifier "x" - op: > - rhs: integer_literal "0" - if_false: - prefix_expression - operation: - - target: integer_literal "1" - if_true: integer_literal "1" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + ternaryExpr + colon: : + condition: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator ">" + leftOperand: + declReferenceExpr + baseName: identifier "x" + rightOperand: + integerLiteralExpr + literal: integerLiteral "0" + questionMark: ? + elseExpression: + prefixOperatorExpr + expression: + integerLiteralExpr + literal: integerLiteral "1" + operator: prefixOperator "-" + thenExpression: + integerLiteralExpr + literal: integerLiteral "1" + pattern: + identifierPattern + identifier: identifier "y" --- @@ -41,13 +59,13 @@ top_level if_expr condition: binary_expr - operator: infix_operator ">" left: name_expr identifier: identifier "x" + operator: infix_operator ">" right: int_literal "0" + then: int_literal "1" else: unary_expr operand: int_literal "1" operator: prefix_operator "-" - then: int_literal "1" diff --git a/unified/extractor/tests/corpus/swift/desugar/additive-expression-is-desugared.output b/unified/extractor/tests/corpus/swift/desugar/additive-expression-is-desugared.output index 07aa40618bd..30d9570f3be 100644 --- a/unified/extractor/tests/corpus/swift/desugar/additive-expression-is-desugared.output +++ b/unified/extractor/tests/corpus/swift/desugar/additive-expression-is-desugared.output @@ -2,12 +2,21 @@ --- -source_file - statement: - additive_expression - lhs: integer_literal "1" - op: + - rhs: integer_literal "2" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "+" + leftOperand: + integerLiteralExpr + literal: integerLiteral "1" + rightOperand: + integerLiteralExpr + literal: integerLiteral "2" --- @@ -16,6 +25,6 @@ top_level block stmt: binary_expr - operator: infix_operator "+" left: int_literal "1" + operator: infix_operator "+" right: int_literal "2" diff --git a/unified/extractor/tests/corpus/swift/desugar/another-additive-expression-is-desugared.output b/unified/extractor/tests/corpus/swift/desugar/another-additive-expression-is-desugared.output index ff830fd4b89..982309ffa5a 100644 --- a/unified/extractor/tests/corpus/swift/desugar/another-additive-expression-is-desugared.output +++ b/unified/extractor/tests/corpus/swift/desugar/another-additive-expression-is-desugared.output @@ -2,12 +2,21 @@ foo + bar --- -source_file - statement: - additive_expression - lhs: simple_identifier "foo" - op: + - rhs: simple_identifier "bar" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "+" + leftOperand: + declReferenceExpr + baseName: identifier "foo" + rightOperand: + declReferenceExpr + baseName: identifier "bar" --- @@ -16,10 +25,10 @@ top_level block stmt: binary_expr - operator: infix_operator "+" left: name_expr identifier: identifier "foo" + operator: infix_operator "+" right: name_expr identifier: identifier "bar" diff --git a/unified/extractor/tests/corpus/swift/desugar/import-with-deeply-nested-path-three-parts.output b/unified/extractor/tests/corpus/swift/desugar/import-with-deeply-nested-path-three-parts.output index 4f312dabb15..cacdc64a46d 100644 --- a/unified/extractor/tests/corpus/swift/desugar/import-with-deeply-nested-path-three-parts.output +++ b/unified/extractor/tests/corpus/swift/desugar/import-with-deeply-nested-path-three-parts.output @@ -2,15 +2,24 @@ import Foundation.Networking.URLSession --- -source_file - statement: - import_declaration - name: - identifier - part: - simple_identifier "Foundation" - simple_identifier "Networking" - simple_identifier "URLSession" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + importDecl + attributes: + modifiers: + importKeyword: import + path: + importPathComponent + name: identifier "Foundation" + trailingPeriod: . + importPathComponent + name: identifier "Networking" + trailingPeriod: . + importPathComponent + name: identifier "URLSession" --- @@ -19,7 +28,6 @@ top_level block stmt: import_declaration - pattern: bulk_importing_pattern "import Foundation.Networking.URLSession" imported_expr: member_access_expr base: @@ -29,3 +37,4 @@ top_level identifier: identifier "Foundation" member: identifier "Networking" member: identifier "URLSession" + pattern: bulk_importing_pattern "import Foundation.Networking.URLSession" diff --git a/unified/extractor/tests/corpus/swift/desugar/import-with-dotted-path-two-parts.output b/unified/extractor/tests/corpus/swift/desugar/import-with-dotted-path-two-parts.output index efd2a646124..4fc053a7bc5 100644 --- a/unified/extractor/tests/corpus/swift/desugar/import-with-dotted-path-two-parts.output +++ b/unified/extractor/tests/corpus/swift/desugar/import-with-dotted-path-two-parts.output @@ -2,14 +2,21 @@ import Foundation.Networking --- -source_file - statement: - import_declaration - name: - identifier - part: - simple_identifier "Foundation" - simple_identifier "Networking" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + importDecl + attributes: + modifiers: + importKeyword: import + path: + importPathComponent + name: identifier "Foundation" + trailingPeriod: . + importPathComponent + name: identifier "Networking" --- @@ -18,10 +25,10 @@ top_level block stmt: import_declaration - pattern: bulk_importing_pattern "import Foundation.Networking" imported_expr: member_access_expr base: name_expr identifier: identifier "Foundation" member: identifier "Networking" + pattern: bulk_importing_pattern "import Foundation.Networking" diff --git a/unified/extractor/tests/corpus/swift/desugar/scoped-import-uses-name-pattern.output b/unified/extractor/tests/corpus/swift/desugar/scoped-import-uses-name-pattern.output index fbf8e8100af..93319c52298 100644 --- a/unified/extractor/tests/corpus/swift/desugar/scoped-import-uses-name-pattern.output +++ b/unified/extractor/tests/corpus/swift/desugar/scoped-import-uses-name-pattern.output @@ -2,15 +2,22 @@ import struct Foundation.Date --- -source_file - statement: - import_declaration - name: - identifier - part: - simple_identifier "Foundation" - simple_identifier "Date" - scoped_import_kind: struct +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + importDecl + attributes: + modifiers: + importKeyword: import + importKindSpecifier: struct + path: + importPathComponent + name: identifier "Foundation" + trailingPeriod: . + importPathComponent + name: identifier "Date" --- @@ -20,12 +27,12 @@ top_level stmt: import_declaration modifier: modifier "struct" - pattern: - name_pattern - identifier: identifier "Date" imported_expr: member_access_expr base: name_expr identifier: identifier "Foundation" member: identifier "Date" + pattern: + name_pattern + identifier: identifier "Date" diff --git a/unified/extractor/tests/corpus/swift/desugar/simple-import-with-single-name.output b/unified/extractor/tests/corpus/swift/desugar/simple-import-with-single-name.output index 7a6be1c35e4..583f33563d1 100644 --- a/unified/extractor/tests/corpus/swift/desugar/simple-import-with-single-name.output +++ b/unified/extractor/tests/corpus/swift/desugar/simple-import-with-single-name.output @@ -2,12 +2,18 @@ import Foundation --- -source_file - statement: - import_declaration - name: - identifier - part: simple_identifier "Foundation" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + importDecl + attributes: + modifiers: + importKeyword: import + path: + importPathComponent + name: identifier "Foundation" --- @@ -16,7 +22,7 @@ top_level block stmt: import_declaration - pattern: bulk_importing_pattern "import Foundation" imported_expr: name_expr identifier: identifier "Foundation" + pattern: bulk_importing_pattern "import Foundation" diff --git a/unified/extractor/tests/corpus/swift/functions/function-call-with-labelled-arguments.output b/unified/extractor/tests/corpus/swift/functions/function-call-with-labelled-arguments.output index ba0c002a452..f885d5762f2 100644 --- a/unified/extractor/tests/corpus/swift/functions/function-call-with-labelled-arguments.output +++ b/unified/extractor/tests/corpus/swift/functions/function-call-with-labelled-arguments.output @@ -2,22 +2,29 @@ greet(person: "Bob") --- -source_file - statement: - call_expression - function: simple_identifier "greet" - suffix: - call_suffix +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) arguments: - value_arguments - argument: - value_argument - name: - value_argument_label - name: simple_identifier "person" - value: - line_string_literal - text: line_str_text "Bob" + labeledExpr + colon: : + label: identifier "person" + expression: + stringLiteralExpr + closingQuote: " + openingQuote: " + segments: + stringSegment + content: stringSegment "Bob" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "greet" --- @@ -26,10 +33,10 @@ top_level block stmt: call_expr + callee: + name_expr + identifier: identifier "greet" argument: argument name: identifier "person" value: string_literal "\"Bob\"" - callee: - name_expr - identifier: identifier "greet" diff --git a/unified/extractor/tests/corpus/swift/functions/function-call.output b/unified/extractor/tests/corpus/swift/functions/function-call.output index ed604730d33..a1e2859ccd6 100644 --- a/unified/extractor/tests/corpus/swift/functions/function-call.output +++ b/unified/extractor/tests/corpus/swift/functions/function-call.output @@ -2,19 +2,28 @@ foo(1, 2) --- -source_file - statement: - call_expression - function: simple_identifier "foo" - suffix: - call_suffix +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) arguments: - value_arguments - argument: - value_argument - value: integer_literal "1" - value_argument - value: integer_literal "2" + labeledExpr + expression: + integerLiteralExpr + literal: integerLiteral "1" + trailingComma: , + labeledExpr + expression: + integerLiteralExpr + literal: integerLiteral "2" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "foo" --- @@ -23,11 +32,11 @@ top_level block stmt: call_expr + callee: + name_expr + identifier: identifier "foo" argument: argument value: int_literal "1" argument value: int_literal "2" - callee: - name_expr - identifier: identifier "foo" diff --git a/unified/extractor/tests/corpus/swift/functions/function-with-default-parameter-value.output b/unified/extractor/tests/corpus/swift/functions/function-with-default-parameter-value.output index fdd737e1258..01c0ddef856 100644 --- a/unified/extractor/tests/corpus/swift/functions/function-with-default-parameter-value.output +++ b/unified/extractor/tests/corpus/swift/functions/function-with-default-parameter-value.output @@ -4,37 +4,60 @@ func greet(name: String = "world") { --- -source_file - statement: - function_declaration - body: - block - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: simple_identifier "name" - name: simple_identifier "greet" - parameter: - function_parameter - default_value: - line_string_literal - text: line_str_text "world" - parameter: - parameter - name: simple_identifier "name" - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "String" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + functionDecl + attributes: + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + declReferenceExpr + baseName: identifier "name" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + name: identifier "greet" + modifiers: + signature: + functionSignature + parameterClause: + functionParameterClause + leftParen: ( + rightParen: ) + parameters: + functionParameter + colon: : + attributes: + modifiers: + type: + identifierType + name: identifier "String" + firstName: identifier "name" + defaultValue: + initializerClause + equal: = + value: + stringLiteralExpr + closingQuote: " + openingQuote: " + segments: + stringSegment + content: stringSegment "world" + funcKeyword: func --- @@ -43,22 +66,22 @@ top_level block stmt: function_declaration + name: identifier "greet" + parameter: + parameter + pattern: + name_pattern + identifier: identifier "name" + default: string_literal "\"world\"" body: block stmt: call_expr + callee: + name_expr + identifier: identifier "print" argument: argument value: name_expr identifier: identifier "name" - callee: - name_expr - identifier: identifier "print" - name: identifier "greet" - parameter: - parameter - default: string_literal "\"world\"" - pattern: - name_pattern - identifier: identifier "name" diff --git a/unified/extractor/tests/corpus/swift/functions/function-with-named-parameters.output b/unified/extractor/tests/corpus/swift/functions/function-with-named-parameters.output index bfa68c645ea..f2024b47362 100644 --- a/unified/extractor/tests/corpus/swift/functions/function-with-named-parameters.output +++ b/unified/extractor/tests/corpus/swift/functions/function-with-named-parameters.output @@ -4,35 +4,51 @@ func greet(person name: String) { --- -source_file - statement: - function_declaration - body: - block - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: simple_identifier "name" - name: simple_identifier "greet" - parameter: - function_parameter - parameter: - parameter - external_name: simple_identifier "person" - name: simple_identifier "name" - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "String" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + functionDecl + attributes: + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + declReferenceExpr + baseName: identifier "name" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + name: identifier "greet" + modifiers: + signature: + functionSignature + parameterClause: + functionParameterClause + leftParen: ( + rightParen: ) + parameters: + functionParameter + colon: : + attributes: + modifiers: + type: + identifierType + name: identifier "String" + firstName: identifier "person" + secondName: identifier "name" + funcKeyword: func --- @@ -41,18 +57,6 @@ top_level block stmt: function_declaration - body: - block - stmt: - call_expr - argument: - argument - value: - name_expr - identifier: identifier "name" - callee: - name_expr - identifier: identifier "print" name: identifier "greet" parameter: parameter @@ -60,3 +64,15 @@ top_level pattern: name_pattern identifier: identifier "name" + body: + block + stmt: + call_expr + callee: + name_expr + identifier: identifier "print" + argument: + argument + value: + name_expr + identifier: identifier "name" diff --git a/unified/extractor/tests/corpus/swift/functions/function-with-no-parameters.output b/unified/extractor/tests/corpus/swift/functions/function-with-no-parameters.output index b5cdfd73d48..4ddc26ae94d 100644 --- a/unified/extractor/tests/corpus/swift/functions/function-with-no-parameters.output +++ b/unified/extractor/tests/corpus/swift/functions/function-with-no-parameters.output @@ -4,24 +4,46 @@ func greet() { --- -source_file - statement: - function_declaration - body: - block - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: - line_string_literal - text: line_str_text "hello" - name: simple_identifier "greet" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + functionDecl + attributes: + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + stringLiteralExpr + closingQuote: " + openingQuote: " + segments: + stringSegment + content: stringSegment "hello" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + name: identifier "greet" + modifiers: + signature: + functionSignature + parameterClause: + functionParameterClause + leftParen: ( + rightParen: ) + parameters: + funcKeyword: func --- @@ -30,14 +52,14 @@ top_level block stmt: function_declaration + name: identifier "greet" body: block stmt: call_expr - argument: - argument - value: string_literal "\"hello\"" callee: name_expr identifier: identifier "print" - name: identifier "greet" + argument: + argument + value: string_literal "\"hello\"" diff --git a/unified/extractor/tests/corpus/swift/functions/function-with-parameters-and-return-type.output b/unified/extractor/tests/corpus/swift/functions/function-with-parameters-and-return-type.output index 6544f4313cd..1ffb391dbc5 100644 --- a/unified/extractor/tests/corpus/swift/functions/function-with-parameters-and-return-type.output +++ b/unified/extractor/tests/corpus/swift/functions/function-with-parameters-and-return-type.output @@ -4,52 +4,68 @@ func add(_ a: Int, _ b: Int) -> Int { --- -source_file - statement: - function_declaration - body: - block - statement: - control_transfer_statement - kind: return - result: - additive_expression - lhs: simple_identifier "a" - op: + - rhs: simple_identifier "b" - name: simple_identifier "add" - parameter: - function_parameter - parameter: - parameter - external_name: simple_identifier "_" - name: simple_identifier "a" - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" - function_parameter - parameter: - parameter - external_name: simple_identifier "_" - name: simple_identifier "b" - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" - return_type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + functionDecl + attributes: + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + returnStmt + expression: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "+" + leftOperand: + declReferenceExpr + baseName: identifier "a" + rightOperand: + declReferenceExpr + baseName: identifier "b" + returnKeyword: return + name: identifier "add" + modifiers: + signature: + functionSignature + parameterClause: + functionParameterClause + leftParen: ( + rightParen: ) + parameters: + functionParameter + colon: : + attributes: + modifiers: + trailingComma: , + type: + identifierType + name: identifier "Int" + firstName: _ + secondName: identifier "a" + functionParameter + colon: : + attributes: + modifiers: + type: + identifierType + name: identifier "Int" + firstName: _ + secondName: identifier "b" + returnClause: + returnClause + arrow: -> + type: + identifierType + name: identifier "Int" + funcKeyword: func --- @@ -58,19 +74,6 @@ top_level block stmt: function_declaration - body: - block - stmt: - return_expr - value: - binary_expr - operator: infix_operator "+" - left: - name_expr - identifier: identifier "a" - right: - name_expr - identifier: identifier "b" name: identifier "add" parameter: parameter @@ -86,3 +89,16 @@ top_level return_type: named_type_expr name: identifier "Int" + body: + block + stmt: + return_expr + value: + binary_expr + left: + name_expr + identifier: identifier "a" + operator: infix_operator "+" + right: + name_expr + identifier: identifier "b" diff --git a/unified/extractor/tests/corpus/swift/functions/generic-function.output b/unified/extractor/tests/corpus/swift/functions/generic-function.output index f42367a8fca..1652f7f4741 100644 --- a/unified/extractor/tests/corpus/swift/functions/generic-function.output +++ b/unified/extractor/tests/corpus/swift/functions/generic-function.output @@ -4,41 +4,58 @@ func identity(_ x: T) -> T { --- -source_file - statement: - function_declaration - body: - block - statement: - control_transfer_statement - kind: return - result: simple_identifier "x" - name: simple_identifier "identity" - parameter: - function_parameter - parameter: - parameter - external_name: simple_identifier "_" - name: simple_identifier "x" - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "T" - return_type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "T" - type_parameters: - type_parameters - parameter: - type_parameter - name: type_identifier "T" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + functionDecl + attributes: + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + returnStmt + expression: + declReferenceExpr + baseName: identifier "x" + returnKeyword: return + name: identifier "identity" + genericParameterClause: + genericParameterClause + parameters: + genericParameter + attributes: + name: identifier "T" + leftAngle: < + rightAngle: > + modifiers: + signature: + functionSignature + parameterClause: + functionParameterClause + leftParen: ( + rightParen: ) + parameters: + functionParameter + colon: : + attributes: + modifiers: + type: + identifierType + name: identifier "T" + firstName: _ + secondName: identifier "x" + returnClause: + returnClause + arrow: -> + type: + identifierType + name: identifier "T" + funcKeyword: func --- @@ -47,13 +64,6 @@ top_level block stmt: function_declaration - body: - block - stmt: - return_expr - value: - name_expr - identifier: identifier "x" name: identifier "identity" parameter: parameter @@ -64,3 +74,10 @@ top_level return_type: named_type_expr name: identifier "T" + body: + block + stmt: + return_expr + value: + name_expr + identifier: identifier "x" diff --git a/unified/extractor/tests/corpus/swift/functions/leading-dot-expression-call.output b/unified/extractor/tests/corpus/swift/functions/leading-dot-expression-call.output index 8db16da8ab8..d0a844d02af 100644 --- a/unified/extractor/tests/corpus/swift/functions/leading-dot-expression-call.output +++ b/unified/extractor/tests/corpus/swift/functions/leading-dot-expression-call.output @@ -2,30 +2,39 @@ let y = .some(1) --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "y" - value: - call_expression - function: - prefix_expression - operation: . - target: simple_identifier "some" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: integer_literal "1" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + integerLiteralExpr + literal: integerLiteral "1" + additionalTrailingClosures: + calledExpression: + memberAccessExpr + period: . + declName: + declReferenceExpr + baseName: identifier "some" + pattern: + identifierPattern + identifier: identifier "y" --- @@ -40,10 +49,10 @@ top_level identifier: identifier "y" value: call_expr - argument: - argument - value: int_literal "1" callee: member_access_expr base: inferred_type_expr ".some" member: identifier "some" + argument: + argument + value: int_literal "1" diff --git a/unified/extractor/tests/corpus/swift/functions/leading-dot-expression-value.output b/unified/extractor/tests/corpus/swift/functions/leading-dot-expression-value.output index 85aeeffde6e..bec014593d9 100644 --- a/unified/extractor/tests/corpus/swift/functions/leading-dot-expression-value.output +++ b/unified/extractor/tests/corpus/swift/functions/leading-dot-expression-value.output @@ -2,21 +2,29 @@ let x = .foo --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "x" - value: - prefix_expression - operation: . - target: simple_identifier "foo" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + memberAccessExpr + period: . + declName: + declReferenceExpr + baseName: identifier "foo" + pattern: + identifierPattern + identifier: identifier "x" --- diff --git a/unified/extractor/tests/corpus/swift/functions/method-call.output b/unified/extractor/tests/corpus/swift/functions/method-call.output index 5a8a23f5658..3c0b150b823 100644 --- a/unified/extractor/tests/corpus/swift/functions/method-call.output +++ b/unified/extractor/tests/corpus/swift/functions/method-call.output @@ -2,22 +2,29 @@ list.append(1) --- -source_file - statement: - call_expression - function: - navigation_expression - suffix: - navigation_suffix - suffix: simple_identifier "append" - target: simple_identifier "list" - suffix: - call_suffix +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) arguments: - value_arguments - argument: - value_argument - value: integer_literal "1" + labeledExpr + expression: + integerLiteralExpr + literal: integerLiteral "1" + additionalTrailingClosures: + calledExpression: + memberAccessExpr + period: . + declName: + declReferenceExpr + baseName: identifier "append" + base: + declReferenceExpr + baseName: identifier "list" --- @@ -26,12 +33,12 @@ top_level block stmt: call_expr - argument: - argument - value: int_literal "1" callee: member_access_expr base: name_expr identifier: identifier "list" member: identifier "append" + argument: + argument + value: int_literal "1" diff --git a/unified/extractor/tests/corpus/swift/functions/nested-function-type.output b/unified/extractor/tests/corpus/swift/functions/nested-function-type.output new file mode 100644 index 00000000000..d9ce0cd7a97 --- /dev/null +++ b/unified/extractor/tests/corpus/swift/functions/nested-function-type.output @@ -0,0 +1,156 @@ +typealias NestedFunction = ((Int) -> Bool) -> Bool + +typealias MixedParametersAndTuples = ((Int) -> Bool, String) -> (Bool, Int) + +--- + +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + typeAliasDecl + attributes: + name: identifier "NestedFunction" + modifiers: + initializer: + typeInitializerClause + equal: = + value: + functionType + leftParen: ( + rightParen: ) + parameters: + tupleTypeElement + type: + functionType + leftParen: ( + rightParen: ) + parameters: + tupleTypeElement + type: + identifierType + name: identifier "Int" + returnClause: + returnClause + arrow: -> + type: + identifierType + name: identifier "Bool" + returnClause: + returnClause + arrow: -> + type: + identifierType + name: identifier "Bool" + typealiasKeyword: typealias + codeBlockItem + item: + typeAliasDecl + attributes: + name: identifier "MixedParametersAndTuples" + modifiers: + initializer: + typeInitializerClause + equal: = + value: + functionType + leftParen: ( + rightParen: ) + parameters: + tupleTypeElement + trailingComma: , + type: + functionType + leftParen: ( + rightParen: ) + parameters: + tupleTypeElement + type: + identifierType + name: identifier "Int" + returnClause: + returnClause + arrow: -> + type: + identifierType + name: identifier "Bool" + tupleTypeElement + type: + identifierType + name: identifier "String" + returnClause: + returnClause + arrow: -> + type: + tupleType + leftParen: ( + rightParen: ) + elements: + tupleTypeElement + trailingComma: , + type: + identifierType + name: identifier "Bool" + tupleTypeElement + type: + identifierType + name: identifier "Int" + typealiasKeyword: typealias + +--- + +top_level + body: + block + stmt: + type_alias_declaration + name: identifier "NestedFunction" + type: + function_type_expr + parameter: + parameter + type: + function_type_expr + parameter: + parameter + type: + named_type_expr + name: identifier "Int" + return_type: + named_type_expr + name: identifier "Bool" + return_type: + named_type_expr + name: identifier "Bool" + type_alias_declaration + name: identifier "MixedParametersAndTuples" + type: + function_type_expr + parameter: + parameter + type: + function_type_expr + parameter: + parameter + type: + named_type_expr + name: identifier "Int" + return_type: + named_type_expr + name: identifier "Bool" + parameter + type: + named_type_expr + name: identifier "String" + return_type: + tuple_type_expr + element: + tuple_type_element + type: + named_type_expr + name: identifier "Bool" + tuple_type_element + type: + named_type_expr + name: identifier "Int" diff --git a/unified/extractor/tests/corpus/swift/functions/nested-function-type.swift b/unified/extractor/tests/corpus/swift/functions/nested-function-type.swift new file mode 100644 index 00000000000..b19128965f3 --- /dev/null +++ b/unified/extractor/tests/corpus/swift/functions/nested-function-type.swift @@ -0,0 +1,3 @@ +typealias NestedFunction = ((Int) -> Bool) -> Bool + +typealias MixedParametersAndTuples = ((Int) -> Bool, String) -> (Bool, Int) diff --git a/unified/extractor/tests/corpus/swift/functions/variadic-function.output b/unified/extractor/tests/corpus/swift/functions/variadic-function.output index 7ca1dfbad4e..571f07c35e0 100644 --- a/unified/extractor/tests/corpus/swift/functions/variadic-function.output +++ b/unified/extractor/tests/corpus/swift/functions/variadic-function.output @@ -4,54 +4,72 @@ func sum(_ values: Int...) -> Int { --- -source_file - statement: - function_declaration - body: - block - statement: - control_transfer_statement - kind: return - result: - call_expression - function: - navigation_expression - suffix: - navigation_suffix - suffix: simple_identifier "reduce" - target: simple_identifier "values" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: integer_literal "0" - value_argument - value: - referenceable_operator - operator: + - name: simple_identifier "sum" - parameter: - function_parameter - parameter: - parameter - external_name: simple_identifier "_" - name: simple_identifier "values" - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" - return_type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + functionDecl + attributes: + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + returnStmt + expression: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + integerLiteralExpr + literal: integerLiteral "0" + trailingComma: , + labeledExpr + expression: + declReferenceExpr + baseName: binaryOperator "+" + additionalTrailingClosures: + calledExpression: + memberAccessExpr + period: . + declName: + declReferenceExpr + baseName: identifier "reduce" + base: + declReferenceExpr + baseName: identifier "values" + returnKeyword: return + name: identifier "sum" + modifiers: + signature: + functionSignature + parameterClause: + functionParameterClause + leftParen: ( + rightParen: ) + parameters: + functionParameter + colon: : + attributes: + modifiers: + type: + identifierType + name: identifier "Int" + ellipsis: ... + firstName: _ + secondName: identifier "values" + returnClause: + returnClause + arrow: -> + type: + identifierType + name: identifier "Int" + funcKeyword: func --- @@ -60,25 +78,6 @@ top_level block stmt: function_declaration - body: - block - stmt: - return_expr - value: - call_expr - argument: - argument - value: int_literal "0" - argument - value: - name_expr - identifier: identifier "+" - callee: - member_access_expr - base: - name_expr - identifier: identifier "values" - member: identifier "reduce" name: identifier "sum" parameter: parameter @@ -89,3 +88,22 @@ top_level return_type: named_type_expr name: identifier "Int" + body: + block + stmt: + return_expr + value: + call_expr + callee: + member_access_expr + base: + name_expr + identifier: identifier "values" + member: identifier "reduce" + argument: + argument + value: int_literal "0" + argument + value: + name_expr + identifier: identifier "+" diff --git a/unified/extractor/tests/corpus/swift/literals/boolean-literals.output b/unified/extractor/tests/corpus/swift/literals/boolean-literals.output index d31893de052..34b394712a3 100644 --- a/unified/extractor/tests/corpus/swift/literals/boolean-literals.output +++ b/unified/extractor/tests/corpus/swift/literals/boolean-literals.output @@ -3,10 +3,17 @@ false --- -source_file - statement: - boolean_literal - boolean_literal +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + booleanLiteralExpr + literal: true + codeBlockItem + item: + booleanLiteralExpr + literal: false --- diff --git a/unified/extractor/tests/corpus/swift/literals/floating-point-literal.output b/unified/extractor/tests/corpus/swift/literals/floating-point-literal.output index 0c374dc4452..19fa40aac77 100644 --- a/unified/extractor/tests/corpus/swift/literals/floating-point-literal.output +++ b/unified/extractor/tests/corpus/swift/literals/floating-point-literal.output @@ -2,8 +2,13 @@ --- -source_file - statement: real_literal "3.14" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + floatLiteralExpr + literal: floatLiteral "3.14" --- diff --git a/unified/extractor/tests/corpus/swift/literals/integer-literal.output b/unified/extractor/tests/corpus/swift/literals/integer-literal.output index 018c5798394..9df79925753 100644 --- a/unified/extractor/tests/corpus/swift/literals/integer-literal.output +++ b/unified/extractor/tests/corpus/swift/literals/integer-literal.output @@ -2,8 +2,13 @@ --- -source_file - statement: integer_literal "42" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + integerLiteralExpr + literal: integerLiteral "42" --- diff --git a/unified/extractor/tests/corpus/swift/literals/negative-integer-literal.output b/unified/extractor/tests/corpus/swift/literals/negative-integer-literal.output index e1ca11e070a..90794405478 100644 --- a/unified/extractor/tests/corpus/swift/literals/negative-integer-literal.output +++ b/unified/extractor/tests/corpus/swift/literals/negative-integer-literal.output @@ -2,11 +2,16 @@ --- -source_file - statement: - prefix_expression - operation: - - target: integer_literal "7" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + prefixOperatorExpr + expression: + integerLiteralExpr + literal: integerLiteral "7" + operator: prefixOperator "-" --- diff --git a/unified/extractor/tests/corpus/swift/literals/nil-literal.output b/unified/extractor/tests/corpus/swift/literals/nil-literal.output index 6c826cabe7d..c7da232131f 100644 --- a/unified/extractor/tests/corpus/swift/literals/nil-literal.output +++ b/unified/extractor/tests/corpus/swift/literals/nil-literal.output @@ -2,8 +2,13 @@ nil --- -source_file - statement: nil +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + nilLiteralExpr + nilKeyword: nil --- diff --git a/unified/extractor/tests/corpus/swift/literals/string-literal.output b/unified/extractor/tests/corpus/swift/literals/string-literal.output index ca2ac6df325..8d3ea8e796c 100644 --- a/unified/extractor/tests/corpus/swift/literals/string-literal.output +++ b/unified/extractor/tests/corpus/swift/literals/string-literal.output @@ -2,10 +2,17 @@ --- -source_file - statement: - line_string_literal - text: line_str_text "hello" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + stringLiteralExpr + closingQuote: " + openingQuote: " + segments: + stringSegment + content: stringSegment "hello" --- diff --git a/unified/extractor/tests/corpus/swift/literals/string-with-interpolation.output b/unified/extractor/tests/corpus/swift/literals/string-with-interpolation.output index eb56fbbbb03..5207085d174 100644 --- a/unified/extractor/tests/corpus/swift/literals/string-with-interpolation.output +++ b/unified/extractor/tests/corpus/swift/literals/string-with-interpolation.output @@ -2,13 +2,28 @@ --- -source_file - statement: - line_string_literal - interpolation: - interpolated_expression - value: simple_identifier "name" - text: line_str_text "hello " +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + stringLiteralExpr + closingQuote: " + openingQuote: " + segments: + stringSegment + content: stringSegment "hello " + expressionSegment + leftParen: ( + rightParen: ) + backslash: \ + expressions: + labeledExpr + expression: + declReferenceExpr + baseName: identifier "name" + stringSegment + content: stringSegment --- diff --git a/unified/extractor/tests/corpus/swift/loops/break-and-continue.output b/unified/extractor/tests/corpus/swift/loops/break-and-continue.output index 702cd0cbc68..7ce86999827 100644 --- a/unified/extractor/tests/corpus/swift/loops/break-and-continue.output +++ b/unified/extractor/tests/corpus/swift/loops/break-and-continue.output @@ -6,51 +6,95 @@ for x in xs { --- -source_file - statement: - for_statement - body: - block - statement: - if_statement - body: - block - statement: - control_transfer_statement - kind: continue - condition: - if_condition - kind: - comparison_expression - lhs: simple_identifier "x" - op: < - rhs: integer_literal "0" - if_statement - body: - block - statement: - control_transfer_statement - kind: break - condition: - if_condition - kind: - comparison_expression - lhs: simple_identifier "x" - op: > - rhs: integer_literal "100" - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: simple_identifier "x" - collection: simple_identifier "xs" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem item: - pattern - bound_identifier: simple_identifier "x" + forStmt + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + expressionStmt + expression: + ifExpr + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + continueStmt + continueKeyword: continue + conditions: + conditionElement + condition: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "<" + leftOperand: + declReferenceExpr + baseName: identifier "x" + rightOperand: + integerLiteralExpr + literal: integerLiteral "0" + ifKeyword: if + codeBlockItem + item: + expressionStmt + expression: + ifExpr + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + breakStmt + breakKeyword: break + conditions: + conditionElement + condition: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator ">" + leftOperand: + declReferenceExpr + baseName: identifier "x" + rightOperand: + integerLiteralExpr + literal: integerLiteral "100" + ifKeyword: if + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + declReferenceExpr + baseName: identifier "x" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + pattern: + identifierPattern + identifier: identifier "x" + inKeyword: in + forKeyword: for + sequence: + declReferenceExpr + baseName: identifier "xs" --- @@ -59,16 +103,22 @@ top_level block stmt: for_each_stmt + pattern: + name_pattern + identifier: identifier "x" + iterable: + name_expr + identifier: identifier "xs" body: block stmt: if_expr condition: binary_expr - operator: infix_operator "<" left: name_expr identifier: identifier "x" + operator: infix_operator "<" right: int_literal "0" then: block @@ -76,26 +126,20 @@ top_level if_expr condition: binary_expr - operator: infix_operator ">" left: name_expr identifier: identifier "x" + operator: infix_operator ">" right: int_literal "100" then: block stmt: break_expr "break" call_expr + callee: + name_expr + identifier: identifier "print" argument: argument value: name_expr identifier: identifier "x" - callee: - name_expr - identifier: identifier "print" - pattern: - name_pattern - identifier: identifier "x" - iterable: - name_expr - identifier: identifier "xs" diff --git a/unified/extractor/tests/corpus/swift/loops/for-in-over-array-literal.output b/unified/extractor/tests/corpus/swift/loops/for-in-over-array-literal.output index bb1711a2341..2d8db27bfe8 100644 --- a/unified/extractor/tests/corpus/swift/loops/for-in-over-array-literal.output +++ b/unified/extractor/tests/corpus/swift/loops/for-in-over-array-literal.output @@ -4,30 +4,55 @@ for x in [1, 2, 3] { --- -source_file - statement: - for_statement - body: - block - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: simple_identifier "x" - collection: - array_literal - element: - integer_literal "1" - integer_literal "2" - integer_literal "3" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem item: - pattern - bound_identifier: simple_identifier "x" + forStmt + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + declReferenceExpr + baseName: identifier "x" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + pattern: + identifierPattern + identifier: identifier "x" + inKeyword: in + forKeyword: for + sequence: + arrayExpr + elements: + arrayElement + expression: + integerLiteralExpr + literal: integerLiteral "1" + trailingComma: , + arrayElement + expression: + integerLiteralExpr + literal: integerLiteral "2" + trailingComma: , + arrayElement + expression: + integerLiteralExpr + literal: integerLiteral "3" + leftSquare: [ + rightSquare: ] --- @@ -36,18 +61,6 @@ top_level block stmt: for_each_stmt - body: - block - stmt: - call_expr - argument: - argument - value: - name_expr - identifier: identifier "x" - callee: - name_expr - identifier: identifier "print" pattern: name_pattern identifier: identifier "x" @@ -57,3 +70,15 @@ top_level int_literal "1" int_literal "2" int_literal "3" + body: + block + stmt: + call_expr + callee: + name_expr + identifier: identifier "print" + argument: + argument + value: + name_expr + identifier: identifier "x" diff --git a/unified/extractor/tests/corpus/swift/loops/for-in-over-range.output b/unified/extractor/tests/corpus/swift/loops/for-in-over-range.output index 87a0baf328b..cb8b4e67b68 100644 --- a/unified/extractor/tests/corpus/swift/loops/for-in-over-range.output +++ b/unified/extractor/tests/corpus/swift/loops/for-in-over-range.output @@ -4,29 +4,47 @@ for i in 0..<10 { --- -source_file - statement: - for_statement - body: - block - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: simple_identifier "i" - collection: - range_expression - end: integer_literal "10" - op: ..< - start: integer_literal "0" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem item: - pattern - bound_identifier: simple_identifier "i" + forStmt + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + declReferenceExpr + baseName: identifier "i" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + pattern: + identifierPattern + identifier: identifier "i" + inKeyword: in + forKeyword: for + sequence: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "..<" + leftOperand: + integerLiteralExpr + literal: integerLiteral "0" + rightOperand: + integerLiteralExpr + literal: integerLiteral "10" --- @@ -35,23 +53,23 @@ top_level block stmt: for_each_stmt - body: - block - stmt: - call_expr - argument: - argument - value: - name_expr - identifier: identifier "i" - callee: - name_expr - identifier: identifier "print" pattern: name_pattern identifier: identifier "i" iterable: binary_expr - operator: infix_operator "..<" left: int_literal "0" + operator: infix_operator "..<" right: int_literal "10" + body: + block + stmt: + call_expr + callee: + name_expr + identifier: identifier "print" + argument: + argument + value: + name_expr + identifier: identifier "i" diff --git a/unified/extractor/tests/corpus/swift/loops/for-in-with-where-clause.output b/unified/extractor/tests/corpus/swift/loops/for-in-with-where-clause.output index 84e832c2be5..6659039029f 100644 --- a/unified/extractor/tests/corpus/swift/loops/for-in-with-where-clause.output +++ b/unified/extractor/tests/corpus/swift/loops/for-in-with-where-clause.output @@ -4,33 +4,53 @@ for x in xs where x > 0 { --- -source_file - statement: - for_statement - body: - block - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: simple_identifier "x" - collection: simple_identifier "xs" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem item: - pattern - bound_identifier: simple_identifier "x" - where: - where_clause - expr: - comparison_expression - lhs: simple_identifier "x" - op: > - rhs: integer_literal "0" - keyword: where_keyword "where" + forStmt + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + declReferenceExpr + baseName: identifier "x" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + pattern: + identifierPattern + identifier: identifier "x" + whereClause: + whereClause + condition: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator ">" + leftOperand: + declReferenceExpr + baseName: identifier "x" + rightOperand: + integerLiteralExpr + literal: integerLiteral "0" + whereKeyword: where + inKeyword: in + forKeyword: for + sequence: + declReferenceExpr + baseName: identifier "xs" --- @@ -39,28 +59,28 @@ top_level block stmt: for_each_stmt + pattern: + name_pattern + identifier: identifier "x" + iterable: + name_expr + identifier: identifier "xs" + guard: + binary_expr + left: + name_expr + identifier: identifier "x" + operator: infix_operator ">" + right: int_literal "0" body: block stmt: call_expr + callee: + name_expr + identifier: identifier "print" argument: argument value: name_expr identifier: identifier "x" - callee: - name_expr - identifier: identifier "print" - pattern: - name_pattern - identifier: identifier "x" - guard: - binary_expr - operator: infix_operator ">" - left: - name_expr - identifier: identifier "x" - right: int_literal "0" - iterable: - name_expr - identifier: identifier "xs" diff --git a/unified/extractor/tests/corpus/swift/loops/repeat-while-loop.output b/unified/extractor/tests/corpus/swift/loops/repeat-while-loop.output index 15b673109f2..71c49fd2cd1 100644 --- a/unified/extractor/tests/corpus/swift/loops/repeat-while-loop.output +++ b/unified/extractor/tests/corpus/swift/loops/repeat-while-loop.output @@ -4,25 +4,42 @@ repeat { --- -source_file - statement: - repeat_while_statement - body: - block - statement: - assignment - operator: -= - result: integer_literal "1" - target: - directly_assignable_expression - expr: simple_identifier "x" - condition: - if_condition - kind: - comparison_expression - lhs: simple_identifier "x" - op: > - rhs: integer_literal "0" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + repeatStmt + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "-=" + leftOperand: + declReferenceExpr + baseName: identifier "x" + rightOperand: + integerLiteralExpr + literal: integerLiteral "1" + condition: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator ">" + leftOperand: + declReferenceExpr + baseName: identifier "x" + rightOperand: + integerLiteralExpr + literal: integerLiteral "0" + repeatKeyword: repeat + whileKeyword: while --- @@ -35,15 +52,15 @@ top_level block stmt: compound_assign_expr - operator: infix_operator "-=" target: name_expr identifier: identifier "x" + operator: infix_operator "-=" value: int_literal "1" condition: binary_expr - operator: infix_operator ">" left: name_expr identifier: identifier "x" + operator: infix_operator ">" right: int_literal "0" diff --git a/unified/extractor/tests/corpus/swift/loops/while-loop.output b/unified/extractor/tests/corpus/swift/loops/while-loop.output index 516ab43a2ee..1132c6f9f81 100644 --- a/unified/extractor/tests/corpus/swift/loops/while-loop.output +++ b/unified/extractor/tests/corpus/swift/loops/while-loop.output @@ -4,25 +4,43 @@ while x > 0 { --- -source_file - statement: - while_statement - body: - block - statement: - assignment - operator: -= - result: integer_literal "1" - target: - directly_assignable_expression - expr: simple_identifier "x" - condition: - if_condition - kind: - comparison_expression - lhs: simple_identifier "x" - op: > - rhs: integer_literal "0" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + whileStmt + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "-=" + leftOperand: + declReferenceExpr + baseName: identifier "x" + rightOperand: + integerLiteralExpr + literal: integerLiteral "1" + conditions: + conditionElement + condition: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator ">" + leftOperand: + declReferenceExpr + baseName: identifier "x" + rightOperand: + integerLiteralExpr + literal: integerLiteral "0" + whileKeyword: while --- @@ -31,19 +49,19 @@ top_level block stmt: while_stmt + condition: + binary_expr + left: + name_expr + identifier: identifier "x" + operator: infix_operator ">" + right: int_literal "0" body: block stmt: compound_assign_expr - operator: infix_operator "-=" target: name_expr identifier: identifier "x" + operator: infix_operator "-=" value: int_literal "1" - condition: - binary_expr - operator: infix_operator ">" - left: - name_expr - identifier: identifier "x" - right: int_literal "0" diff --git a/unified/extractor/tests/corpus/swift/operators/addition.output b/unified/extractor/tests/corpus/swift/operators/addition.output index 42c0ca9de61..9ba0f4de477 100644 --- a/unified/extractor/tests/corpus/swift/operators/addition.output +++ b/unified/extractor/tests/corpus/swift/operators/addition.output @@ -2,12 +2,21 @@ a + b --- -source_file - statement: - additive_expression - lhs: simple_identifier "a" - op: + - rhs: simple_identifier "b" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "+" + leftOperand: + declReferenceExpr + baseName: identifier "a" + rightOperand: + declReferenceExpr + baseName: identifier "b" --- @@ -16,10 +25,10 @@ top_level block stmt: binary_expr - operator: infix_operator "+" left: name_expr identifier: identifier "a" + operator: infix_operator "+" right: name_expr identifier: identifier "b" diff --git a/unified/extractor/tests/corpus/swift/operators/comparison.output b/unified/extractor/tests/corpus/swift/operators/comparison.output index f9428ad1758..49bb8c996f2 100644 --- a/unified/extractor/tests/corpus/swift/operators/comparison.output +++ b/unified/extractor/tests/corpus/swift/operators/comparison.output @@ -2,12 +2,21 @@ a < b --- -source_file - statement: - comparison_expression - lhs: simple_identifier "a" - op: < - rhs: simple_identifier "b" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "<" + leftOperand: + declReferenceExpr + baseName: identifier "a" + rightOperand: + declReferenceExpr + baseName: identifier "b" --- @@ -16,10 +25,10 @@ top_level block stmt: binary_expr - operator: infix_operator "<" left: name_expr identifier: identifier "a" + operator: infix_operator "<" right: name_expr identifier: identifier "b" diff --git a/unified/extractor/tests/corpus/swift/operators/division.output b/unified/extractor/tests/corpus/swift/operators/division.output index 76554954302..f15dce8e8ea 100644 --- a/unified/extractor/tests/corpus/swift/operators/division.output +++ b/unified/extractor/tests/corpus/swift/operators/division.output @@ -2,12 +2,21 @@ a / b --- -source_file - statement: - multiplicative_expression - lhs: simple_identifier "a" - op: / - rhs: simple_identifier "b" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "/" + leftOperand: + declReferenceExpr + baseName: identifier "a" + rightOperand: + declReferenceExpr + baseName: identifier "b" --- @@ -16,10 +25,10 @@ top_level block stmt: binary_expr - operator: infix_operator "/" left: name_expr identifier: identifier "a" + operator: infix_operator "/" right: name_expr identifier: identifier "b" diff --git a/unified/extractor/tests/corpus/swift/operators/equality.output b/unified/extractor/tests/corpus/swift/operators/equality.output index cc891492c75..7cf139ffa18 100644 --- a/unified/extractor/tests/corpus/swift/operators/equality.output +++ b/unified/extractor/tests/corpus/swift/operators/equality.output @@ -2,12 +2,21 @@ a == b --- -source_file - statement: - equality_expression - lhs: simple_identifier "a" - op: == - rhs: simple_identifier "b" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "==" + leftOperand: + declReferenceExpr + baseName: identifier "a" + rightOperand: + declReferenceExpr + baseName: identifier "b" --- @@ -16,10 +25,10 @@ top_level block stmt: binary_expr - operator: infix_operator "==" left: name_expr identifier: identifier "a" + operator: infix_operator "==" right: name_expr identifier: identifier "b" diff --git a/unified/extractor/tests/corpus/swift/operators/logical-and.output b/unified/extractor/tests/corpus/swift/operators/logical-and.output index bf852cd4614..32a71ed1088 100644 --- a/unified/extractor/tests/corpus/swift/operators/logical-and.output +++ b/unified/extractor/tests/corpus/swift/operators/logical-and.output @@ -2,12 +2,21 @@ a && b --- -source_file - statement: - conjunction_expression - lhs: simple_identifier "a" - op: && - rhs: simple_identifier "b" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "&&" + leftOperand: + declReferenceExpr + baseName: identifier "a" + rightOperand: + declReferenceExpr + baseName: identifier "b" --- @@ -16,10 +25,10 @@ top_level block stmt: binary_expr - operator: infix_operator "&&" left: name_expr identifier: identifier "a" + operator: infix_operator "&&" right: name_expr identifier: identifier "b" diff --git a/unified/extractor/tests/corpus/swift/operators/logical-not.output b/unified/extractor/tests/corpus/swift/operators/logical-not.output index d07e357620f..1e80aa2ca71 100644 --- a/unified/extractor/tests/corpus/swift/operators/logical-not.output +++ b/unified/extractor/tests/corpus/swift/operators/logical-not.output @@ -2,11 +2,16 @@ --- -source_file - statement: - prefix_expression - operation: bang "!" - target: simple_identifier "a" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + prefixOperatorExpr + expression: + declReferenceExpr + baseName: identifier "a" + operator: prefixOperator "!" --- diff --git a/unified/extractor/tests/corpus/swift/operators/logical-or.output b/unified/extractor/tests/corpus/swift/operators/logical-or.output index e246174844c..b75d018dea7 100644 --- a/unified/extractor/tests/corpus/swift/operators/logical-or.output +++ b/unified/extractor/tests/corpus/swift/operators/logical-or.output @@ -2,12 +2,21 @@ a || b --- -source_file - statement: - disjunction_expression - lhs: simple_identifier "a" - op: || - rhs: simple_identifier "b" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "||" + leftOperand: + declReferenceExpr + baseName: identifier "a" + rightOperand: + declReferenceExpr + baseName: identifier "b" --- @@ -16,10 +25,10 @@ top_level block stmt: binary_expr - operator: infix_operator "||" left: name_expr identifier: identifier "a" + operator: infix_operator "||" right: name_expr identifier: identifier "b" diff --git a/unified/extractor/tests/corpus/swift/operators/multiplication.output b/unified/extractor/tests/corpus/swift/operators/multiplication.output index b4c33b13286..387c16439c4 100644 --- a/unified/extractor/tests/corpus/swift/operators/multiplication.output +++ b/unified/extractor/tests/corpus/swift/operators/multiplication.output @@ -2,12 +2,21 @@ a * b --- -source_file - statement: - multiplicative_expression - lhs: simple_identifier "a" - op: * - rhs: simple_identifier "b" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "*" + leftOperand: + declReferenceExpr + baseName: identifier "a" + rightOperand: + declReferenceExpr + baseName: identifier "b" --- @@ -16,10 +25,10 @@ top_level block stmt: binary_expr - operator: infix_operator "*" left: name_expr identifier: identifier "a" + operator: infix_operator "*" right: name_expr identifier: identifier "b" diff --git a/unified/extractor/tests/corpus/swift/operators/operator-precedence-addition-and-multiplication.output b/unified/extractor/tests/corpus/swift/operators/operator-precedence-addition-and-multiplication.output index b1467474e7c..f458c3ef847 100644 --- a/unified/extractor/tests/corpus/swift/operators/operator-precedence-addition-and-multiplication.output +++ b/unified/extractor/tests/corpus/swift/operators/operator-precedence-addition-and-multiplication.output @@ -2,16 +2,29 @@ a + b * c --- -source_file - statement: - additive_expression - lhs: simple_identifier "a" - op: + - rhs: - multiplicative_expression - lhs: simple_identifier "b" - op: * - rhs: simple_identifier "c" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "+" + leftOperand: + declReferenceExpr + baseName: identifier "a" + rightOperand: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "*" + leftOperand: + declReferenceExpr + baseName: identifier "b" + rightOperand: + declReferenceExpr + baseName: identifier "c" --- @@ -20,16 +33,16 @@ top_level block stmt: binary_expr - operator: infix_operator "+" left: name_expr identifier: identifier "a" + operator: infix_operator "+" right: binary_expr - operator: infix_operator "*" left: name_expr identifier: identifier "b" + operator: infix_operator "*" right: name_expr identifier: identifier "c" diff --git a/unified/extractor/tests/corpus/swift/operators/parenthesised-expression.output b/unified/extractor/tests/corpus/swift/operators/parenthesised-expression.output index dfc60e5b7f7..0324476ce99 100644 --- a/unified/extractor/tests/corpus/swift/operators/parenthesised-expression.output +++ b/unified/extractor/tests/corpus/swift/operators/parenthesised-expression.output @@ -2,20 +2,35 @@ --- -source_file - statement: - multiplicative_expression - lhs: - tuple_expression - element: - tuple_expression_item - value: - additive_expression - lhs: simple_identifier "a" - op: + - rhs: simple_identifier "b" - op: * - rhs: simple_identifier "c" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "*" + leftOperand: + tupleExpr + leftParen: ( + rightParen: ) + elements: + labeledExpr + expression: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "+" + leftOperand: + declReferenceExpr + baseName: identifier "a" + rightOperand: + declReferenceExpr + baseName: identifier "b" + rightOperand: + declReferenceExpr + baseName: identifier "c" --- @@ -24,8 +39,8 @@ top_level block stmt: binary_expr - operator: infix_operator "*" left: tuple_expr "(a + b)" + operator: infix_operator "*" right: name_expr identifier: identifier "c" diff --git a/unified/extractor/tests/corpus/swift/operators/range-operator.output b/unified/extractor/tests/corpus/swift/operators/range-operator.output index 03d0290bb7c..55ed3ab971a 100644 --- a/unified/extractor/tests/corpus/swift/operators/range-operator.output +++ b/unified/extractor/tests/corpus/swift/operators/range-operator.output @@ -2,12 +2,21 @@ --- -source_file - statement: - range_expression - end: integer_literal "10" - op: ... - start: integer_literal "1" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "..." + leftOperand: + integerLiteralExpr + literal: integerLiteral "1" + rightOperand: + integerLiteralExpr + literal: integerLiteral "10" --- @@ -16,6 +25,6 @@ top_level block stmt: binary_expr - operator: infix_operator "..." left: int_literal "1" + operator: infix_operator "..." right: int_literal "10" diff --git a/unified/extractor/tests/corpus/swift/operators/subtraction.output b/unified/extractor/tests/corpus/swift/operators/subtraction.output index 69f75e72040..5c6fd51a2d3 100644 --- a/unified/extractor/tests/corpus/swift/operators/subtraction.output +++ b/unified/extractor/tests/corpus/swift/operators/subtraction.output @@ -2,12 +2,21 @@ a - b --- -source_file - statement: - additive_expression - lhs: simple_identifier "a" - op: - - rhs: simple_identifier "b" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "-" + leftOperand: + declReferenceExpr + baseName: identifier "a" + rightOperand: + declReferenceExpr + baseName: identifier "b" --- @@ -16,10 +25,10 @@ top_level block stmt: binary_expr - operator: infix_operator "-" left: name_expr identifier: identifier "a" + operator: infix_operator "-" right: name_expr identifier: identifier "b" diff --git a/unified/extractor/tests/corpus/swift/optionals-and-errors/do-catch.output b/unified/extractor/tests/corpus/swift/optionals-and-errors/do-catch.output index c807bd9b7b9..81491b295fc 100644 --- a/unified/extractor/tests/corpus/swift/optionals-and-errors/do-catch.output +++ b/unified/extractor/tests/corpus/swift/optionals-and-errors/do-catch.output @@ -6,37 +6,54 @@ do { --- -source_file - statement: - do_statement - body: - block - statement: - try_expression - expr: - call_expression - function: simple_identifier "foo" - suffix: - call_suffix - arguments: - value_arguments - operator: - try_operator - catch: - catch_block +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + doStmt body: - block - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: simple_identifier "error" - keyword: catch_keyword "catch" + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + tryExpr + expression: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "foo" + tryKeyword: try + catchClauses: + catchClause + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + declReferenceExpr + baseName: identifier "error" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + catchItems: + catchKeyword: catch + doKeyword: do --- @@ -61,11 +78,11 @@ top_level block stmt: call_expr + callee: + name_expr + identifier: identifier "print" argument: argument value: name_expr identifier: identifier "error" - callee: - name_expr - identifier: identifier "print" diff --git a/unified/extractor/tests/corpus/swift/optionals-and-errors/force-unwrap.output b/unified/extractor/tests/corpus/swift/optionals-and-errors/force-unwrap.output index 96fb627e18b..2c6fd1a6f76 100644 --- a/unified/extractor/tests/corpus/swift/optionals-and-errors/force-unwrap.output +++ b/unified/extractor/tests/corpus/swift/optionals-and-errors/force-unwrap.output @@ -2,21 +2,29 @@ let n = opt! --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "n" - value: - postfix_expression - operation: bang "!" - target: simple_identifier "opt" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + forceUnwrapExpr + expression: + declReferenceExpr + baseName: identifier "opt" + exclamationMark: ! + pattern: + identifierPattern + identifier: identifier "n" --- diff --git a/unified/extractor/tests/corpus/swift/optionals-and-errors/nil-coalescing.output b/unified/extractor/tests/corpus/swift/optionals-and-errors/nil-coalescing.output index 81a9a9187c0..31a039ad006 100644 --- a/unified/extractor/tests/corpus/swift/optionals-and-errors/nil-coalescing.output +++ b/unified/extractor/tests/corpus/swift/optionals-and-errors/nil-coalescing.output @@ -2,21 +2,34 @@ let n = opt ?? 0 --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "n" - value: - nil_coalescing_expression - if_nil: integer_literal "0" - value: simple_identifier "opt" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "??" + leftOperand: + declReferenceExpr + baseName: identifier "opt" + rightOperand: + integerLiteralExpr + literal: integerLiteral "0" + pattern: + identifierPattern + identifier: identifier "n" --- @@ -31,8 +44,8 @@ top_level identifier: identifier "n" value: binary_expr - operator: infix_operator "??" left: name_expr identifier: identifier "opt" + operator: infix_operator "??" right: int_literal "0" diff --git a/unified/extractor/tests/corpus/swift/optionals-and-errors/optional-chaining.output b/unified/extractor/tests/corpus/swift/optionals-and-errors/optional-chaining.output index 6c5b27a64fe..b69b0ae47d5 100644 --- a/unified/extractor/tests/corpus/swift/optionals-and-errors/optional-chaining.output +++ b/unified/extractor/tests/corpus/swift/optionals-and-errors/optional-chaining.output @@ -2,32 +2,44 @@ let n = obj?.foo?.bar --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "n" - value: - navigation_expression - suffix: - navigation_suffix - suffix: simple_identifier "bar" - target: - optional_chain_marker - expr: - navigation_expression - suffix: - navigation_suffix - suffix: simple_identifier "foo" - target: - optional_chain_marker - expr: simple_identifier "obj" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + memberAccessExpr + period: . + declName: + declReferenceExpr + baseName: identifier "bar" + base: + optionalChainingExpr + expression: + memberAccessExpr + period: . + declName: + declReferenceExpr + baseName: identifier "foo" + base: + optionalChainingExpr + expression: + declReferenceExpr + baseName: identifier "obj" + questionMark: ? + questionMark: ? + pattern: + identifierPattern + identifier: identifier "n" --- diff --git a/unified/extractor/tests/corpus/swift/optionals-and-errors/optional-type-annotation.output b/unified/extractor/tests/corpus/swift/optionals-and-errors/optional-type-annotation.output index 06191891496..1927b494ef9 100644 --- a/unified/extractor/tests/corpus/swift/optionals-and-errors/optional-type-annotation.output +++ b/unified/extractor/tests/corpus/swift/optionals-and-errors/optional-type-annotation.output @@ -2,29 +2,35 @@ let x: Int? = nil --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "x" - type: - type_annotation - type: - type - name: - optional_type - wrapped: - user_type - part: - simple_user_type - name: type_identifier "Int" - value: nil +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + nilLiteralExpr + nilKeyword: nil + pattern: + identifierPattern + identifier: identifier "x" + typeAnnotation: + typeAnnotation + colon: : + type: + optionalType + wrappedType: + identifierType + name: identifier "Int" + questionMark: ? --- diff --git a/unified/extractor/tests/corpus/swift/optionals-and-errors/throwing-function.output b/unified/extractor/tests/corpus/swift/optionals-and-errors/throwing-function.output index f1240bd0b3e..e68b4c3c57a 100644 --- a/unified/extractor/tests/corpus/swift/optionals-and-errors/throwing-function.output +++ b/unified/extractor/tests/corpus/swift/optionals-and-errors/throwing-function.output @@ -4,25 +4,50 @@ func read() throws -> String { --- -source_file - statement: - function_declaration - body: - block - statement: - control_transfer_statement - kind: return - result: - line_string_literal - name: simple_identifier "read" - return_type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "String" - throws: throws "throws" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + functionDecl + attributes: + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + returnStmt + expression: + stringLiteralExpr + closingQuote: " + openingQuote: " + segments: + stringSegment + content: stringSegment + returnKeyword: return + name: identifier "read" + modifiers: + signature: + functionSignature + effectSpecifiers: + functionEffectSpecifiers + throwsClause: + throwsClause + throwsSpecifier: throws + parameterClause: + functionParameterClause + leftParen: ( + rightParen: ) + parameters: + returnClause: + returnClause + arrow: -> + type: + identifierType + name: identifier "String" + funcKeyword: func --- @@ -31,12 +56,12 @@ top_level block stmt: function_declaration + name: identifier "read" + return_type: + named_type_expr + name: identifier "String" body: block stmt: return_expr value: string_literal "\"\"" - name: identifier "read" - return_type: - named_type_expr - name: identifier "String" diff --git a/unified/extractor/tests/corpus/swift/optionals-and-errors/try-expression-2.output b/unified/extractor/tests/corpus/swift/optionals-and-errors/try-expression-2.output index 9d5ff032d75..aafc0c67c21 100644 --- a/unified/extractor/tests/corpus/swift/optionals-and-errors/try-expression-2.output +++ b/unified/extractor/tests/corpus/swift/optionals-and-errors/try-expression-2.output @@ -2,28 +2,36 @@ let result = try! foo() --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "result" - value: - try_expression - expr: - call_expression - function: simple_identifier "foo" - suffix: - call_suffix - arguments: - value_arguments - operator: - try_operator +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + tryExpr + expression: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "foo" + questionOrExclamationMark: ! + tryKeyword: try + pattern: + identifierPattern + identifier: identifier "result" --- diff --git a/unified/extractor/tests/corpus/swift/optionals-and-errors/try-expression.output b/unified/extractor/tests/corpus/swift/optionals-and-errors/try-expression.output index e6a7bfef344..5b1b0b24d0f 100644 --- a/unified/extractor/tests/corpus/swift/optionals-and-errors/try-expression.output +++ b/unified/extractor/tests/corpus/swift/optionals-and-errors/try-expression.output @@ -2,28 +2,36 @@ let result = try? foo() --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "result" - value: - try_expression - expr: - call_expression - function: simple_identifier "foo" - suffix: - call_suffix - arguments: - value_arguments - operator: - try_operator +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + tryExpr + expression: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "foo" + questionOrExclamationMark: ? + tryKeyword: try + pattern: + identifierPattern + identifier: identifier "result" --- diff --git a/unified/extractor/tests/corpus/swift/types/binding-modifier-does-not-leak-into-accessor-body.output b/unified/extractor/tests/corpus/swift/types/binding-modifier-does-not-leak-into-accessor-body.output index 4ee195672bb..c05bc779443 100644 --- a/unified/extractor/tests/corpus/swift/types/binding-modifier-does-not-leak-into-accessor-body.output +++ b/unified/extractor/tests/corpus/swift/types/binding-modifier-does-not-leak-into-accessor-body.output @@ -11,54 +11,84 @@ var p: Int { --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: var - declarator: - property_binding - computed_value: - computed_property - accessor: - computed_getter - body: - block - statement: - switch_statement - entry: - switch_entry - pattern: - switch_pattern - pattern: - pattern - kind: simple_identifier "someConstant" - statement: - control_transfer_statement - kind: return - result: integer_literal "1" - switch_entry - default: default_keyword "default" - statement: - control_transfer_statement - kind: return - result: integer_literal "2" - expr: simple_identifier "y" - specifier: - getter_specifier - name: - pattern - bound_identifier: simple_identifier "p" - type: - type_annotation - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: var + bindings: + patternBinding + pattern: + identifierPattern + identifier: identifier "p" + typeAnnotation: + typeAnnotation + colon: : + type: + identifierType + name: identifier "Int" + accessorBlock: + accessorBlock + accessors: + accessorDecl + accessorSpecifier: get + attributes: + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + expressionStmt + expression: + switchExpr + leftBrace: { + rightBrace: } + cases: + switchCase + label: + switchCaseLabel + colon: : + caseKeyword: case + caseItems: + switchCaseItem + pattern: + expressionPattern + expression: + declReferenceExpr + baseName: identifier "someConstant" + statements: + codeBlockItem + item: + returnStmt + expression: + integerLiteralExpr + literal: integerLiteral "1" + returnKeyword: return + switchCase + label: + switchDefaultLabel + colon: : + defaultKeyword: default + statements: + codeBlockItem + item: + returnStmt + expression: + integerLiteralExpr + literal: integerLiteral "2" + returnKeyword: return + subject: + declReferenceExpr + baseName: identifier "y" + switchKeyword: switch + leftBrace: { + rightBrace: } --- @@ -67,34 +97,34 @@ top_level block stmt: accessor_declaration + modifier: modifier "var" + name: identifier "p" + accessor_kind: accessor_kind "get" + type: + named_type_expr + name: identifier "Int" body: block stmt: switch_expr + value: + name_expr + identifier: identifier "y" case: switch_case - body: - block - stmt: - return_expr - value: int_literal "1" pattern: expr_equality_pattern expr: name_expr identifier: identifier "someConstant" + body: + block + stmt: + return_expr + value: int_literal "1" switch_case body: block stmt: return_expr value: int_literal "2" - value: - name_expr - identifier: identifier "y" - modifier: modifier "var" - name: identifier "p" - type: - named_type_expr - name: identifier "Int" - accessor_kind: accessor_kind "get" diff --git a/unified/extractor/tests/corpus/swift/types/class-inheritance.output b/unified/extractor/tests/corpus/swift/types/class-inheritance.output index e90cccc4a8b..62a0a43414d 100644 --- a/unified/extractor/tests/corpus/swift/types/class-inheritance.output +++ b/unified/extractor/tests/corpus/swift/types/class-inheritance.output @@ -2,20 +2,29 @@ class Dog: Animal {} --- -source_file - statement: - class_declaration - body: - class_body - declaration_kind: class - inherits: - inheritance_specifier - inherits_from: - user_type - part: - simple_user_type - name: type_identifier "Animal" - name: type_identifier "Dog" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + classDecl + attributes: + name: identifier "Dog" + inheritanceClause: + inheritanceClause + colon: : + inheritedTypes: + inheritedType + type: + identifierType + name: identifier "Animal" + memberBlock: + memberBlock + leftBrace: { + rightBrace: } + members: + modifiers: + classKeyword: class --- diff --git a/unified/extractor/tests/corpus/swift/types/class-with-initializer.output b/unified/extractor/tests/corpus/swift/types/class-with-initializer.output index 13e0097172c..5e712d739e5 100644 --- a/unified/extractor/tests/corpus/swift/types/class-with-initializer.output +++ b/unified/extractor/tests/corpus/swift/types/class-with-initializer.output @@ -7,60 +7,82 @@ class Point { --- -source_file - statement: - class_declaration - body: - class_body - member: - property_declaration - binding: - value_binding_pattern - mutability: var - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "x" - type: - type_annotation - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" - init_declaration - body: - block - statement: - assignment - operator: = - result: simple_identifier "x" - target: - directly_assignable_expression - expr: - navigation_expression - suffix: - navigation_suffix - suffix: simple_identifier "x" - target: - self_expression - parameter: - function_parameter - parameter: - parameter - name: simple_identifier "x" - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" - declaration_kind: class - name: type_identifier "Point" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + classDecl + attributes: + name: identifier "Point" + memberBlock: + memberBlock + leftBrace: { + rightBrace: } + members: + memberBlockItem + decl: + variableDecl + attributes: + modifiers: + bindingSpecifier: var + bindings: + patternBinding + pattern: + identifierPattern + identifier: identifier "x" + typeAnnotation: + typeAnnotation + colon: : + type: + identifierType + name: identifier "Int" + memberBlockItem + decl: + initializerDecl + attributes: + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + assignmentExpr + equal: = + leftOperand: + memberAccessExpr + period: . + declName: + declReferenceExpr + baseName: identifier "x" + base: + declReferenceExpr + baseName: self + rightOperand: + declReferenceExpr + baseName: identifier "x" + modifiers: + signature: + functionSignature + parameterClause: + functionParameterClause + leftParen: ( + rightParen: ) + parameters: + functionParameter + colon: : + attributes: + modifiers: + type: + identifierType + name: identifier "Int" + firstName: identifier "x" + initKeyword: init + modifiers: + classKeyword: class --- @@ -69,6 +91,8 @@ top_level block stmt: class_like_declaration + modifier: modifier "class" + name: identifier "Point" member: variable_declaration modifier: modifier "var" @@ -92,5 +116,3 @@ top_level value: name_expr identifier: identifier "x" - modifier: modifier "class" - name: identifier "Point" diff --git a/unified/extractor/tests/corpus/swift/types/class-with-method.output b/unified/extractor/tests/corpus/swift/types/class-with-method.output index 770030d884a..f45cb31e53a 100644 --- a/unified/extractor/tests/corpus/swift/types/class-with-method.output +++ b/unified/extractor/tests/corpus/swift/types/class-with-method.output @@ -7,35 +7,69 @@ class Counter { --- -source_file - statement: - class_declaration - body: - class_body - member: - property_declaration - binding: - value_binding_pattern - mutability: var - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "n" - value: integer_literal "0" - function_declaration - body: - block - statement: - assignment - operator: += - result: integer_literal "1" - target: - directly_assignable_expression - expr: simple_identifier "n" - name: simple_identifier "bump" - declaration_kind: class - name: type_identifier "Counter" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + classDecl + attributes: + name: identifier "Counter" + memberBlock: + memberBlock + leftBrace: { + rightBrace: } + members: + memberBlockItem + decl: + variableDecl + attributes: + modifiers: + bindingSpecifier: var + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + integerLiteralExpr + literal: integerLiteral "0" + pattern: + identifierPattern + identifier: identifier "n" + memberBlockItem + decl: + functionDecl + attributes: + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "+=" + leftOperand: + declReferenceExpr + baseName: identifier "n" + rightOperand: + integerLiteralExpr + literal: integerLiteral "1" + name: identifier "bump" + modifiers: + signature: + functionSignature + parameterClause: + functionParameterClause + leftParen: ( + rightParen: ) + parameters: + funcKeyword: func + modifiers: + classKeyword: class --- @@ -44,6 +78,8 @@ top_level block stmt: class_like_declaration + modifier: modifier "class" + name: identifier "Counter" member: variable_declaration modifier: modifier "var" @@ -52,15 +88,13 @@ top_level identifier: identifier "n" value: int_literal "0" function_declaration + name: identifier "bump" body: block stmt: compound_assign_expr - operator: infix_operator "+=" target: name_expr identifier: identifier "n" + operator: infix_operator "+=" value: int_literal "1" - name: identifier "bump" - modifier: modifier "class" - name: identifier "Counter" diff --git a/unified/extractor/tests/corpus/swift/types/class-with-stored-properties.output b/unified/extractor/tests/corpus/swift/types/class-with-stored-properties.output index 9d28afe6ae0..e184426eb75 100644 --- a/unified/extractor/tests/corpus/swift/types/class-with-stored-properties.output +++ b/unified/extractor/tests/corpus/swift/types/class-with-stored-properties.output @@ -5,50 +5,55 @@ class Point { --- -source_file - statement: - class_declaration - body: - class_body - member: - property_declaration - binding: - value_binding_pattern - mutability: var - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "x" - type: - type_annotation - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" - property_declaration - binding: - value_binding_pattern - mutability: var - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "y" - type: - type_annotation - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" - declaration_kind: class - name: type_identifier "Point" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + classDecl + attributes: + name: identifier "Point" + memberBlock: + memberBlock + leftBrace: { + rightBrace: } + members: + memberBlockItem + decl: + variableDecl + attributes: + modifiers: + bindingSpecifier: var + bindings: + patternBinding + pattern: + identifierPattern + identifier: identifier "x" + typeAnnotation: + typeAnnotation + colon: : + type: + identifierType + name: identifier "Int" + memberBlockItem + decl: + variableDecl + attributes: + modifiers: + bindingSpecifier: var + bindings: + patternBinding + pattern: + identifierPattern + identifier: identifier "y" + typeAnnotation: + typeAnnotation + colon: : + type: + identifierType + name: identifier "Int" + modifiers: + classKeyword: class --- @@ -57,6 +62,8 @@ top_level block stmt: class_like_declaration + modifier: modifier "class" + name: identifier "Point" member: variable_declaration modifier: modifier "var" @@ -74,5 +81,3 @@ top_level type: named_type_expr name: identifier "Int" - modifier: modifier "class" - name: identifier "Point" diff --git a/unified/extractor/tests/corpus/swift/types/computed-property.output b/unified/extractor/tests/corpus/swift/types/computed-property.output index 8803e652d31..24e816be009 100644 --- a/unified/extractor/tests/corpus/swift/types/computed-property.output +++ b/unified/extractor/tests/corpus/swift/types/computed-property.output @@ -8,78 +8,92 @@ class Rect { --- -source_file - statement: - class_declaration - body: - class_body - member: - property_declaration - binding: - value_binding_pattern - mutability: var - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "w" - type: - type_annotation - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Double" - property_declaration - binding: - value_binding_pattern - mutability: var - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "h" - type: - type_annotation - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Double" - property_declaration - binding: - value_binding_pattern - mutability: var - declarator: - property_binding - computed_value: - computed_property - statement: - control_transfer_statement - kind: return - result: - multiplicative_expression - lhs: simple_identifier "w" - op: * - rhs: simple_identifier "h" - name: - pattern - bound_identifier: simple_identifier "area" - type: - type_annotation - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Double" - declaration_kind: class - name: type_identifier "Rect" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + classDecl + attributes: + name: identifier "Rect" + memberBlock: + memberBlock + leftBrace: { + rightBrace: } + members: + memberBlockItem + decl: + variableDecl + attributes: + modifiers: + bindingSpecifier: var + bindings: + patternBinding + pattern: + identifierPattern + identifier: identifier "w" + typeAnnotation: + typeAnnotation + colon: : + type: + identifierType + name: identifier "Double" + memberBlockItem + decl: + variableDecl + attributes: + modifiers: + bindingSpecifier: var + bindings: + patternBinding + pattern: + identifierPattern + identifier: identifier "h" + typeAnnotation: + typeAnnotation + colon: : + type: + identifierType + name: identifier "Double" + memberBlockItem + decl: + variableDecl + attributes: + modifiers: + bindingSpecifier: var + bindings: + patternBinding + pattern: + identifierPattern + identifier: identifier "area" + typeAnnotation: + typeAnnotation + colon: : + type: + identifierType + name: identifier "Double" + accessorBlock: + accessorBlock + accessors: + codeBlockItem + item: + returnStmt + expression: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "*" + leftOperand: + declReferenceExpr + baseName: identifier "w" + rightOperand: + declReferenceExpr + baseName: identifier "h" + returnKeyword: return + leftBrace: { + rightBrace: } + modifiers: + classKeyword: class --- @@ -88,6 +102,8 @@ top_level block stmt: class_like_declaration + modifier: modifier "class" + name: identifier "Rect" member: variable_declaration modifier: modifier "var" @@ -106,24 +122,22 @@ top_level named_type_expr name: identifier "Double" accessor_declaration + modifier: modifier "var" + name: identifier "area" + accessor_kind: accessor_kind "get" + type: + named_type_expr + name: identifier "Double" body: block stmt: return_expr value: binary_expr - operator: infix_operator "*" left: name_expr identifier: identifier "w" + operator: infix_operator "*" right: name_expr identifier: identifier "h" - modifier: modifier "var" - name: identifier "area" - type: - named_type_expr - name: identifier "Double" - accessor_kind: accessor_kind "get" - modifier: modifier "class" - name: identifier "Rect" diff --git a/unified/extractor/tests/corpus/swift/types/empty-class.output b/unified/extractor/tests/corpus/swift/types/empty-class.output index 761de778543..6693744c9b4 100644 --- a/unified/extractor/tests/corpus/swift/types/empty-class.output +++ b/unified/extractor/tests/corpus/swift/types/empty-class.output @@ -2,13 +2,21 @@ class Foo {} --- -source_file - statement: - class_declaration - body: - class_body - declaration_kind: class - name: type_identifier "Foo" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + classDecl + attributes: + name: identifier "Foo" + memberBlock: + memberBlock + leftBrace: { + rightBrace: } + members: + modifiers: + classKeyword: class --- diff --git a/unified/extractor/tests/corpus/swift/types/enum-with-associated-values.output b/unified/extractor/tests/corpus/swift/types/enum-with-associated-values.output index 12b5191f69f..399239bbea3 100644 --- a/unified/extractor/tests/corpus/swift/types/enum-with-associated-values.output +++ b/unified/extractor/tests/corpus/swift/types/enum-with-associated-values.output @@ -5,46 +5,63 @@ enum Shape { --- -source_file - statement: - class_declaration - body: - enum_class_body - member: - enum_entry - case: - enum_case_entry - data_contents: - enum_type_parameters - parameter: - enum_type_parameter - name: simple_identifier "radius" - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Double" - name: simple_identifier "circle" - enum_entry - case: - enum_case_entry - data_contents: - enum_type_parameters - parameter: - enum_type_parameter - name: simple_identifier "side" - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Double" - name: simple_identifier "square" - declaration_kind: enum - name: type_identifier "Shape" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + enumDecl + attributes: + name: identifier "Shape" + memberBlock: + memberBlock + leftBrace: { + rightBrace: } + members: + memberBlockItem + decl: + enumCaseDecl + attributes: + modifiers: + elements: + enumCaseElement + name: identifier "circle" + parameterClause: + enumCaseParameterClause + leftParen: ( + rightParen: ) + parameters: + enumCaseParameter + colon: : + modifiers: + type: + identifierType + name: identifier "Double" + firstName: identifier "radius" + caseKeyword: case + memberBlockItem + decl: + enumCaseDecl + attributes: + modifiers: + elements: + enumCaseElement + name: identifier "square" + parameterClause: + enumCaseParameterClause + leftParen: ( + rightParen: ) + parameters: + enumCaseParameter + colon: : + modifiers: + type: + identifierType + name: identifier "Double" + firstName: identifier "side" + caseKeyword: case + modifiers: + enumKeyword: enum --- @@ -53,34 +70,34 @@ top_level block stmt: class_like_declaration + modifier: modifier "enum" + name: identifier "Shape" member: class_like_declaration + modifier: modifier "enum_case" + name: identifier "circle" member: constructor_declaration - body: block "circle(radius: Double)" parameter: parameter + type: + named_type_expr + name: identifier "Double" pattern: name_pattern identifier: identifier "radius" + body: block "circle(radius: Double)" + class_like_declaration + modifier: modifier "enum_case" + name: identifier "square" + member: + constructor_declaration + parameter: + parameter type: named_type_expr name: identifier "Double" - modifier: modifier "enum_case" - name: identifier "circle" - class_like_declaration - member: - constructor_declaration - body: block "square(side: Double)" - parameter: - parameter pattern: name_pattern identifier: identifier "side" - type: - named_type_expr - name: identifier "Double" - modifier: modifier "enum_case" - name: identifier "square" - modifier: modifier "enum" - name: identifier "Shape" + body: block "square(side: Double)" diff --git a/unified/extractor/tests/corpus/swift/types/enum-with-cases.output b/unified/extractor/tests/corpus/swift/types/enum-with-cases.output index 72435fd2b1f..f081c2a8204 100644 --- a/unified/extractor/tests/corpus/swift/types/enum-with-cases.output +++ b/unified/extractor/tests/corpus/swift/types/enum-with-cases.output @@ -7,30 +7,57 @@ enum Direction { --- -source_file - statement: - class_declaration - body: - enum_class_body - member: - enum_entry - case: - enum_case_entry - name: simple_identifier "north" - enum_entry - case: - enum_case_entry - name: simple_identifier "south" - enum_entry - case: - enum_case_entry - name: simple_identifier "east" - enum_entry - case: - enum_case_entry - name: simple_identifier "west" - declaration_kind: enum - name: type_identifier "Direction" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + enumDecl + attributes: + name: identifier "Direction" + memberBlock: + memberBlock + leftBrace: { + rightBrace: } + members: + memberBlockItem + decl: + enumCaseDecl + attributes: + modifiers: + elements: + enumCaseElement + name: identifier "north" + caseKeyword: case + memberBlockItem + decl: + enumCaseDecl + attributes: + modifiers: + elements: + enumCaseElement + name: identifier "south" + caseKeyword: case + memberBlockItem + decl: + enumCaseDecl + attributes: + modifiers: + elements: + enumCaseElement + name: identifier "east" + caseKeyword: case + memberBlockItem + decl: + enumCaseDecl + attributes: + modifiers: + elements: + enumCaseElement + name: identifier "west" + caseKeyword: case + modifiers: + enumKeyword: enum --- @@ -39,6 +66,8 @@ top_level block stmt: class_like_declaration + modifier: modifier "enum" + name: identifier "Direction" member: variable_declaration modifier: modifier "enum_case" @@ -60,5 +89,3 @@ top_level pattern: name_pattern identifier: identifier "west" - modifier: modifier "enum" - name: identifier "Direction" diff --git a/unified/extractor/tests/corpus/swift/types/enum-with-comma-separated-cases-chained-declaration.output b/unified/extractor/tests/corpus/swift/types/enum-with-comma-separated-cases-chained-declaration.output index 6a4aac4b552..6a4ea95e9a6 100644 --- a/unified/extractor/tests/corpus/swift/types/enum-with-comma-separated-cases-chained-declaration.output +++ b/unified/extractor/tests/corpus/swift/types/enum-with-comma-separated-cases-chained-declaration.output @@ -4,24 +4,39 @@ enum Suit { --- -source_file - statement: - class_declaration - body: - enum_class_body - member: - enum_entry - case: - enum_case_entry - name: simple_identifier "clubs" - enum_case_entry - name: simple_identifier "diamonds" - enum_case_entry - name: simple_identifier "hearts" - enum_case_entry - name: simple_identifier "spades" - declaration_kind: enum - name: type_identifier "Suit" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + enumDecl + attributes: + name: identifier "Suit" + memberBlock: + memberBlock + leftBrace: { + rightBrace: } + members: + memberBlockItem + decl: + enumCaseDecl + attributes: + modifiers: + elements: + enumCaseElement + name: identifier "clubs" + trailingComma: , + enumCaseElement + name: identifier "diamonds" + trailingComma: , + enumCaseElement + name: identifier "hearts" + trailingComma: , + enumCaseElement + name: identifier "spades" + caseKeyword: case + modifiers: + enumKeyword: enum --- @@ -30,6 +45,8 @@ top_level block stmt: class_like_declaration + modifier: modifier "enum" + name: identifier "Suit" member: variable_declaration modifier: modifier "enum_case" @@ -57,5 +74,3 @@ top_level pattern: name_pattern identifier: identifier "spades" - modifier: modifier "enum" - name: identifier "Suit" diff --git a/unified/extractor/tests/corpus/swift/types/extension.output b/unified/extractor/tests/corpus/swift/types/extension.output index ad663cb86e1..1b1d02ebdda 100644 --- a/unified/extractor/tests/corpus/swift/types/extension.output +++ b/unified/extractor/tests/corpus/swift/types/extension.output @@ -4,39 +4,63 @@ extension Int { --- -source_file - statement: - class_declaration - body: - class_body - member: - function_declaration - body: - block - statement: - control_transfer_statement - kind: return - result: - multiplicative_expression - lhs: - self_expression - op: * - rhs: - self_expression - name: simple_identifier "squared" - return_type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" - declaration_kind: extension - name: - user_type - part: - simple_user_type - name: type_identifier "Int" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + extensionDecl + attributes: + memberBlock: + memberBlock + leftBrace: { + rightBrace: } + members: + memberBlockItem + decl: + functionDecl + attributes: + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + returnStmt + expression: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "*" + leftOperand: + declReferenceExpr + baseName: self + rightOperand: + declReferenceExpr + baseName: self + returnKeyword: return + name: identifier "squared" + modifiers: + signature: + functionSignature + parameterClause: + functionParameterClause + leftParen: ( + rightParen: ) + parameters: + returnClause: + returnClause + arrow: -> + type: + identifierType + name: identifier "Int" + funcKeyword: func + modifiers: + extendedType: + identifierType + name: identifier "Int" + extensionKeyword: extension --- @@ -45,24 +69,24 @@ top_level block stmt: class_like_declaration + modifier: modifier "extension" + name: identifier "Int" member: function_declaration + name: identifier "squared" + return_type: + named_type_expr + name: identifier "Int" body: block stmt: return_expr value: binary_expr - operator: infix_operator "*" left: name_expr identifier: identifier "self" + operator: infix_operator "*" right: name_expr identifier: identifier "self" - name: identifier "squared" - return_type: - named_type_expr - name: identifier "Int" - modifier: modifier "extension" - name: identifier "Int" diff --git a/unified/extractor/tests/corpus/swift/types/property-with-getter-and-setter.output b/unified/extractor/tests/corpus/swift/types/property-with-getter-and-setter.output index 0a8b2d8cc5d..951768cd844 100644 --- a/unified/extractor/tests/corpus/swift/types/property-with-getter-and-setter.output +++ b/unified/extractor/tests/corpus/swift/types/property-with-getter-and-setter.output @@ -8,70 +8,97 @@ class Box { --- -source_file - statement: - class_declaration - body: - class_body - member: - property_declaration - binding: - value_binding_pattern - mutability: var - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "_v" - value: integer_literal "0" - modifiers: - modifiers - modifier: - visibility_modifier - property_declaration - binding: - value_binding_pattern - mutability: var - declarator: - property_binding - computed_value: - computed_property - accessor: - computed_getter - body: - block - statement: - control_transfer_statement - kind: return - result: simple_identifier "_v" - specifier: - getter_specifier - computed_setter - body: - block - statement: - assignment - operator: = - result: simple_identifier "newValue" - target: - directly_assignable_expression - expr: simple_identifier "_v" - specifier: - setter_specifier - name: - pattern - bound_identifier: simple_identifier "v" - type: - type_annotation - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" - declaration_kind: class - name: type_identifier "Box" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + classDecl + attributes: + name: identifier "Box" + memberBlock: + memberBlock + leftBrace: { + rightBrace: } + members: + memberBlockItem + decl: + variableDecl + attributes: + modifiers: + declModifier + name: private + bindingSpecifier: var + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + integerLiteralExpr + literal: integerLiteral "0" + pattern: + identifierPattern + identifier: identifier "_v" + memberBlockItem + decl: + variableDecl + attributes: + modifiers: + bindingSpecifier: var + bindings: + patternBinding + pattern: + identifierPattern + identifier: identifier "v" + typeAnnotation: + typeAnnotation + colon: : + type: + identifierType + name: identifier "Int" + accessorBlock: + accessorBlock + accessors: + accessorDecl + accessorSpecifier: get + attributes: + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + returnStmt + expression: + declReferenceExpr + baseName: identifier "_v" + returnKeyword: return + accessorDecl + accessorSpecifier: set + attributes: + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + assignmentExpr + equal: = + leftOperand: + declReferenceExpr + baseName: identifier "_v" + rightOperand: + declReferenceExpr + baseName: identifier "newValue" + leftBrace: { + rightBrace: } + modifiers: + classKeyword: class --- @@ -80,14 +107,24 @@ top_level block stmt: class_like_declaration + modifier: modifier "class" + name: identifier "Box" member: variable_declaration - modifier: modifier "var" + modifier: + modifier "var" + modifier "private" pattern: name_pattern identifier: identifier "_v" value: int_literal "0" accessor_declaration + modifier: modifier "var" + name: identifier "v" + accessor_kind: accessor_kind "get" + type: + named_type_expr + name: identifier "Int" body: block stmt: @@ -95,13 +132,15 @@ top_level value: name_expr identifier: identifier "_v" - modifier: modifier "var" + accessor_declaration + modifier: + modifier "var" + modifier "chained_declaration" name: identifier "v" + accessor_kind: accessor_kind "set" type: named_type_expr name: identifier "Int" - accessor_kind: accessor_kind "get" - accessor_declaration body: block stmt: @@ -112,13 +151,3 @@ top_level value: name_expr identifier: identifier "newValue" - modifier: - modifier "var" - modifier "chained_declaration" - name: identifier "v" - type: - named_type_expr - name: identifier "Int" - accessor_kind: accessor_kind "set" - modifier: modifier "class" - name: identifier "Box" diff --git a/unified/extractor/tests/corpus/swift/types/protocol-declaration.output b/unified/extractor/tests/corpus/swift/types/protocol-declaration.output index 55a71218c18..e848fb23eb3 100644 --- a/unified/extractor/tests/corpus/swift/types/protocol-declaration.output +++ b/unified/extractor/tests/corpus/swift/types/protocol-declaration.output @@ -4,15 +4,35 @@ protocol Drawable { --- -source_file - statement: - protocol_declaration - body: - protocol_body - member: - protocol_function_declaration - name: simple_identifier "draw" - name: type_identifier "Drawable" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + protocolDecl + attributes: + name: identifier "Drawable" + memberBlock: + memberBlock + leftBrace: { + rightBrace: } + members: + memberBlockItem + decl: + functionDecl + attributes: + name: identifier "draw" + modifiers: + signature: + functionSignature + parameterClause: + functionParameterClause + leftParen: ( + rightParen: ) + parameters: + funcKeyword: func + modifiers: + protocolKeyword: protocol --- @@ -21,9 +41,9 @@ top_level block stmt: class_like_declaration - member: - function_declaration - body: block "func draw()" - name: identifier "draw" modifier: modifier "protocol" name: identifier "Drawable" + member: + function_declaration + name: identifier "draw" + body: block "func draw()" diff --git a/unified/extractor/tests/corpus/swift/types/protocol-with-read-only-and-read-write-property-requirements.output b/unified/extractor/tests/corpus/swift/types/protocol-with-read-only-and-read-write-property-requirements.output index 0293534adf7..83362740bb1 100644 --- a/unified/extractor/tests/corpus/swift/types/protocol-with-read-only-and-read-write-property-requirements.output +++ b/unified/extractor/tests/corpus/swift/types/protocol-with-read-only-and-read-write-property-requirements.output @@ -5,54 +5,74 @@ protocol P { --- -source_file - statement: - protocol_declaration - body: - protocol_body - member: - protocol_property_declaration - name: - pattern - binding: - value_binding_pattern - mutability: var - bound_identifier: simple_identifier "foo" - requirements: - protocol_property_requirements - accessor: - getter_specifier - type: - type_annotation - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" - protocol_property_declaration - name: - pattern - binding: - value_binding_pattern - mutability: var - bound_identifier: simple_identifier "bar" - requirements: - protocol_property_requirements - accessor: - getter_specifier - setter_specifier - type: - type_annotation - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "String" - name: type_identifier "P" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + protocolDecl + attributes: + name: identifier "P" + memberBlock: + memberBlock + leftBrace: { + rightBrace: } + members: + memberBlockItem + decl: + variableDecl + attributes: + modifiers: + bindingSpecifier: var + bindings: + patternBinding + pattern: + identifierPattern + identifier: identifier "foo" + typeAnnotation: + typeAnnotation + colon: : + type: + identifierType + name: identifier "Int" + accessorBlock: + accessorBlock + accessors: + accessorDecl + accessorSpecifier: get + attributes: + leftBrace: { + rightBrace: } + memberBlockItem + decl: + variableDecl + attributes: + modifiers: + bindingSpecifier: var + bindings: + patternBinding + pattern: + identifierPattern + identifier: identifier "bar" + typeAnnotation: + typeAnnotation + colon: : + type: + identifierType + name: identifier "String" + accessorBlock: + accessorBlock + accessors: + accessorDecl + accessorSpecifier: get + attributes: + accessorDecl + accessorSpecifier: set + attributes: + leftBrace: { + rightBrace: } + modifiers: + protocolKeyword: protocol --- @@ -61,25 +81,25 @@ top_level block stmt: class_like_declaration + modifier: modifier "protocol" + name: identifier "P" member: accessor_declaration name: identifier "foo" + accessor_kind: accessor_kind "get" type: named_type_expr name: identifier "Int" - accessor_kind: accessor_kind "get" accessor_declaration name: identifier "bar" + accessor_kind: accessor_kind "get" type: named_type_expr name: identifier "String" - accessor_kind: accessor_kind "get" accessor_declaration modifier: modifier "chained_declaration" name: identifier "bar" + accessor_kind: accessor_kind "set" type: named_type_expr name: identifier "String" - accessor_kind: accessor_kind "set" - modifier: modifier "protocol" - name: identifier "P" diff --git a/unified/extractor/tests/corpus/swift/types/struct.output b/unified/extractor/tests/corpus/swift/types/struct.output index e130ef9b862..57fb25c9e19 100644 --- a/unified/extractor/tests/corpus/swift/types/struct.output +++ b/unified/extractor/tests/corpus/swift/types/struct.output @@ -5,50 +5,55 @@ struct Point { --- -source_file - statement: - class_declaration - body: - class_body - member: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "x" - type: - type_annotation - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "y" - type: - type_annotation - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" - declaration_kind: struct - name: type_identifier "Point" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + structDecl + attributes: + name: identifier "Point" + memberBlock: + memberBlock + leftBrace: { + rightBrace: } + members: + memberBlockItem + decl: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + pattern: + identifierPattern + identifier: identifier "x" + typeAnnotation: + typeAnnotation + colon: : + type: + identifierType + name: identifier "Int" + memberBlockItem + decl: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + pattern: + identifierPattern + identifier: identifier "y" + typeAnnotation: + typeAnnotation + colon: : + type: + identifierType + name: identifier "Int" + modifiers: + structKeyword: struct --- @@ -57,6 +62,8 @@ top_level block stmt: class_like_declaration + modifier: modifier "struct" + name: identifier "Point" member: variable_declaration modifier: modifier "let" @@ -74,5 +81,3 @@ top_level type: named_type_expr name: identifier "Int" - modifier: modifier "struct" - name: identifier "Point" diff --git a/unified/extractor/tests/corpus/swift/variables/assignment.output b/unified/extractor/tests/corpus/swift/variables/assignment.output index 9d1a61e89a8..a011eb76caf 100644 --- a/unified/extractor/tests/corpus/swift/variables/assignment.output +++ b/unified/extractor/tests/corpus/swift/variables/assignment.output @@ -2,14 +2,21 @@ x = 1 --- -source_file - statement: - assignment - operator: = - result: integer_literal "1" - target: - directly_assignable_expression - expr: simple_identifier "x" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + assignmentExpr + equal: = + leftOperand: + declReferenceExpr + baseName: identifier "x" + rightOperand: + integerLiteralExpr + literal: integerLiteral "1" --- diff --git a/unified/extractor/tests/corpus/swift/variables/binding-modifier-does-not-leak-into-initializer.output b/unified/extractor/tests/corpus/swift/variables/binding-modifier-does-not-leak-into-initializer.output index 65364c1ef91..d159cd6b37c 100644 --- a/unified/extractor/tests/corpus/swift/variables/binding-modifier-does-not-leak-into-initializer.output +++ b/unified/extractor/tests/corpus/swift/variables/binding-modifier-does-not-leak-into-initializer.output @@ -5,31 +5,59 @@ default: 2 --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "x" - value: - switch_statement - entry: - switch_entry - pattern: - switch_pattern - pattern: - pattern - kind: simple_identifier "someConstant" - statement: integer_literal "1" - switch_entry - default: default_keyword "default" - statement: integer_literal "2" - expr: simple_identifier "y" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + switchExpr + leftBrace: { + rightBrace: } + cases: + switchCase + label: + switchCaseLabel + colon: : + caseKeyword: case + caseItems: + switchCaseItem + pattern: + expressionPattern + expression: + declReferenceExpr + baseName: identifier "someConstant" + statements: + codeBlockItem + item: + integerLiteralExpr + literal: integerLiteral "1" + switchCase + label: + switchDefaultLabel + colon: : + defaultKeyword: default + statements: + codeBlockItem + item: + integerLiteralExpr + literal: integerLiteral "2" + subject: + declReferenceExpr + baseName: identifier "y" + switchKeyword: switch + pattern: + identifierPattern + identifier: identifier "x" --- @@ -44,20 +72,20 @@ top_level identifier: identifier "x" value: switch_expr + value: + name_expr + identifier: identifier "y" case: switch_case - body: - block - stmt: int_literal "1" pattern: expr_equality_pattern expr: name_expr identifier: identifier "someConstant" + body: + block + stmt: int_literal "1" switch_case body: block stmt: int_literal "2" - value: - name_expr - identifier: identifier "y" diff --git a/unified/extractor/tests/corpus/swift/variables/compound-assignment.output b/unified/extractor/tests/corpus/swift/variables/compound-assignment.output index 95b7edb0a6b..485b5044ad6 100644 --- a/unified/extractor/tests/corpus/swift/variables/compound-assignment.output +++ b/unified/extractor/tests/corpus/swift/variables/compound-assignment.output @@ -2,14 +2,21 @@ x += 1 --- -source_file - statement: - assignment - operator: += - result: integer_literal "1" - target: - directly_assignable_expression - expr: simple_identifier "x" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + infixOperatorExpr + operator: + binaryOperatorExpr + operator: binaryOperator "+=" + leftOperand: + declReferenceExpr + baseName: identifier "x" + rightOperand: + integerLiteralExpr + literal: integerLiteral "1" --- @@ -18,8 +25,8 @@ top_level block stmt: compound_assign_expr - operator: infix_operator "+=" target: name_expr identifier: identifier "x" + operator: infix_operator "+=" value: int_literal "1" diff --git a/unified/extractor/tests/corpus/swift/variables/let-binding.output b/unified/extractor/tests/corpus/swift/variables/let-binding.output index b8b16dc8014..4774eb3eeca 100644 --- a/unified/extractor/tests/corpus/swift/variables/let-binding.output +++ b/unified/extractor/tests/corpus/swift/variables/let-binding.output @@ -2,18 +2,26 @@ let x = 1 --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "x" - value: integer_literal "1" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + integerLiteralExpr + literal: integerLiteral "1" + pattern: + identifierPattern + identifier: identifier "x" --- diff --git a/unified/extractor/tests/corpus/swift/variables/let-with-type-annotation.output b/unified/extractor/tests/corpus/swift/variables/let-with-type-annotation.output index 3fd78d09fa4..a56feb445ac 100644 --- a/unified/extractor/tests/corpus/swift/variables/let-with-type-annotation.output +++ b/unified/extractor/tests/corpus/swift/variables/let-with-type-annotation.output @@ -2,27 +2,32 @@ let x: Int = 1 --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "x" - type: - type_annotation - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" - value: integer_literal "1" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + integerLiteralExpr + literal: integerLiteral "1" + pattern: + identifierPattern + identifier: identifier "x" + typeAnnotation: + typeAnnotation + colon: : + type: + identifierType + name: identifier "Int" --- diff --git a/unified/extractor/tests/corpus/swift/variables/multiple-bindings-on-one-line.output b/unified/extractor/tests/corpus/swift/variables/multiple-bindings-on-one-line.output index 7f202885b9b..9470a578ad6 100644 --- a/unified/extractor/tests/corpus/swift/variables/multiple-bindings-on-one-line.output +++ b/unified/extractor/tests/corpus/swift/variables/multiple-bindings-on-one-line.output @@ -2,23 +2,37 @@ let x = 1, y = 2 --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "x" - value: integer_literal "1" - property_binding - name: - pattern - bound_identifier: simple_identifier "y" - value: integer_literal "2" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + trailingComma: , + initializer: + initializerClause + equal: = + value: + integerLiteralExpr + literal: integerLiteral "1" + pattern: + identifierPattern + identifier: identifier "x" + patternBinding + initializer: + initializerClause + equal: = + value: + integerLiteralExpr + literal: integerLiteral "2" + pattern: + identifierPattern + identifier: identifier "y" --- diff --git a/unified/extractor/tests/corpus/swift/variables/property-with-willset-and-didset-observers.output b/unified/extractor/tests/corpus/swift/variables/property-with-willset-and-didset-observers.output index 2d612e1d950..8071d5d5290 100644 --- a/unified/extractor/tests/corpus/swift/variables/property-with-willset-and-didset-observers.output +++ b/unified/extractor/tests/corpus/swift/variables/property-with-willset-and-didset-observers.output @@ -7,63 +7,93 @@ class C { --- -source_file - statement: - class_declaration - body: - class_body - member: - property_declaration - binding: - value_binding_pattern - mutability: var - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "x" - observers: - willset_didset_block - didset: - didset_clause - body: - block - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: simple_identifier "oldValue" - willset: - willset_clause - body: - block - statement: - call_expression - function: simple_identifier "print" - suffix: - call_suffix - arguments: - value_arguments - argument: - value_argument - value: simple_identifier "newValue" - type: - type_annotation - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" - value: integer_literal "0" - declaration_kind: class - name: type_identifier "C" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + classDecl + attributes: + name: identifier "C" + memberBlock: + memberBlock + leftBrace: { + rightBrace: } + members: + memberBlockItem + decl: + variableDecl + attributes: + modifiers: + bindingSpecifier: var + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + integerLiteralExpr + literal: integerLiteral "0" + pattern: + identifierPattern + identifier: identifier "x" + typeAnnotation: + typeAnnotation + colon: : + type: + identifierType + name: identifier "Int" + accessorBlock: + accessorBlock + accessors: + accessorDecl + accessorSpecifier: willSet + attributes: + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + declReferenceExpr + baseName: identifier "newValue" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + accessorDecl + accessorSpecifier: didSet + attributes: + body: + codeBlock + leftBrace: { + rightBrace: } + statements: + codeBlockItem + item: + functionCallExpr + leftParen: ( + rightParen: ) + arguments: + labeledExpr + expression: + declReferenceExpr + baseName: identifier "oldValue" + additionalTrailingClosures: + calledExpression: + declReferenceExpr + baseName: identifier "print" + leftBrace: { + rightBrace: } + modifiers: + classKeyword: class --- @@ -72,6 +102,8 @@ top_level block stmt: class_like_declaration + modifier: modifier "class" + name: identifier "C" member: variable_declaration modifier: modifier "var" @@ -83,40 +115,38 @@ top_level name: identifier "Int" value: int_literal "0" accessor_declaration - body: - block - stmt: - call_expr - argument: - argument - value: - name_expr - identifier: identifier "newValue" - callee: - name_expr - identifier: identifier "print" modifier: modifier "var" modifier "chained_declaration" name: identifier "x" accessor_kind: accessor_kind "willSet" - accessor_declaration body: block stmt: call_expr + callee: + name_expr + identifier: identifier "print" argument: argument value: name_expr - identifier: identifier "oldValue" - callee: - name_expr - identifier: identifier "print" + identifier: identifier "newValue" + accessor_declaration modifier: modifier "var" modifier "chained_declaration" name: identifier "x" accessor_kind: accessor_kind "didSet" - modifier: modifier "class" - name: identifier "C" + body: + block + stmt: + call_expr + callee: + name_expr + identifier: identifier "print" + argument: + argument + value: + name_expr + identifier: identifier "oldValue" diff --git a/unified/extractor/tests/corpus/swift/variables/tuple-destructuring-binding.output b/unified/extractor/tests/corpus/swift/variables/tuple-destructuring-binding.output index 5709911171c..6b6bd81115e 100644 --- a/unified/extractor/tests/corpus/swift/variables/tuple-destructuring-binding.output +++ b/unified/extractor/tests/corpus/swift/variables/tuple-destructuring-binding.output @@ -2,28 +2,37 @@ let (a, b) = pair --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: let - declarator: - property_binding - name: - pattern - kind: - tuple_pattern - item: - tuple_pattern_item +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: let + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + declReferenceExpr + baseName: identifier "pair" + pattern: + tuplePattern + leftParen: ( + rightParen: ) + elements: + tuplePatternElement + trailingComma: , pattern: - pattern - kind: simple_identifier "a" - tuple_pattern_item + identifierPattern + identifier: identifier "a" + tuplePatternElement pattern: - pattern - kind: simple_identifier "b" - value: simple_identifier "pair" + identifierPattern + identifier: identifier "b" --- diff --git a/unified/extractor/tests/corpus/swift/variables/var-binding.output b/unified/extractor/tests/corpus/swift/variables/var-binding.output index b7dc8e77270..63498105dc7 100644 --- a/unified/extractor/tests/corpus/swift/variables/var-binding.output +++ b/unified/extractor/tests/corpus/swift/variables/var-binding.output @@ -2,18 +2,26 @@ var x = 1 --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: var - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "x" - value: integer_literal "1" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: var + bindings: + patternBinding + initializer: + initializerClause + equal: = + value: + integerLiteralExpr + literal: integerLiteral "1" + pattern: + identifierPattern + identifier: identifier "x" --- diff --git a/unified/extractor/tests/corpus/swift/variables/var-without-initialiser.output b/unified/extractor/tests/corpus/swift/variables/var-without-initialiser.output index 692befea855..d841ce2bb58 100644 --- a/unified/extractor/tests/corpus/swift/variables/var-without-initialiser.output +++ b/unified/extractor/tests/corpus/swift/variables/var-without-initialiser.output @@ -2,26 +2,26 @@ var x: Int --- -source_file - statement: - property_declaration - binding: - value_binding_pattern - mutability: var - declarator: - property_binding - name: - pattern - bound_identifier: simple_identifier "x" - type: - type_annotation - type: - type - name: - user_type - part: - simple_user_type - name: type_identifier "Int" +sourceFile + endOfFileToken: endOfFile + statements: + codeBlockItem + item: + variableDecl + attributes: + modifiers: + bindingSpecifier: var + bindings: + patternBinding + pattern: + identifierPattern + identifier: identifier "x" + typeAnnotation: + typeAnnotation + colon: : + type: + identifierType + name: identifier "Int" --- diff --git a/unified/extractor/tests/corpus_tests.rs b/unified/extractor/tests/corpus_tests.rs index ac93f06622c..b2700b999b7 100644 --- a/unified/extractor/tests/corpus_tests.rs +++ b/unified/extractor/tests/corpus_tests.rs @@ -1,8 +1,8 @@ use std::fs; use std::path::Path; -use codeql_extractor::extractor::simple; -use yeast::{Runner, dump::dump_ast, dump::dump_ast_with_type_errors}; +use codeql_extractor::extractor::desugaring; +use yeast::{dump::dump_ast, dump::dump_ast_with_type_errors}; #[path = "../src/languages/mod.rs"] mod languages; @@ -20,6 +20,20 @@ fn update_mode_enabled() -> bool { .unwrap_or(false) } +/// Whether the external swift-syntax parser is available. When the parser +/// binary genuinely cannot be found/launched (e.g. no Swift toolchain, and +/// neither `CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE` nor a `swift-syntax-parse` +/// on `PATH`), the corpus test is skipped rather than failed — it cannot run +/// without the Swift-backed parser. +/// +/// Crucially this checks only that the executable *launches*: a parser that is +/// present but crashes, emits invalid JSON, or otherwise regresses is +/// considered available, so the suite runs and fails (rather than silently +/// skipping the very failures CI needs to catch). +fn parser_available() -> bool { + languages::swift_parse::binary_available() +} + /// Parse a corpus `.output` file. The file holds a single test case made of /// three sections separated by `---` delimiter lines: /// @@ -28,7 +42,7 @@ fn update_mode_enabled() -> bool { /// /// --- /// -/// +/// /// /// --- /// @@ -59,40 +73,21 @@ fn render_corpus(case: &CorpusCase) -> String { ) } -fn run_desugaring(lang: &simple::LanguageSpec, input: &str) -> Result { - match lang.desugar.as_deref() { - Some(desugarer) => { - // Parse the input ourselves so we don't depend on the desugarer - // knowing about the language. - let mut parser = tree_sitter::Parser::new(); - parser - .set_language(&lang.ts_language) - .map_err(|e| format!("Failed to set language: {e}"))?; - let tree = parser - .parse(input, None) - .ok_or_else(|| "Failed to parse input".to_string())?; - desugarer - .run_from_tree(&tree, input.as_bytes()) - .map_err(|e| format!("Desugaring failed: {e}")) - } - None => { - let runner: Runner = Runner::new(lang.ts_language.clone(), &[]); - runner - .run(input) - .map_err(|e| format!("Failed to parse input: {e}")) - } - } +/// Parse `input` through the language's parser and desugar it, returning the +/// mapped AST. +fn run_desugaring(lang: &desugaring::LanguageSpec, input: &str) -> Result { + let parsed = (lang.parser)(input.as_bytes())?; + lang.desugarer + .run_from_ast(parsed.ast) + .map_err(|e| format!("Desugaring failed: {e}")) } -/// Produce the raw tree-sitter parse tree dump for `input`, with no -/// desugaring rules applied. Uses a `Runner` with an empty phase list and -/// the input grammar's own schema. -fn dump_raw_parse(lang: &simple::LanguageSpec, input: &str) -> Result { - let runner: Runner = Runner::new(lang.ts_language.clone(), &[]); - let ast = runner - .run(input) - .map_err(|e| format!("Failed to parse input: {e}"))?; - Ok(dump_ast(&ast, ast.get_root(), input)) +/// Produce the raw (pre-desugar) parse tree dump for `input` — the `yeast::Ast` +/// the language's parser builds, before any desugaring rules. Useful for seeing +/// what the mapping rules operate on. +fn dump_raw_parse(lang: &desugaring::LanguageSpec, input: &str) -> Result { + let parsed = (lang.parser)(input.as_bytes())?; + Ok(dump_ast(&parsed.ast, parsed.ast.get_root(), input)) } /// Collect the set of corpus test "stems" (paths without an extension) under @@ -117,16 +112,21 @@ fn collect_corpus_stems(dir: &Path, out: &mut Vec) { #[test] fn test_corpus() { + if !parser_available() { + eprintln!( + "skipping test_corpus: the swift-syntax parser is unavailable \ + (set CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE or put \ + `swift-syntax-parse` on PATH)" + ); + return; + } let update_mode = update_mode_enabled(); let all_languages = languages::all_language_specs(); let corpus_dir = Path::new("tests/corpus"); for lang in all_languages { - let output_schema = yeast::node_types_yaml::schema_from_yaml_with_language( - languages::OUTPUT_AST_SCHEMA, - &lang.ts_language, - ) - .expect("Failed to parse OUTPUT_AST_SCHEMA YAML"); + let output_schema = yeast::node_types_yaml::schema_from_yaml(languages::OUTPUT_AST_SCHEMA) + .expect("Failed to parse OUTPUT_AST_SCHEMA YAML"); let lang_corpus_dir = corpus_dir.join(&lang.prefix); if !lang_corpus_dir.exists() { @@ -236,8 +236,7 @@ fn test_corpus() { ); if update_mode { case.expected = actual_dump.trim().to_string(); - } else if output_path.exists() - && case.expected.trim() != actual_dump.trim() + } else if output_path.exists() && case.expected.trim() != actual_dump.trim() { failures.push(format!( "Test failed in {}:\nEXPECTED:\n\n{}\n\nACTUAL:\n\n{}", diff --git a/unified/extractor/tests/swift_syntax_pipeline.rs b/unified/extractor/tests/swift_syntax_pipeline.rs index cdaae1f47c6..fde060f9820 100644 --- a/unified/extractor/tests/swift_syntax_pipeline.rs +++ b/unified/extractor/tests/swift_syntax_pipeline.rs @@ -22,7 +22,7 @@ fn swift_syntax_json_runs_through_the_desugarer() { .into_iter() .find(|l| l.file_globs.iter().any(|g| g.contains("swift"))) .expect("swift language spec"); - let desugarer = lang.desugar.as_deref().expect("swift desugarer"); + let desugarer = lang.desugarer.as_ref(); // Adapt the swift-syntax JSON into a yeast AST (pure Rust, no Swift FFI). let adapted = diff --git a/unified/ql/lib/codeql/unified/Ast.qll b/unified/ql/lib/codeql/unified/Ast.qll index 8adb4c2e44a..e9a827269cc 100644 --- a/unified/ql/lib/codeql/unified/Ast.qll +++ b/unified/ql/lib/codeql/unified/Ast.qll @@ -567,6 +567,8 @@ module Unified { final override AstNode getAFieldOrChild() { unified_expr_equality_pattern_def(this, result) } } + final class ExprOrOperator extends @unified_expr_or_operator, AstNodeImpl { } + final class ExprOrPattern extends @unified_expr_or_pattern, AstNodeImpl { } final class ExprOrType extends @unified_expr_or_type, AstNodeImpl { } @@ -1407,6 +1409,22 @@ module Unified { } } + /** A class representing `unresolved_operator_sequence` nodes. */ + final class UnresolvedOperatorSequence extends @unified_unresolved_operator_sequence, AstNodeImpl { + /** Gets the name of the primary QL class for this element. */ + final override string getAPrimaryQlClass() { result = "UnresolvedOperatorSequence" } + + /** Gets the node corresponding to the field `element`. */ + final ExprOrOperator getElement(int i) { + unified_unresolved_operator_sequence_element(this, i, result) + } + + /** Gets a field or child node of this node. */ + final override AstNode getAFieldOrChild() { + unified_unresolved_operator_sequence_element(this, _, result) + } + } + /** A class representing `unsupported_node` tokens. */ final class UnsupportedNode extends @unified_token_unsupported_node, TokenImpl { /** Gets the name of the primary QL class for this element. */ @@ -1773,6 +1791,8 @@ module Unified { or result = node.(UnaryExpr).getOperator() and i = -1 and name = "getOperator" or + result = node.(UnresolvedOperatorSequence).getElement(i) and name = "getElement" + or result = node.(VariableDeclaration).getModifier(i) and name = "getModifier" or result = node.(VariableDeclaration).getPattern() and i = -1 and name = "getPattern" diff --git a/unified/ql/lib/unified.dbscheme b/unified/ql/lib/unified.dbscheme index e957e303c22..3aafb2a494f 100644 --- a/unified/ql/lib/unified.dbscheme +++ b/unified/ql/lib/unified.dbscheme @@ -452,13 +452,15 @@ unified_equality_type_constraint_def( int right: @unified_type_expr ref ); -@unified_expr = @unified_array_literal | @unified_assign_expr | @unified_binary_expr | @unified_block | @unified_break_expr | @unified_call_expr | @unified_compound_assign_expr | @unified_continue_expr | @unified_function_expr | @unified_if_expr | @unified_key_value_pair | @unified_map_literal | @unified_member_access_expr | @unified_name_expr | @unified_pattern_guard_expr | @unified_return_expr | @unified_switch_expr | @unified_throw_expr | @unified_token_boolean_literal | @unified_token_builtin_expr | @unified_token_empty_expr | @unified_token_float_literal | @unified_token_int_literal | @unified_token_regex_literal | @unified_token_string_literal | @unified_token_super_expr | @unified_token_unsupported_node | @unified_try_expr | @unified_tuple_expr | @unified_type_cast_expr | @unified_type_test_expr | @unified_unary_expr +@unified_expr = @unified_array_literal | @unified_assign_expr | @unified_binary_expr | @unified_block | @unified_break_expr | @unified_call_expr | @unified_compound_assign_expr | @unified_continue_expr | @unified_function_expr | @unified_if_expr | @unified_key_value_pair | @unified_map_literal | @unified_member_access_expr | @unified_name_expr | @unified_pattern_guard_expr | @unified_return_expr | @unified_switch_expr | @unified_throw_expr | @unified_token_boolean_literal | @unified_token_builtin_expr | @unified_token_empty_expr | @unified_token_float_literal | @unified_token_int_literal | @unified_token_regex_literal | @unified_token_string_literal | @unified_token_super_expr | @unified_token_unsupported_node | @unified_try_expr | @unified_tuple_expr | @unified_type_cast_expr | @unified_type_test_expr | @unified_unary_expr | @unified_unresolved_operator_sequence unified_expr_equality_pattern_def( unique int id: @unified_expr_equality_pattern, int expr: @unified_expr ref ); +@unified_expr_or_operator = @unified_expr | @unified_token_infix_operator + @unified_expr_or_pattern = @unified_expr | @unified_pattern @unified_expr_or_type = @unified_expr | @unified_type_expr @@ -999,6 +1001,17 @@ unified_unary_expr_def( int operator: @unified_operator ref ); +#keyset[unified_unresolved_operator_sequence, index] +unified_unresolved_operator_sequence_element( + int unified_unresolved_operator_sequence: @unified_unresolved_operator_sequence ref, + int index: int ref, + unique int element: @unified_expr_or_operator ref +); + +unified_unresolved_operator_sequence_def( + unique int id: @unified_unresolved_operator_sequence +); + #keyset[unified_variable_declaration, index] unified_variable_declaration_modifier( int unified_variable_declaration: @unified_variable_declaration ref, @@ -1072,7 +1085,7 @@ unified_trivia_tokeninfo( string value: string ref ); -@unified_ast_node = @unified_accessor_declaration | @unified_argument | @unified_array_literal | @unified_assign_expr | @unified_associated_type_declaration | @unified_base_type | @unified_binary_expr | @unified_block | @unified_bound_type_constraint | @unified_break_expr | @unified_bulk_importing_pattern | @unified_call_expr | @unified_catch_clause | @unified_class_like_declaration | @unified_compound_assign_expr | @unified_constructor_declaration | @unified_constructor_pattern | @unified_continue_expr | @unified_destructor_declaration | @unified_do_while_stmt | @unified_equality_type_constraint | @unified_expr_equality_pattern | @unified_for_each_stmt | @unified_function_declaration | @unified_function_expr | @unified_function_type_expr | @unified_generic_type_expr | @unified_guard_if_stmt | @unified_if_expr | @unified_import_declaration | @unified_initializer_declaration | @unified_key_value_pair | @unified_labeled_stmt | @unified_map_literal | @unified_member_access_expr | @unified_name_expr | @unified_name_pattern | @unified_named_type_expr | @unified_operator_syntax_declaration | @unified_or_pattern | @unified_parameter | @unified_pattern_element | @unified_pattern_guard_expr | @unified_return_expr | @unified_switch_case | @unified_switch_expr | @unified_throw_expr | @unified_token | @unified_top_level | @unified_trivia_token | @unified_try_expr | @unified_tuple_expr | @unified_tuple_pattern | @unified_tuple_type_element | @unified_tuple_type_expr | @unified_type_alias_declaration | @unified_type_cast_expr | @unified_type_parameter | @unified_type_test_expr | @unified_type_test_pattern | @unified_unary_expr | @unified_variable_declaration | @unified_while_stmt +@unified_ast_node = @unified_accessor_declaration | @unified_argument | @unified_array_literal | @unified_assign_expr | @unified_associated_type_declaration | @unified_base_type | @unified_binary_expr | @unified_block | @unified_bound_type_constraint | @unified_break_expr | @unified_bulk_importing_pattern | @unified_call_expr | @unified_catch_clause | @unified_class_like_declaration | @unified_compound_assign_expr | @unified_constructor_declaration | @unified_constructor_pattern | @unified_continue_expr | @unified_destructor_declaration | @unified_do_while_stmt | @unified_equality_type_constraint | @unified_expr_equality_pattern | @unified_for_each_stmt | @unified_function_declaration | @unified_function_expr | @unified_function_type_expr | @unified_generic_type_expr | @unified_guard_if_stmt | @unified_if_expr | @unified_import_declaration | @unified_initializer_declaration | @unified_key_value_pair | @unified_labeled_stmt | @unified_map_literal | @unified_member_access_expr | @unified_name_expr | @unified_name_pattern | @unified_named_type_expr | @unified_operator_syntax_declaration | @unified_or_pattern | @unified_parameter | @unified_pattern_element | @unified_pattern_guard_expr | @unified_return_expr | @unified_switch_case | @unified_switch_expr | @unified_throw_expr | @unified_token | @unified_top_level | @unified_trivia_token | @unified_try_expr | @unified_tuple_expr | @unified_tuple_pattern | @unified_tuple_type_element | @unified_tuple_type_expr | @unified_type_alias_declaration | @unified_type_cast_expr | @unified_type_parameter | @unified_type_test_expr | @unified_type_test_pattern | @unified_unary_expr | @unified_unresolved_operator_sequence | @unified_variable_declaration | @unified_while_stmt unified_ast_node_location( unique int node: @unified_ast_node ref, diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index 5c2360d4deb..db451e5b7b9 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -1,4 +1,6 @@ load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test") +load("@rules_shell//shell:sh_binary.bzl", "sh_binary") +load("//misc/bazel:pkg.bzl", "codeql_pkg_runfiles") load(":swift_runtime.bzl", "swift_runtime_libs") load(":xcode_transition.bzl", "xcode_transition_swift_library") @@ -27,6 +29,7 @@ xcode_transition_swift_library( module_name = "SwiftSyntaxFFI", target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, deps = [ + "@swift-syntax//:SwiftOperators", "@swift-syntax//:SwiftParser", "@swift-syntax//:SwiftSyntax", ], @@ -47,13 +50,19 @@ rust_library( ], ) +# The Swift front-end parser. We ship it like `//swift/extractor`: a small shell +# wrapper (`swift-syntax-parse`) sets `LD_LIBRARY_PATH`/`DYLD_LIBRARY_PATH` to its +# own directory and execs the real binary (`swift-syntax-parse.real`); the Swift +# runtime shared libraries are packaged alongside them. `parse.rs` resolves the +# wrapper as a sibling of the extractor executable. rust_binary( - name = "swift-syntax-parse", + name = "swift-syntax-parse.real", srcs = ["src/main.rs"], - # `rust_binary` doesn't copy the Swift runtime into runfiles the way - # `swift_binary` does. On Linux, ship the standalone toolchain's runtime; - # on macOS the OS provides it at `/usr/lib/swift` (rpath'd by - # `xcode_swift_toolchain`). + # Target name carries `.real` (invalid in a crate name), so set it explicitly. + crate_name = "swift_syntax_parse", + # On Linux, carry the toolchain's runtime shared libraries as runfiles so + # they get packaged next to the binary. On macOS the OS provides the Swift + # runtime, so nothing extra is bundled. data = select({ "@platforms//os:macos": [], "@platforms//os:linux": [":swift_runtime_libs"], @@ -63,6 +72,28 @@ rust_binary( deps = [":swift_syntax_rs"], ) +# `swift-syntax-parse` wrapper (see `swift-syntax-parse.sh`). Its runfiles carry +# the real binary and the runtime libraries; packaging flattens them into one +# directory. +sh_binary( + name = "swift-syntax-parse", + srcs = ["swift-syntax-parse.sh"], + data = [":swift-syntax-parse.real"], + target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, +) + +# Packaged form for the extractor pack: the wrapper (as `swift-syntax-parse`), +# the real binary, and the runtime libraries, flattened into one directory. +codeql_pkg_runfiles( + name = "swift-syntax-parse-pkg", + # The `.sh` source is shipped as `swift-syntax-parse` (the wrapper); drop the + # original filename. + excludes = ["swift-syntax-parse.sh"], + exes = [":swift-syntax-parse"], + target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, + visibility = ["//unified:__pkg__"], +) + rust_test( name = "swift_syntax_rs_test", size = "small", diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index d43969199f9..7d15c6f2c23 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -91,6 +91,37 @@ arrays (their collection nodes are elided), layout children such as comment rides along as `trailingTrivia` on the token it follows. Tokens without trivia (most of them) simply omit the `leadingTrivia`/`trailingTrivia` keys. +### Operator folding + +Swift's grammar does not encode operator precedence, so the parser represents an +expression like `a + b * c` as a flat `sequenceExpr` (an alternating list of +operands and operators). Before serializing, we fold these sequences into +precedence-correct `infixOperatorExpr` (and `ternaryExpr`) trees — so `1 + 2 * 3` +becomes `1 + (2 * 3)`. + +Folding needs to know each operator's precedence group, which comes from +declarations rather than the grammar. We resolve operators from two sources: + +- the **Swift standard library** operators (a built-in approximation), and +- operator / precedence-group declarations **in the file being parsed**. + +Operators defined anywhere else (for example, imported from another module) are +unknown, so their precedence cannot be determined. Rather than guess — which +would silently produce a wrongly-structured tree — each top-level sequence is +folded independently, and any sequence that uses an unknown operator is left as +a flat `sequenceExpr`. So `a <+> b` (with an undeclared `<+>`) stays flat, while +a neighbouring `1 + 2` in the same file still folds. Supporting operators from +other modules is future work. + +Folding is bottom-up, so a *grouped* subexpression still folds even when the +sequence enclosing it uses an unknown operator: in `a *** (b + c)` (with an +unknown `***`) the parenthesised `b + c` is its own sequence and folds, while +the outer `a *** …` stays flat. This only applies when the subexpression is +syntactically isolated (parentheses, call arguments, collection elements, …); +an unparenthesised `a *** b + c` is a single flat sequence whose structure +cannot be determined without knowing `***`'s precedence, so it is left flat in +its entirety. + ## Prerequisites The build does not depend on any particular version manager. You need: diff --git a/unified/swift-syntax-rs/build.rs b/unified/swift-syntax-rs/build.rs index c599cdf4e70..6a58ddb4d8d 100644 --- a/unified/swift-syntax-rs/build.rs +++ b/unified/swift-syntax-rs/build.rs @@ -24,20 +24,34 @@ fn main() { println!("cargo:rerun-if-env-changed=SWIFTC"); // Build the Swift FFI package as a release dynamic library. + // + // Degrade gracefully when there is no runnable Swift toolchain. This crate + // is a workspace member, so a plain `cargo check`/`fmt`/`clippy` at the repo + // root runs this build script; if `swift build` cannot even be spawned we + // emit a warning and skip the link directives rather than panicking, so + // those Swift-free workflows keep working. Only `cargo build`/`cargo test` + // then fail — at link time, which is fair: they genuinely need Swift (and CI + // builds go through Bazel anyway). A Swift toolchain that *is* present but + // whose build fails is still surfaced as a hard error below. let mut command = Command::new(swift_bin()); command .args(["build", "-c", "release"]) .current_dir(&swift_dir); apply_bare_repository_workaround(&mut command); - let status = command.status().unwrap_or_else(|e| { - panic!( - "failed to run `{swift} build`: {e}\n\ - Install a Swift toolchain (see https://www.swift.org/install/, e.g. via \ - swiftly) and ensure `swift` is on PATH, or set the `SWIFT` environment \ - variable to the `swift` executable. The pinned version is in `.swift-version`.", - swift = swift_bin(), - ) - }); + let status = match command.status() { + Ok(status) => status, + Err(e) => { + println!( + "cargo:warning=skipping the Swift FFI build: failed to run `{swift} build`: {e}. \ + Install a Swift toolchain (see https://www.swift.org/install/, e.g. via swiftly) \ + and ensure `swift` is on PATH, or set the `SWIFT` environment variable, to build \ + or test this crate. `cargo check`/`fmt`/`clippy` work without it. The pinned \ + version is in `.swift-version`.", + swift = swift_bin(), + ); + return; + } + }; assert!(status.success(), "`swift build` failed"); // Link against the freshly built dynamic library. diff --git a/unified/swift-syntax-rs/src/lib.rs b/unified/swift-syntax-rs/src/lib.rs index 2a8fe0411af..5f558d942fc 100644 --- a/unified/swift-syntax-rs/src/lib.rs +++ b/unified/swift-syntax-rs/src/lib.rs @@ -132,4 +132,87 @@ mod tests { "comment trivia not captured: {json}" ); } + + #[test] + fn folds_standard_library_operators() { + // Standard-library operators are folded into a precedence-correct tree: + // the flat `sequenceExpr` becomes nested `infixOperatorExpr` nodes. + let json = parse_to_json("let x = 1 + 2 * 3").expect("parsing should succeed"); + assert!( + json.contains("\"kind\":\"infixOperatorExpr\""), + "operators were not folded: {json}" + ); + assert!( + !json.contains("\"kind\":\"sequenceExpr\""), + "a foldable sequence was left flat: {json}" + ); + } + + #[test] + fn folds_file_defined_operators() { + // An operator declared in the same file (with a known precedence group) + // is folded, just like a standard-library one. + let json = parse_to_json("infix operator |>: AdditionPrecedence\nlet y = a |> b |> c") + .expect("parsing should succeed"); + assert!( + json.contains("\"kind\":\"infixOperatorExpr\""), + "file-defined operator was not folded: {json}" + ); + assert!( + !json.contains("\"kind\":\"sequenceExpr\""), + "a foldable sequence was left flat: {json}" + ); + } + + #[test] + fn leaves_unknown_operators_unfolded() { + // An operator that is neither in the standard library nor declared in + // this file (e.g. imported from another module) has unknown precedence, + // so its sequence is left flat rather than folded incorrectly. + let json = parse_to_json("let z = a <+> b").expect("parsing should succeed"); + assert!( + json.contains("\"kind\":\"sequenceExpr\""), + "unknown-operator sequence should stay flat: {json}" + ); + assert!( + !json.contains("\"kind\":\"infixOperatorExpr\""), + "unknown operator should not be folded: {json}" + ); + } + + #[test] + fn folds_each_sequence_independently() { + // Folding is isolated per sequence: a statement using a known operator + // folds even when another statement uses an unknown one. One unfoldable + // expression does not suppress folding elsewhere. + let json = parse_to_json("let x = 1 + 2\nlet y = a <+> b").expect("parsing should succeed"); + assert!( + json.contains("\"kind\":\"infixOperatorExpr\""), + "the known-operator statement should fold: {json}" + ); + assert!( + json.contains("\"kind\":\"sequenceExpr\""), + "the unknown-operator statement should stay flat: {json}" + ); + } + + #[test] + fn folds_grouped_subexpressions_under_unknown_operators() { + // A parenthesized (or otherwise bracketed) subexpression is its own + // sequence, so it folds independently even when the enclosing sequence + // uses an unknown operator. Here `***` is unknown (outer stays flat) but + // the grouped `b + c` still folds. Note this only works because the + // parentheses isolate `b + c`: in an unparenthesized `a *** b + c` the + // whole thing is one flat sequence whose structure can't be determined + // without `***`'s precedence, so it is left flat entirely. + let json = parse_to_json("let r = a *** (b + c)").expect("parsing should succeed"); + assert!( + json.contains("\"kind\":\"infixOperatorExpr\""), + "the grouped known-operator subexpression should fold: {json}" + ); + assert!( + json.contains("\"kind\":\"sequenceExpr\""), + "the enclosing unknown-operator sequence should stay flat: {json}" + ); + } } diff --git a/unified/swift-syntax-rs/swift-syntax-parse.sh b/unified/swift-syntax-rs/swift-syntax-parse.sh new file mode 100755 index 00000000000..811697c820c --- /dev/null +++ b/unified/swift-syntax-rs/swift-syntax-parse.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Wrapper that lets the shipped `swift-syntax-parse` find its Swift runtime +# libraries, which are packaged in the same directory as this script (and the +# real binary). Mirrors `swift/extractor/extractor.sh`. +if [[ "$(uname)" == Darwin ]]; then + export DYLD_LIBRARY_PATH=$(dirname "$0") +else + export LD_LIBRARY_PATH=$(dirname "$0") +fi + +exec -a "$0" "$0.real" "$@" diff --git a/unified/swift-syntax-rs/swift/Package.swift b/unified/swift-syntax-rs/swift/Package.swift index a58b7c1a479..37fa5a1eed7 100644 --- a/unified/swift-syntax-rs/swift/Package.swift +++ b/unified/swift-syntax-rs/swift/Package.swift @@ -29,6 +29,7 @@ let package = Package( dependencies: [ .product(name: "SwiftSyntax", package: "swift-syntax"), .product(name: "SwiftParser", package: "swift-syntax"), + .product(name: "SwiftOperators", package: "swift-syntax"), ] ) ] diff --git a/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift b/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift index 6305b1610d0..9471c2ea143 100644 --- a/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift +++ b/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift @@ -1,4 +1,5 @@ import Foundation +import SwiftOperators import SwiftParser // `@_spi(RawSyntax)` exposes the `childName(_:)` helper that maps a child's @@ -151,6 +152,68 @@ private func serialize( return result } +/// Fold the flat operator sequences in `tree` into structured binary/ternary +/// expressions, using operator precedence. +/// +/// Swift's grammar does not encode operator precedence: an expression like +/// `a + b * c` is parsed as a flat `SequenceExpr` (a list of operands and +/// operators). Resolving it into a precedence-correct tree requires knowing the +/// operators' precedence groups, which live in declarations rather than the +/// grammar. We fold using two sources of operator definitions: +/// +/// * `OperatorTable.standardOperators` — the Swift standard library's +/// operators (a built-in approximation), and +/// * the operator / precedence-group declarations in *this* file +/// (via `addSourceFile`). +/// +/// Operators from anywhere else (e.g. imported from another module) are +/// unknown to us. Rather than guess their precedence — which would silently +/// produce an incorrectly-structured tree — we fold each top-level sequence +/// *independently* and leave any sequence containing an unknown operator as a +/// flat `SequenceExpr`. Downstream can treat such a sequence as unsupported. +/// This keeps folding correct for the operators we do know while isolating the +/// rest per sequence, so one exotic operator doesn't block folding elsewhere. +private func foldOperators(in tree: SourceFileSyntax) -> Syntax { + var operators = OperatorTable.standardOperators + // Register operators and precedence groups declared in this file. Swallow + // errors (e.g. a redeclaration of a standard operator): keep whatever we + // can and carry on. + operators.addSourceFile(tree, errorHandler: { _ in }) + return PerSequenceFolder(operators: operators).rewrite(tree) +} + +/// A `SyntaxRewriter` that folds each `SequenceExpr` on its own, leaving +/// sequences it cannot fold (because they use an operator we don't know) flat. +/// +/// This is deliberately more conservative than `OperatorTable.foldAll`, which +/// is all-or-nothing: with a throwing error handler a single unknown operator +/// aborts folding for the *entire* tree, while with a non-throwing handler +/// unknown operators are folded *arbitrarily* (left-associatively), silently +/// producing a wrong tree. Folding each sequence with a throwing handler and +/// catching the error gives us per-sequence isolation instead. +private final class PerSequenceFolder: SyntaxRewriter { + private let operators: OperatorTable + + init(operators: OperatorTable) { + self.operators = operators + super.init() + } + + override func visit(_ node: SequenceExprSyntax) -> ExprSyntax { + // Fold any nested sequences first (bottom-up), so that an unfoldable + // outer sequence still keeps its foldable inner sequences folded. + let inner = super.visit(node).as(SequenceExprSyntax.self)! + do { + // The default error handler throws on the first unknown operator or + // incomparable-precedence pair. + return try operators.foldSingle(inner) + } catch { + // Leave this sequence flat; it uses an operator we don't know. + return ExprSyntax(inner) + } + } +} + /// Parse the given NUL-terminated Swift source string and return a /// heap-allocated, NUL-terminated JSON representation of the syntax tree. /// @@ -161,8 +224,12 @@ public func ssr_parse_json(_ source: UnsafePointer?) -> UnsafeMutablePoin guard let source = source else { return nil } let code = String(cString: source) let tree = Parser.parse(source: code) + // Fold operator sequences before serializing. Source positions are + // preserved by folding (the same tokens, in the same places), so a + // converter built from the original tree maps the folded tree correctly. + let folded = foldOperators(in: tree) let converter = SourceLocationConverter(fileName: "", tree: tree) - let json = serialize(Syntax(tree), converter) + let json = serialize(folded, converter) guard let data = try? JSONSerialization.data(