From e6b5040909e19cdce25cbc6eefb7a45d13b467ea Mon Sep 17 00:00:00 2001 From: Asger F Date: Wed, 5 Feb 2025 13:17:08 +0100 Subject: [PATCH 1/3] Actions: add test with many quoted strings --- .../.github/workflows/many_strings.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 actions/ql/test/library-tests/.github/workflows/many_strings.yml diff --git a/actions/ql/test/library-tests/.github/workflows/many_strings.yml b/actions/ql/test/library-tests/.github/workflows/many_strings.yml new file mode 100644 index 00000000000..f5fe1ce7b83 --- /dev/null +++ b/actions/ql/test/library-tests/.github/workflows/many_strings.yml @@ -0,0 +1,18 @@ +on: + workflow_run: + workflows: ["Prev"] + types: + - completed + +jobs: + Test: + runs-on: ubuntu-latest + steps: + - run: | + # Avoid choking on large chunks of data containing quotes + echo '["string1", "string2", "string3", "string4", "string5", "string6", "string7", "string8", "string9", "string10", "string11", "string12", "string13", "string14", "string15", "string16", "string17", "string18", "string19", "string20", "string21", "string22", "string23", "string24", "string25", "string26", "string27", "string28", "string29", "string30", "string31", "string32", "string33", "string34", "string35", "string36", "string37", "string38", "string39", "string40", "string41", "string42", "string43", "string44", "string45", "string46", "string47", "string48", "string49", "string50", "string51", "string52", "string53", "string54", "string55", "string56", "string57", "string58", "string59", "string60", "string61", "string62", "string63", "string64", "string65", "string66", "string67", "string68", "string69", "string70", "string71", "string72", "string73", "string74", "string75", "string76", "string77", "string78", "string79", "string80", "string81", "string82", "string83", "string84", "string85", "string86", "string87", "string88", "string89", "string90", "string91", "string92", "string93", "string94", "string95", "string96", "string97", "string98", "string99", "string100"]' + echo "['string1', 'string2', 'string3', 'string4', 'string5', 'string6', 'string7', 'string8', 'string9', 'string10', 'string11', 'string12', 'string13', 'string14', 'string15', 'string16', 'string17', 'string18', 'string19', 'string20', 'string21', 'string22', 'string23', 'string24', 'string25', 'string26', 'string27', 'string28', 'string29', 'string30', 'string31', 'string32', 'string33', 'string34', 'string35', 'string36', 'string37', 'string38', 'string39', 'string40', 'string41', 'string42', 'string43', 'string44', 'string45', 'string46', 'string47', 'string48', 'string49', 'string50', 'string51', 'string52', 'string53', 'string54', 'string55', 'string56', 'string57', 'string58', 'string59', 'string60', 'string61', 'string62', 'string63', 'string64', 'string65', 'string66', 'string67', 'string68', 'string69', 'string70', 'string71', 'string72', 'string73', 'string74', 'string75', 'string76', 'string77', 'string78', 'string79', 'string80', 'string81', 'string82', 'string83', 'string84', 'string85', 'string86', 'string87', 'string88', 'string89', 'string90', 'string91', 'string92', 'string93', 'string94', 'string95', 'string96', 'string97', 'string98', 'string99', 'string100']" + + # Same as above but where each line has an unbalanced internal quote near the end + echo '["string1", "string2", "string3", "string4", "string5", "string6", "string7", "string8", "string9", "string10", "string11", "string12", "string13", "string14", "string15", "string16", "string17", "string18", "string19", "string20", "string21", "string22", "string23", "string24", "string25", "string26", "string27", "string28", "string29", "string30", "string31", "string32", "string33", "string34", "string35", "string36", "string37", "string38", "string39", "string40", "string41", "string42", "string43", "string44", "string45", "string46", "string47", "string48", "string49", "string50", "string51", "string52", "string53", "string54", "string55", "string56", "string57", "string58", "string59", "string60", "string61", "string62", "string63", "string64", "string65", "string66", "string67", "string68", "string69", "string70", "string71", "string72", "string73", "string74", "string75", "string76", "string77", "string78", "string79", "string80", "string81", "string82", "string83", "string84", "string85", "string86", "string87", "string88", "string89", "string90", "string91", "string92", "string93", "string94", "string95", "string96", "string97", "string98", "string99", "string100"]"' + echo "['string1', 'string2', 'string3', 'string4', 'string5', 'string6', 'string7', 'string8', 'string9', 'string10', 'string11', 'string12', 'string13', 'string14', 'string15', 'string16', 'string17', 'string18', 'string19', 'string20', 'string21', 'string22', 'string23', 'string24', 'string25', 'string26', 'string27', 'string28', 'string29', 'string30', 'string31', 'string32', 'string33', 'string34', 'string35', 'string36', 'string37', 'string38', 'string39', 'string40', 'string41', 'string42', 'string43', 'string44', 'string45', 'string46', 'string47', 'string48', 'string49', 'string50', 'string51', 'string52', 'string53', 'string54', 'string55', 'string56', 'string57', 'string58', 'string59', 'string60', 'string61', 'string62', 'string63', 'string64', 'string65', 'string66', 'string67', 'string68', 'string69', 'string70', 'string71', 'string72', 'string73', 'string74', 'string75', 'string76', 'string77', 'string78', 'string79', 'string80', 'string81', 'string82', 'string83', 'string84', 'string85', 'string86', 'string87', 'string88', 'string89', 'string90', 'string91', 'string92', 'string93', 'string94', 'string95', 'string96', 'string97', 'string98', 'string99', 'string100']'" From 1904b026b2bf62b441b3809fa4ec0263f46c6b61 Mon Sep 17 00:00:00 2001 From: Asger F Date: Wed, 5 Feb 2025 13:16:47 +0100 Subject: [PATCH 2/3] Actions: Avoid blowup in quotation parser The parser has an inherent N^2 blowup and will need a rewrite eventually. For now I'm just trying to make it not blow up as often. --- actions/ql/lib/codeql/actions/Bash.qll | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/actions/ql/lib/codeql/actions/Bash.qll b/actions/ql/lib/codeql/actions/Bash.qll index d6be3c8e2d0..4519a8949d7 100644 --- a/actions/ql/lib/codeql/actions/Bash.qll +++ b/actions/ql/lib/codeql/actions/Bash.qll @@ -81,7 +81,9 @@ class BashShellScript extends ShellScript { "qstr:" + k + ":" + i + ":" + j + ":" + quotedStr.length() + ":" + quotedStr.regexpReplaceAll("[^a-zA-Z0-9]", "") ) - ) + ) and + // Only do this for strings that might otherwise disrupt subsequent parsing + quotedStr.regexpMatch("[\"'].*[$\n\r'\"" + Bash::separator() + "].*[\"']") } private predicate rankedQuotedStringReplacements(int i, string old, string new) { From 4ec84e932746e872e0d8608bc7a887d44a55b6fa Mon Sep 17 00:00:00 2001 From: Asger F Date: Wed, 5 Feb 2025 13:36:38 +0100 Subject: [PATCH 3/3] Actions: update expected output --- .../ql/test/library-tests/commands.expected | 4 +++ actions/ql/test/library-tests/test.expected | 35 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/actions/ql/test/library-tests/commands.expected b/actions/ql/test/library-tests/commands.expected index 35305671cf0..82ea1739b28 100644 --- a/actions/ql/test/library-tests/commands.expected +++ b/actions/ql/test/library-tests/commands.expected @@ -25,6 +25,10 @@ | .github/workflows/expression_nodes.yml:16:9:20:6 | Run Step | LINE 2 echo '${{github.event.issue.body}}' | | .github/workflows/expression_nodes.yml:16:9:20:6 | Run Step | LINE 3 echo '${{ github.event.comment.body }}' | | .github/workflows/expression_nodes.yml:20:9:21:47 | Run Step | LINE 1 echo '${{ github.event.comment.body }}' echo '${{github.event.issue.body}}' | +| .github/workflows/many_strings.yml:11:9:18:1211 | Run Step | echo "['string1', 'string2', 'string3', 'string4', 'string5', 'string6', 'string7', 'string8', 'string9', 'string10', 'string11', 'string12', 'string13', 'string14', 'string15', 'string16', 'string17', 'string18', 'string19', 'string20', 'string21', 'string22', 'string23', 'string24', 'string25', 'string26', 'string27', 'string28', 'string29', 'string30', 'string31', 'string32', 'string33', 'string34', 'string35', 'string36', 'string37', 'string38', 'string39', 'string40', 'string41', 'string42', 'string43', 'string44', 'string45', 'string46', 'string47', 'string48', 'string49', 'string50', 'string51', 'string52', 'string53', 'string54', 'string55', 'string56', 'string57', 'string58', 'string59', 'string60', 'string61', 'string62', 'string63', 'string64', 'string65', 'string66', 'string67', 'string68', 'string69', 'string70', 'string71', 'string72', 'string73', 'string74', 'string75', 'string76', 'string77', 'string78', 'string79', 'string80', 'string81', 'string82', 'string83', 'string84', 'string85', 'string86', 'string87', 'string88', 'string89', 'string90', 'string91', 'string92', 'string93', 'string94', 'string95', 'string96', 'string97', 'string98', 'string99', 'string100']" | +| .github/workflows/many_strings.yml:11:9:18:1211 | Run Step | echo "['string1', 'string2', 'string3', 'string4', 'string5', 'string6', 'string7', 'string8', 'string9', 'string10', 'string11', 'string12', 'string13', 'string14', 'string15', 'string16', 'string17', 'string18', 'string19', 'string20', 'string21', 'string22', 'string23', 'string24', 'string25', 'string26', 'string27', 'string28', 'string29', 'string30', 'string31', 'string32', 'string33', 'string34', 'string35', 'string36', 'string37', 'string38', 'string39', 'string40', 'string41', 'string42', 'string43', 'string44', 'string45', 'string46', 'string47', 'string48', 'string49', 'string50', 'string51', 'string52', 'string53', 'string54', 'string55', 'string56', 'string57', 'string58', 'string59', 'string60', 'string61', 'string62', 'string63', 'string64', 'string65', 'string66', 'string67', 'string68', 'string69', 'string70', 'string71', 'string72', 'string73', 'string74', 'string75', 'string76', 'string77', 'string78', 'string79', 'string80', 'string81', 'string82', 'string83', 'string84', 'string85', 'string86', 'string87', 'string88', 'string89', 'string90', 'string91', 'string92', 'string93', 'string94', 'string95', 'string96', 'string97', 'string98', 'string99', 'string100']'" | +| .github/workflows/many_strings.yml:11:9:18:1211 | Run Step | echo '["string1", "string2", "string3", "string4", "string5", "string6", "string7", "string8", "string9", "string10", "string11", "string12", "string13", "string14", "string15", "string16", "string17", "string18", "string19", "string20", "string21", "string22", "string23", "string24", "string25", "string26", "string27", "string28", "string29", "string30", "string31", "string32", "string33", "string34", "string35", "string36", "string37", "string38", "string39", "string40", "string41", "string42", "string43", "string44", "string45", "string46", "string47", "string48", "string49", "string50", "string51", "string52", "string53", "string54", "string55", "string56", "string57", "string58", "string59", "string60", "string61", "string62", "string63", "string64", "string65", "string66", "string67", "string68", "string69", "string70", "string71", "string72", "string73", "string74", "string75", "string76", "string77", "string78", "string79", "string80", "string81", "string82", "string83", "string84", "string85", "string86", "string87", "string88", "string89", "string90", "string91", "string92", "string93", "string94", "string95", "string96", "string97", "string98", "string99", "string100"]"' | +| .github/workflows/many_strings.yml:11:9:18:1211 | Run Step | echo '["string1", "string2", "string3", "string4", "string5", "string6", "string7", "string8", "string9", "string10", "string11", "string12", "string13", "string14", "string15", "string16", "string17", "string18", "string19", "string20", "string21", "string22", "string23", "string24", "string25", "string26", "string27", "string28", "string29", "string30", "string31", "string32", "string33", "string34", "string35", "string36", "string37", "string38", "string39", "string40", "string41", "string42", "string43", "string44", "string45", "string46", "string47", "string48", "string49", "string50", "string51", "string52", "string53", "string54", "string55", "string56", "string57", "string58", "string59", "string60", "string61", "string62", "string63", "string64", "string65", "string66", "string67", "string68", "string69", "string70", "string71", "string72", "string73", "string74", "string75", "string76", "string77", "string78", "string79", "string80", "string81", "string82", "string83", "string84", "string85", "string86", "string87", "string88", "string89", "string90", "string91", "string92", "string93", "string94", "string95", "string96", "string97", "string98", "string99", "string100"]' | | .github/workflows/multiline2.yml:11:9:15:6 | Run Step | echo "CHANGELOGEOF" | | .github/workflows/multiline2.yml:11:9:15:6 | Run Step | echo "changelog< | | .github/workflows/multiline2.yml:11:9:15:6 | Run Step | echo -e "$FILTERED_CHANGELOG" | diff --git a/actions/ql/test/library-tests/test.expected b/actions/ql/test/library-tests/test.expected index a8cf50334ce..4de3336aba7 100644 --- a/actions/ql/test/library-tests/test.expected +++ b/actions/ql/test/library-tests/test.expected @@ -1,6 +1,7 @@ files | .github/workflows/commands.yml:0:0:0:0 | .github/workflows/commands.yml | | .github/workflows/expression_nodes.yml:0:0:0:0 | .github/workflows/expression_nodes.yml | +| .github/workflows/many_strings.yml:0:0:0:0 | .github/workflows/many_strings.yml | | .github/workflows/multiline2.yml:0:0:0:0 | .github/workflows/multiline2.yml | | .github/workflows/multiline.yml:0:0:0:0 | .github/workflows/multiline.yml | | .github/workflows/poisonable_steps.yml:0:0:0:0 | .github/workflows/poisonable_steps.yml | @@ -9,6 +10,7 @@ files workflows | .github/workflows/commands.yml:1:1:39:30 | on: push | | .github/workflows/expression_nodes.yml:1:1:21:47 | on: issue_comment | +| .github/workflows/many_strings.yml:1:1:18:1211 | on: | | .github/workflows/multiline2.yml:1:1:89:35 | on: | | .github/workflows/multiline.yml:1:1:89:29 | on: | | .github/workflows/poisonable_steps.yml:1:1:46:111 | on: push | @@ -20,6 +22,7 @@ jobs | .github/workflows/commands.yml:9:5:31:2 | Job: local_commands | | .github/workflows/commands.yml:32:5:39:30 | Job: local_commands2 | | .github/workflows/expression_nodes.yml:5:5:21:47 | Job: echo-chamber | +| .github/workflows/many_strings.yml:9:5:18:1211 | Job: Test | | .github/workflows/multiline2.yml:9:5:89:35 | Job: Test | | .github/workflows/multiline.yml:9:5:89:29 | Job: Test | | .github/workflows/poisonable_steps.yml:5:5:46:111 | Job: local_commands | @@ -33,6 +36,7 @@ localJobs | .github/workflows/commands.yml:9:5:31:2 | Job: local_commands | | .github/workflows/commands.yml:32:5:39:30 | Job: local_commands2 | | .github/workflows/expression_nodes.yml:5:5:21:47 | Job: echo-chamber | +| .github/workflows/many_strings.yml:9:5:18:1211 | Job: Test | | .github/workflows/multiline2.yml:9:5:89:35 | Job: Test | | .github/workflows/multiline.yml:9:5:89:29 | Job: Test | | .github/workflows/poisonable_steps.yml:5:5:46:111 | Job: local_commands | @@ -59,6 +63,7 @@ steps | .github/workflows/expression_nodes.yml:13:9:16:6 | Run Step | | .github/workflows/expression_nodes.yml:16:9:20:6 | Run Step | | .github/workflows/expression_nodes.yml:20:9:21:47 | Run Step | +| .github/workflows/many_strings.yml:11:9:18:1211 | Run Step | | .github/workflows/multiline2.yml:11:9:15:6 | Run Step | | .github/workflows/multiline2.yml:15:9:20:6 | Run Step | | .github/workflows/multiline2.yml:20:9:24:6 | Run Step | @@ -181,6 +186,7 @@ runStepChildren | .github/workflows/expression_nodes.yml:13:9:16:6 | Run Step | .github/workflows/expression_nodes.yml:13:14:15:46 | LINE 1 echo '${{ github.event.comment.body }}' echo '${{github.event.issue.body}}'\n | | .github/workflows/expression_nodes.yml:16:9:20:6 | Run Step | .github/workflows/expression_nodes.yml:16:14:19:57 | LINE 1 echo '${{ github.event.comment.body }}'\nLINE 2 echo '${{github.event.issue.body}}'\nLINE 3 echo '${{ github.event.comment.body }}'\n | | .github/workflows/expression_nodes.yml:20:9:21:47 | Run Step | .github/workflows/expression_nodes.yml:20:14:21:46 | LINE 1 echo '${{ github.event.comment.body }}' echo '${{github.event.issue.body}}' | +| .github/workflows/many_strings.yml:11:9:18:1211 | Run Step | .github/workflows/many_strings.yml:11:14:18:1211 | # Avoid choking on large chunks of data containing quotes\necho '["string1", "string2", "string3", "string4", "string5", "string6", "string7", "string8", "string9", "string10", "string11", "string12", "string13", "string14", "string15", "string16", "string17", "string18", "string19", "string20", "string21", "string22", "string23", "string24", "string25", "string26", "string27", "string28", "string29", "string30", "string31", "string32", "string33", "string34", "string35", "string36", "string37", "string38", "string39", "string40", "string41", "string42", "string43", "string44", "string45", "string46", "string47", "string48", "string49", "string50", "string51", "string52", "string53", "string54", "string55", "string56", "string57", "string58", "string59", "string60", "string61", "string62", "string63", "string64", "string65", "string66", "string67", "string68", "string69", "string70", "string71", "string72", "string73", "string74", "string75", "string76", "string77", "string78", "string79", "string80", "string81", "string82", "string83", "string84", "string85", "string86", "string87", "string88", "string89", "string90", "string91", "string92", "string93", "string94", "string95", "string96", "string97", "string98", "string99", "string100"]'\necho "['string1', 'string2', 'string3', 'string4', 'string5', 'string6', 'string7', 'string8', 'string9', 'string10', 'string11', 'string12', 'string13', 'string14', 'string15', 'string16', 'string17', 'string18', 'string19', 'string20', 'string21', 'string22', 'string23', 'string24', 'string25', 'string26', 'string27', 'string28', 'string29', 'string30', 'string31', 'string32', 'string33', 'string34', 'string35', 'string36', 'string37', 'string38', 'string39', 'string40', 'string41', 'string42', 'string43', 'string44', 'string45', 'string46', 'string47', 'string48', 'string49', 'string50', 'string51', 'string52', 'string53', 'string54', 'string55', 'string56', 'string57', 'string58', 'string59', 'string60', 'string61', 'string62', 'string63', 'string64', 'string65', 'string66', 'string67', 'string68', 'string69', 'string70', 'string71', 'string72', 'string73', 'string74', 'string75', 'string76', 'string77', 'string78', 'string79', 'string80', 'string81', 'string82', 'string83', 'string84', 'string85', 'string86', 'string87', 'string88', 'string89', 'string90', 'string91', 'string92', 'string93', 'string94', 'string95', 'string96', 'string97', 'string98', 'string99', 'string100']"\n\n# Same as above but where each line has an unbalanced internal quote near the end\necho '["string1", "string2", "string3", "string4", "string5", "string6", "string7", "string8", "string9", "string10", "string11", "string12", "string13", "string14", "string15", "string16", "string17", "string18", "string19", "string20", "string21", "string22", "string23", "string24", "string25", "string26", "string27", "string28", "string29", "string30", "string31", "string32", "string33", "string34", "string35", "string36", "string37", "string38", "string39", "string40", "string41", "string42", "string43", "string44", "string45", "string46", "string47", "string48", "string49", "string50", "string51", "string52", "string53", "string54", "string55", "string56", "string57", "string58", "string59", "string60", "string61", "string62", "string63", "string64", "string65", "string66", "string67", "string68", "string69", "string70", "string71", "string72", "string73", "string74", "string75", "string76", "string77", "string78", "string79", "string80", "string81", "string82", "string83", "string84", "string85", "string86", "string87", "string88", "string89", "string90", "string91", "string92", "string93", "string94", "string95", "string96", "string97", "string98", "string99", "string100"]"'\necho "['string1', 'string2', 'string3', 'string4', 'string5', 'string6', 'string7', 'string8', 'string9', 'string10', 'string11', 'string12', 'string13', 'string14', 'string15', 'string16', 'string17', 'string18', 'string19', 'string20', 'string21', 'string22', 'string23', 'string24', 'string25', 'string26', 'string27', 'string28', 'string29', 'string30', 'string31', 'string32', 'string33', 'string34', 'string35', 'string36', 'string37', 'string38', 'string39', 'string40', 'string41', 'string42', 'string43', 'string44', 'string45', 'string46', 'string47', 'string48', 'string49', 'string50', 'string51', 'string52', 'string53', 'string54', 'string55', 'string56', 'string57', 'string58', 'string59', 'string60', 'string61', 'string62', 'string63', 'string64', 'string65', 'string66', 'string67', 'string68', 'string69', 'string70', 'string71', 'string72', 'string73', 'string74', 'string75', 'string76', 'string77', 'string78', 'string79', 'string80', 'string81', 'string82', 'string83', 'string84', 'string85', 'string86', 'string87', 'string88', 'string89', 'string90', 'string91', 'string92', 'string93', 'string94', 'string95', 'string96', 'string97', 'string98', 'string99', 'string100']'"\n | | .github/workflows/multiline2.yml:11:9:15:6 | Run Step | .github/workflows/multiline2.yml:11:14:14:54 | echo "changelog<