From 2ab72c4eef7b9d36ea174737a9da8e91c15b6494 Mon Sep 17 00:00:00 2001 From: Asger F Date: Mon, 1 Jul 2019 11:15:03 +0100 Subject: [PATCH] JS: Support line breaks in types --- .../src/com/semmle/js/parser/JSDocParser.java | 54 +++++++++++-------- .../test/library-tests/JSDoc/tests.expected | 9 ++++ javascript/ql/test/library-tests/JSDoc/tst.js | 6 +++ 3 files changed, 48 insertions(+), 21 deletions(-) diff --git a/javascript/extractor/src/com/semmle/js/parser/JSDocParser.java b/javascript/extractor/src/com/semmle/js/parser/JSDocParser.java index bc17c4cf8bd..97a4c222af3 100644 --- a/javascript/extractor/src/com/semmle/js/parser/JSDocParser.java +++ b/javascript/extractor/src/com/semmle/js/parser/JSDocParser.java @@ -109,6 +109,10 @@ public class JSDocParser { return Character.isWhitespace(ch) && !isLineTerminator(ch) || ch == '\u00a0'; } + private static boolean isWhiteSpaceOrLineTerminator(char ch) { + return Character.isWhitespace(ch) || ch == '\u00a0'; + } + private static boolean isDecimalDigit(char ch) { return "0123456789".indexOf(ch) >= 0; } @@ -300,7 +304,14 @@ public class JSDocParser { private int advance() { if (index >= source.length()) return -1; - return source.charAt(index++); + int ch = source.charAt(index); + ++index; + if (isLineTerminator(ch) && !(ch == '\r' && index < endIndex && source.charAt(index) == '\n')) { + lineNumber += 1; + lineStart = index; + index = skipStars(index, endIndex); + } + return ch; } private String scanHexEscape(char prefix) { @@ -571,7 +582,7 @@ public class JSDocParser { endOfPrevToken = index; - while (index < endIndex && isWhiteSpace(source.charAt(index))) { + while (index < endIndex && isWhiteSpaceOrLineTerminator(source.charAt(index))) { advance(); } if (index >= endIndex) { @@ -1211,36 +1222,37 @@ public class JSDocParser { } } + /** Skips the leading indentation and '*' at the beginning of a line. */ + private int skipStars(int index, int end) { + while (index < end + && isWhiteSpace(source.charAt(index)) + && !isLineTerminator(source.charAt(index))) { + index += 1; + } + while (index < end && source.charAt(index) == '*') { + index += 1; + } + while (index < end + && isWhiteSpace(source.charAt(index)) + && !isLineTerminator(source.charAt(index))) { + index += 1; + } + return index; + } + private TypeExpressionParser typed = new TypeExpressionParser(); private class JSDocTagParser { int index, lineNumber, lineStart, length; boolean recoverable = true, sloppy = false; - private int skipStars(int index) { - while (index < length - && isWhiteSpace(source.charAt(index)) - && !isLineTerminator(source.charAt(index))) { - index += 1; - } - while (index < length && source.charAt(index) == '*') { - index += 1; - } - while (index < length - && isWhiteSpace(source.charAt(index)) - && !isLineTerminator(source.charAt(index))) { - index += 1; - } - return index; - } - private char advance() { char ch = source.charAt(index); index += 1; if (isLineTerminator(ch) && !(ch == '\r' && index < length && source.charAt(index) == '\n')) { lineNumber += 1; lineStart = index; - index = skipStars(index); + index = skipStars(index, length); } return ch; } @@ -1268,7 +1280,7 @@ public class JSDocParser { && !(ch == '\r' && last + 1 < length && source.charAt(last + 1) == '\n')) { lineNumber += 1; lineStart = last + 1; - last = skipStars(last + 1) - 1; + last = skipStars(last + 1, length) - 1; waiting = true; } else if (waiting) { if (ch == '@') { diff --git a/javascript/ql/test/library-tests/JSDoc/tests.expected b/javascript/ql/test/library-tests/JSDoc/tests.expected index 2eb5c0485ef..0a4a82ef1f3 100644 --- a/javascript/ql/test/library-tests/JSDoc/tests.expected +++ b/javascript/ql/test/library-tests/JSDoc/tests.expected @@ -121,6 +121,7 @@ test_JSDoc | tst.js:368:3:370:5 | /**\\n ... p\\n */ | | tst.js:368:3:370:5 | /**\\n ... p\\n */ | | tst.js:375:3:377:5 | /**\\n ... p\\n */ | | tst.js:375:3:377:5 | /**\\n ... p\\n */ | | tst.js:380:3:382:5 | /**\\n ... p\\n */ | | tst.js:380:3:382:5 | /**\\n ... p\\n */ | +| tst.js:386:1:389:3 | /**\\n * ... } x\\n */ | | tst.js:386:1:389:3 | /**\\n * ... } x\\n */ | test_JSDocTag | tst.js:5:5:5:10 | @const | const | tst.js:5:1:5:13 | /** @const */ | 0 | (none) | (none) | (none) | | tst.js:9:4:9:9 | @const | const | tst.js:7:1:11:3 | /**\\n * ... ng}\\n */ | 0 | (none) | (none) | (none) | @@ -228,6 +229,7 @@ test_JSDocTag | tst.js:369:6:369:11 | @param | param | tst.js:368:3:370:5 | /**\\n ... p\\n */ | 0 | (none) | p | T2 | | tst.js:376:6:376:11 | @param | param | tst.js:375:3:377:5 | /**\\n ... p\\n */ | 0 | (none) | p | T3 | | tst.js:381:6:381:11 | @param | param | tst.js:380:3:382:5 | /**\\n ... p\\n */ | 0 | (none) | p | T4 | +| tst.js:387:4:387:9 | @param | param | tst.js:386:1:389:3 | /**\\n * ... } x\\n */ | 0 | (none) | x | Array. | test_ObjectExpr_getDocumentation | tst.js:48:25:48:36 | { 'x': 321 } | tst.js:48:12:48:23 | /** @dict */ | | tst.js:55:20:59:1 | {\\n TRU ... BE: 0\\n} | tst.js:51:1:54:3 | /**\\n * ... er}\\n */ | @@ -339,6 +341,7 @@ test_next_token | tst.js:368:3:370:5 | /**\\n ... p\\n */ | tst.js:371:3:371:13 | fancyMethod | | tst.js:375:3:377:5 | /**\\n ... p\\n */ | tst.js:378:3:378:13 | constructor | | tst.js:380:3:382:5 | /**\\n ... p\\n */ | tst.js:383:3:383:13 | classMethod | +| tst.js:386:1:389:3 | /**\\n * ... } x\\n */ | tst.js:390:1:390:8 | function | test_JSDocTypeExpr | tst.js:10:11:10:16 | string | tst.js:10:4:10:8 | @type | 0 | | tst.js:23:14:23:20 | boolean | tst.js:23:5:23:11 | @define | 0 | @@ -461,6 +464,9 @@ test_JSDocTypeExpr | tst.js:369:14:369:15 | T2 | tst.js:369:6:369:11 | @param | 0 | | tst.js:376:14:376:15 | T3 | tst.js:376:6:376:11 | @param | 0 | | tst.js:381:14:381:15 | T4 | tst.js:381:6:381:11 | @param | 0 | +| tst.js:387:12:387:16 | Array | tst.js:387:12:388:13 | Array. | -1 | +| tst.js:387:12:388:13 | Array. | tst.js:387:4:387:9 | @param | 0 | +| tst.js:388:7:388:12 | number | tst.js:387:12:388:13 | Array. | 0 | test_Function_getDocumentation | tst.js:20:1:21:1 | functio ... t() {\\n} | tst.js:16:1:19:3 | /**\\n * ... tor\\n */ | | tst.js:36:34:37:1 | function(node) {\\n} | tst.js:29:1:35:3 | /**\\n * ... ().\\n */ | @@ -505,6 +511,7 @@ test_Function_getDocumentation | tst.js:371:14:371:19 | (p) {} | tst.js:368:3:370:5 | /**\\n ... p\\n */ | | tst.js:378:14:378:19 | (p) {} | tst.js:375:3:377:5 | /**\\n ... p\\n */ | | tst.js:383:14:383:19 | (p) {} | tst.js:380:3:382:5 | /**\\n ... p\\n */ | +| tst.js:390:1:390:24 | functio ... e(x) {} | tst.js:386:1:389:3 | /**\\n * ... } x\\n */ | test_JSDocOptionalParameterTypeExpr | tst.js:239:12:239:18 | number= | tst.js:239:12:239:17 | number | | tst.js:240:21:240:28 | ?string= | tst.js:240:21:240:27 | ?string | @@ -525,6 +532,7 @@ test_getParameterTag | tst.js:371:15:371:15 | p | p | tst.js:369:6:369:11 | @param | p | tst.js:369:14:369:15 | T2 | | tst.js:378:15:378:15 | p | p | tst.js:376:6:376:11 | @param | p | tst.js:376:14:376:15 | T3 | | tst.js:383:15:383:15 | p | p | tst.js:381:6:381:11 | @param | p | tst.js:381:14:381:15 | T4 | +| tst.js:390:20:390:20 | x | x | tst.js:387:4:387:9 | @param | x | tst.js:387:12:388:13 | Array. | test_VarDeclStmt_getDocumentation | tst.js:5:15:5:36 | var MY_ ... stout'; | tst.js:5:1:5:13 | /** @const */ | | tst.js:24:1:24:24 | var ENA ... = true; | tst.js:23:1:23:24 | /** @de ... ean} */ | @@ -557,6 +565,7 @@ test_JSDocAppliedTypeExpr | tst.js:333:17:333:28 | Foo. | tst.js:333:17:333:19 | Foo | 0 | tst.js:333:22:333:27 | string | | tst.js:334:17:334:28 | Foo. | tst.js:334:17:334:19 | Foo | 0 | tst.js:334:22:334:27 | number | | tst.js:357:12:357:25 | Array. | tst.js:357:12:357:16 | Array | 0 | tst.js:357:19:357:24 | number | +| tst.js:387:12:388:13 | Array. | tst.js:387:12:387:16 | Array | 0 | tst.js:388:7:388:12 | number | test_JSDocNonNullableTypeExpr | tst.js:229:12:229:18 | !Object | tst.js:229:13:229:18 | Object | prefix | | tst.js:258:12:258:24 | !Foo. | tst.js:258:13:258:24 | Foo. | prefix | diff --git a/javascript/ql/test/library-tests/JSDoc/tst.js b/javascript/ql/test/library-tests/JSDoc/tst.js index 0a540dd960c..15599b1f444 100644 --- a/javascript/ql/test/library-tests/JSDoc/tst.js +++ b/javascript/ql/test/library-tests/JSDoc/tst.js @@ -382,3 +382,9 @@ class C { */ classMethod(p) {} } + +/** + * @param {Array.< + * number>} x + */ +function multiline(x) {}