From 9ffe03bcce97ad81b5790d2e4ac64c5db965e5f7 Mon Sep 17 00:00:00 2001 From: Esben Sparre Andreasen Date: Mon, 25 Nov 2019 13:53:29 +0100 Subject: [PATCH] JS: support additional Flow syntax: `...` in object types --- change-notes/1.23/extractor-javascript.md | 1 + .../src/com/semmle/jcorn/flow/FlowParser.java | 6 +- .../src/com/semmle/js/extractor/Main.java | 2 +- .../tests/flow/input/objectTypeSpread.js | 4 +- .../flow/output/trap/objectTypeSpread.js.trap | 374 ++++++++++++++---- 5 files changed, 303 insertions(+), 84 deletions(-) diff --git a/change-notes/1.23/extractor-javascript.md b/change-notes/1.23/extractor-javascript.md index 8b7a35f5b4f..92e6c80e8cd 100644 --- a/change-notes/1.23/extractor-javascript.md +++ b/change-notes/1.23/extractor-javascript.md @@ -16,6 +16,7 @@ extraction: - include: "**/bower_components" ``` +* Additional [Flow](https://flow.org/) syntax is now supported. * Recognition of CommonJS modules has improved. As a result, some files that were previously extracted as global scripts are now extracted as modules. * Top-level `await` is now supported. diff --git a/javascript/extractor/src/com/semmle/jcorn/flow/FlowParser.java b/javascript/extractor/src/com/semmle/jcorn/flow/FlowParser.java index 65a9dce89a3..fbd428e47df 100644 --- a/javascript/extractor/src/com/semmle/jcorn/flow/FlowParser.java +++ b/javascript/extractor/src/com/semmle/jcorn/flow/FlowParser.java @@ -541,7 +541,11 @@ public class FlowParser extends ESNextParser { if (isStatic && this.type == TokenType.colon) { this.parseIdent(false); } else if (allowSpread && this.eat(TokenType.ellipsis)) { - flowParseType(); + boolean hasType = + this.type != endDelim && this.type != TokenType.comma && this.type != TokenType.semi; + if (hasType) { + flowParseType(); + } flowObjectTypeSemicolon(); continue; } else { diff --git a/javascript/extractor/src/com/semmle/js/extractor/Main.java b/javascript/extractor/src/com/semmle/js/extractor/Main.java index 524cd7524e6..6f2a85e4994 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/Main.java +++ b/javascript/extractor/src/com/semmle/js/extractor/Main.java @@ -37,7 +37,7 @@ public class Main { * A version identifier that should be updated every time the extractor changes in such a way that * it may produce different tuples for the same file under the same {@link ExtractorConfig}. */ - public static final String EXTRACTOR_VERSION = "2019-11-17"; + public static final String EXTRACTOR_VERSION = "2019-11-26"; public static final Pattern NEWLINE = Pattern.compile("\n"); diff --git a/javascript/extractor/tests/flow/input/objectTypeSpread.js b/javascript/extractor/tests/flow/input/objectTypeSpread.js index 955c4c22272..9730c6c49be 100644 --- a/javascript/extractor/tests/flow/input/objectTypeSpread.js +++ b/javascript/extractor/tests/flow/input/objectTypeSpread.js @@ -1,2 +1,4 @@ type A = { x: int }; -type B = { ...A, y: string }; \ No newline at end of file +type B = { ...A, y: string }; +type C = {c: null | T, ...} | ((r: null | T) => mixed); +type D = { d: D, ..., }; \ No newline at end of file diff --git a/javascript/extractor/tests/flow/output/trap/objectTypeSpread.js.trap b/javascript/extractor/tests/flow/output/trap/objectTypeSpread.js.trap index b3a181e0469..f7c948b9ba8 100644 --- a/javascript/extractor/tests/flow/output/trap/objectTypeSpread.js.trap +++ b/javascript/extractor/tests/flow/output/trap/objectTypeSpread.js.trap @@ -16,133 +16,345 @@ lines(#20002,#20001,"type A = { x: int };"," locations_default(#20003,#10000,1,1,1,20) hasLocation(#20002,#20003) #20004=* -lines(#20004,#20001,"type B = { ...A, y: string };","") +lines(#20004,#20001,"type B = { ...A, y: string };"," +") #20005=@"loc,{#10000},2,1,2,29" locations_default(#20005,#10000,2,1,2,29) hasLocation(#20004,#20005) -numlines(#20001,2,2,0) #20006=* -tokeninfo(#20006,6,#20001,0,"type") -#20007=@"loc,{#10000},1,1,1,4" -locations_default(#20007,#10000,1,1,1,4) +lines(#20006,#20001,"type C = {c: null | T, ...} | ((r: null | T) => mixed);"," +") +#20007=@"loc,{#10000},3,1,3,58" +locations_default(#20007,#10000,3,1,3,58) hasLocation(#20006,#20007) #20008=* -tokeninfo(#20008,6,#20001,1,"A") -#20009=@"loc,{#10000},1,6,1,6" -locations_default(#20009,#10000,1,6,1,6) +lines(#20008,#20001,"type D = { d: D, ..., };","") +#20009=@"loc,{#10000},4,1,4,24" +locations_default(#20009,#10000,4,1,4,24) hasLocation(#20008,#20009) +numlines(#20001,4,4,0) #20010=* -tokeninfo(#20010,8,#20001,2,"=") -#20011=@"loc,{#10000},1,8,1,8" -locations_default(#20011,#10000,1,8,1,8) +tokeninfo(#20010,6,#20001,0,"type") +#20011=@"loc,{#10000},1,1,1,4" +locations_default(#20011,#10000,1,1,1,4) hasLocation(#20010,#20011) #20012=* -tokeninfo(#20012,8,#20001,3,"{") -#20013=@"loc,{#10000},1,10,1,10" -locations_default(#20013,#10000,1,10,1,10) +tokeninfo(#20012,6,#20001,1,"A") +#20013=@"loc,{#10000},1,6,1,6" +locations_default(#20013,#10000,1,6,1,6) hasLocation(#20012,#20013) #20014=* -tokeninfo(#20014,6,#20001,4,"x") -#20015=@"loc,{#10000},1,12,1,12" -locations_default(#20015,#10000,1,12,1,12) +tokeninfo(#20014,8,#20001,2,"=") +#20015=@"loc,{#10000},1,8,1,8" +locations_default(#20015,#10000,1,8,1,8) hasLocation(#20014,#20015) #20016=* -tokeninfo(#20016,8,#20001,5,":") -#20017=@"loc,{#10000},1,13,1,13" -locations_default(#20017,#10000,1,13,1,13) +tokeninfo(#20016,8,#20001,3,"{") +#20017=@"loc,{#10000},1,10,1,10" +locations_default(#20017,#10000,1,10,1,10) hasLocation(#20016,#20017) #20018=* -tokeninfo(#20018,6,#20001,6,"int") -#20019=@"loc,{#10000},1,15,1,17" -locations_default(#20019,#10000,1,15,1,17) +tokeninfo(#20018,6,#20001,4,"x") +#20019=@"loc,{#10000},1,12,1,12" +locations_default(#20019,#10000,1,12,1,12) hasLocation(#20018,#20019) #20020=* -tokeninfo(#20020,8,#20001,7,"}") -#20021=@"loc,{#10000},1,19,1,19" -locations_default(#20021,#10000,1,19,1,19) +tokeninfo(#20020,8,#20001,5,":") +#20021=@"loc,{#10000},1,13,1,13" +locations_default(#20021,#10000,1,13,1,13) hasLocation(#20020,#20021) #20022=* -tokeninfo(#20022,8,#20001,8,";") -#20023=@"loc,{#10000},1,20,1,20" -locations_default(#20023,#10000,1,20,1,20) +tokeninfo(#20022,6,#20001,6,"int") +#20023=@"loc,{#10000},1,15,1,17" +locations_default(#20023,#10000,1,15,1,17) hasLocation(#20022,#20023) #20024=* -tokeninfo(#20024,6,#20001,9,"type") -#20025=@"loc,{#10000},2,1,2,4" -locations_default(#20025,#10000,2,1,2,4) +tokeninfo(#20024,8,#20001,7,"}") +#20025=@"loc,{#10000},1,19,1,19" +locations_default(#20025,#10000,1,19,1,19) hasLocation(#20024,#20025) #20026=* -tokeninfo(#20026,6,#20001,10,"B") -#20027=@"loc,{#10000},2,6,2,6" -locations_default(#20027,#10000,2,6,2,6) +tokeninfo(#20026,8,#20001,8,";") +#20027=@"loc,{#10000},1,20,1,20" +locations_default(#20027,#10000,1,20,1,20) hasLocation(#20026,#20027) #20028=* -tokeninfo(#20028,8,#20001,11,"=") -#20029=@"loc,{#10000},2,8,2,8" -locations_default(#20029,#10000,2,8,2,8) +tokeninfo(#20028,6,#20001,9,"type") +#20029=@"loc,{#10000},2,1,2,4" +locations_default(#20029,#10000,2,1,2,4) hasLocation(#20028,#20029) #20030=* -tokeninfo(#20030,8,#20001,12,"{") -#20031=@"loc,{#10000},2,10,2,10" -locations_default(#20031,#10000,2,10,2,10) +tokeninfo(#20030,6,#20001,10,"B") +#20031=@"loc,{#10000},2,6,2,6" +locations_default(#20031,#10000,2,6,2,6) hasLocation(#20030,#20031) #20032=* -tokeninfo(#20032,8,#20001,13,"...") -#20033=@"loc,{#10000},2,12,2,14" -locations_default(#20033,#10000,2,12,2,14) +tokeninfo(#20032,8,#20001,11,"=") +#20033=@"loc,{#10000},2,8,2,8" +locations_default(#20033,#10000,2,8,2,8) hasLocation(#20032,#20033) #20034=* -tokeninfo(#20034,6,#20001,14,"A") -#20035=@"loc,{#10000},2,15,2,15" -locations_default(#20035,#10000,2,15,2,15) +tokeninfo(#20034,8,#20001,12,"{") +#20035=@"loc,{#10000},2,10,2,10" +locations_default(#20035,#10000,2,10,2,10) hasLocation(#20034,#20035) #20036=* -tokeninfo(#20036,8,#20001,15,",") -#20037=@"loc,{#10000},2,16,2,16" -locations_default(#20037,#10000,2,16,2,16) +tokeninfo(#20036,8,#20001,13,"...") +#20037=@"loc,{#10000},2,12,2,14" +locations_default(#20037,#10000,2,12,2,14) hasLocation(#20036,#20037) #20038=* -tokeninfo(#20038,6,#20001,16,"y") -#20039=@"loc,{#10000},2,18,2,18" -locations_default(#20039,#10000,2,18,2,18) +tokeninfo(#20038,6,#20001,14,"A") +#20039=@"loc,{#10000},2,15,2,15" +locations_default(#20039,#10000,2,15,2,15) hasLocation(#20038,#20039) #20040=* -tokeninfo(#20040,8,#20001,17,":") -#20041=@"loc,{#10000},2,19,2,19" -locations_default(#20041,#10000,2,19,2,19) +tokeninfo(#20040,8,#20001,15,",") +#20041=@"loc,{#10000},2,16,2,16" +locations_default(#20041,#10000,2,16,2,16) hasLocation(#20040,#20041) #20042=* -tokeninfo(#20042,6,#20001,18,"string") -#20043=@"loc,{#10000},2,21,2,26" -locations_default(#20043,#10000,2,21,2,26) +tokeninfo(#20042,6,#20001,16,"y") +#20043=@"loc,{#10000},2,18,2,18" +locations_default(#20043,#10000,2,18,2,18) hasLocation(#20042,#20043) #20044=* -tokeninfo(#20044,8,#20001,19,"}") -#20045=@"loc,{#10000},2,28,2,28" -locations_default(#20045,#10000,2,28,2,28) +tokeninfo(#20044,8,#20001,17,":") +#20045=@"loc,{#10000},2,19,2,19" +locations_default(#20045,#10000,2,19,2,19) hasLocation(#20044,#20045) #20046=* -tokeninfo(#20046,8,#20001,20,";") -#20047=@"loc,{#10000},2,29,2,29" -locations_default(#20047,#10000,2,29,2,29) +tokeninfo(#20046,6,#20001,18,"string") +#20047=@"loc,{#10000},2,21,2,26" +locations_default(#20047,#10000,2,21,2,26) hasLocation(#20046,#20047) #20048=* -tokeninfo(#20048,0,#20001,21,"") -#20049=@"loc,{#10000},2,30,2,29" -locations_default(#20049,#10000,2,30,2,29) +tokeninfo(#20048,8,#20001,19,"}") +#20049=@"loc,{#10000},2,28,2,28" +locations_default(#20049,#10000,2,28,2,28) hasLocation(#20048,#20049) +#20050=* +tokeninfo(#20050,8,#20001,20,";") +#20051=@"loc,{#10000},2,29,2,29" +locations_default(#20051,#10000,2,29,2,29) +hasLocation(#20050,#20051) +#20052=* +tokeninfo(#20052,6,#20001,21,"type") +#20053=@"loc,{#10000},3,1,3,4" +locations_default(#20053,#10000,3,1,3,4) +hasLocation(#20052,#20053) +#20054=* +tokeninfo(#20054,6,#20001,22,"C") +#20055=@"loc,{#10000},3,6,3,6" +locations_default(#20055,#10000,3,6,3,6) +hasLocation(#20054,#20055) +#20056=* +tokeninfo(#20056,8,#20001,23,"<") +#20057=@"loc,{#10000},3,7,3,7" +locations_default(#20057,#10000,3,7,3,7) +hasLocation(#20056,#20057) +#20058=* +tokeninfo(#20058,6,#20001,24,"T") +#20059=@"loc,{#10000},3,8,3,8" +locations_default(#20059,#10000,3,8,3,8) +hasLocation(#20058,#20059) +#20060=* +tokeninfo(#20060,8,#20001,25,">") +#20061=@"loc,{#10000},3,9,3,9" +locations_default(#20061,#10000,3,9,3,9) +hasLocation(#20060,#20061) +#20062=* +tokeninfo(#20062,8,#20001,26,"=") +#20063=@"loc,{#10000},3,11,3,11" +locations_default(#20063,#10000,3,11,3,11) +hasLocation(#20062,#20063) +#20064=* +tokeninfo(#20064,8,#20001,27,"{") +#20065=@"loc,{#10000},3,13,3,13" +locations_default(#20065,#10000,3,13,3,13) +hasLocation(#20064,#20065) +#20066=* +tokeninfo(#20066,6,#20001,28,"c") +#20067=@"loc,{#10000},3,14,3,14" +locations_default(#20067,#10000,3,14,3,14) +hasLocation(#20066,#20067) +#20068=* +tokeninfo(#20068,8,#20001,29,":") +#20069=@"loc,{#10000},3,15,3,15" +locations_default(#20069,#10000,3,15,3,15) +hasLocation(#20068,#20069) +#20070=* +tokeninfo(#20070,1,#20001,30,"null") +#20071=@"loc,{#10000},3,17,3,20" +locations_default(#20071,#10000,3,17,3,20) +hasLocation(#20070,#20071) +#20072=* +tokeninfo(#20072,8,#20001,31,"|") +#20073=@"loc,{#10000},3,22,3,22" +locations_default(#20073,#10000,3,22,3,22) +hasLocation(#20072,#20073) +#20074=* +tokeninfo(#20074,6,#20001,32,"T") +#20075=@"loc,{#10000},3,24,3,24" +locations_default(#20075,#10000,3,24,3,24) +hasLocation(#20074,#20075) +#20076=* +tokeninfo(#20076,8,#20001,33,",") +#20077=@"loc,{#10000},3,25,3,25" +locations_default(#20077,#10000,3,25,3,25) +hasLocation(#20076,#20077) +#20078=* +tokeninfo(#20078,8,#20001,34,"...") +#20079=@"loc,{#10000},3,27,3,29" +locations_default(#20079,#10000,3,27,3,29) +hasLocation(#20078,#20079) +#20080=* +tokeninfo(#20080,8,#20001,35,"}") +#20081=@"loc,{#10000},3,30,3,30" +locations_default(#20081,#10000,3,30,3,30) +hasLocation(#20080,#20081) +#20082=* +tokeninfo(#20082,8,#20001,36,"|") +#20083=@"loc,{#10000},3,32,3,32" +locations_default(#20083,#10000,3,32,3,32) +hasLocation(#20082,#20083) +#20084=* +tokeninfo(#20084,8,#20001,37,"(") +#20085=@"loc,{#10000},3,34,3,34" +locations_default(#20085,#10000,3,34,3,34) +hasLocation(#20084,#20085) +#20086=* +tokeninfo(#20086,8,#20001,38,"(") +#20087=@"loc,{#10000},3,35,3,35" +locations_default(#20087,#10000,3,35,3,35) +hasLocation(#20086,#20087) +#20088=* +tokeninfo(#20088,6,#20001,39,"r") +#20089=@"loc,{#10000},3,36,3,36" +locations_default(#20089,#10000,3,36,3,36) +hasLocation(#20088,#20089) +#20090=* +tokeninfo(#20090,8,#20001,40,":") +#20091=@"loc,{#10000},3,37,3,37" +locations_default(#20091,#10000,3,37,3,37) +hasLocation(#20090,#20091) +#20092=* +tokeninfo(#20092,1,#20001,41,"null") +#20093=@"loc,{#10000},3,39,3,42" +locations_default(#20093,#10000,3,39,3,42) +hasLocation(#20092,#20093) +#20094=* +tokeninfo(#20094,8,#20001,42,"|") +#20095=@"loc,{#10000},3,44,3,44" +locations_default(#20095,#10000,3,44,3,44) +hasLocation(#20094,#20095) +#20096=* +tokeninfo(#20096,6,#20001,43,"T") +#20097=@"loc,{#10000},3,46,3,46" +locations_default(#20097,#10000,3,46,3,46) +hasLocation(#20096,#20097) +#20098=* +tokeninfo(#20098,8,#20001,44,")") +#20099=@"loc,{#10000},3,47,3,47" +locations_default(#20099,#10000,3,47,3,47) +hasLocation(#20098,#20099) +#20100=* +tokeninfo(#20100,8,#20001,45,"=>") +#20101=@"loc,{#10000},3,49,3,50" +locations_default(#20101,#10000,3,49,3,50) +hasLocation(#20100,#20101) +#20102=* +tokeninfo(#20102,6,#20001,46,"mixed") +#20103=@"loc,{#10000},3,52,3,56" +locations_default(#20103,#10000,3,52,3,56) +hasLocation(#20102,#20103) +#20104=* +tokeninfo(#20104,8,#20001,47,")") +#20105=@"loc,{#10000},3,57,3,57" +locations_default(#20105,#10000,3,57,3,57) +hasLocation(#20104,#20105) +#20106=* +tokeninfo(#20106,8,#20001,48,";") +#20107=@"loc,{#10000},3,58,3,58" +locations_default(#20107,#10000,3,58,3,58) +hasLocation(#20106,#20107) +#20108=* +tokeninfo(#20108,6,#20001,49,"type") +#20109=@"loc,{#10000},4,1,4,4" +locations_default(#20109,#10000,4,1,4,4) +hasLocation(#20108,#20109) +#20110=* +tokeninfo(#20110,6,#20001,50,"D") +#20111=@"loc,{#10000},4,6,4,6" +locations_default(#20111,#10000,4,6,4,6) +hasLocation(#20110,#20111) +#20112=* +tokeninfo(#20112,8,#20001,51,"=") +#20113=@"loc,{#10000},4,8,4,8" +locations_default(#20113,#10000,4,8,4,8) +hasLocation(#20112,#20113) +#20114=* +tokeninfo(#20114,8,#20001,52,"{") +#20115=@"loc,{#10000},4,10,4,10" +locations_default(#20115,#10000,4,10,4,10) +hasLocation(#20114,#20115) +#20116=* +tokeninfo(#20116,6,#20001,53,"d") +#20117=@"loc,{#10000},4,12,4,12" +locations_default(#20117,#10000,4,12,4,12) +hasLocation(#20116,#20117) +#20118=* +tokeninfo(#20118,8,#20001,54,":") +#20119=@"loc,{#10000},4,13,4,13" +locations_default(#20119,#10000,4,13,4,13) +hasLocation(#20118,#20119) +#20120=* +tokeninfo(#20120,6,#20001,55,"D") +#20121=@"loc,{#10000},4,15,4,15" +locations_default(#20121,#10000,4,15,4,15) +hasLocation(#20120,#20121) +#20122=* +tokeninfo(#20122,8,#20001,56,",") +#20123=@"loc,{#10000},4,16,4,16" +locations_default(#20123,#10000,4,16,4,16) +hasLocation(#20122,#20123) +#20124=* +tokeninfo(#20124,8,#20001,57,"...") +#20125=@"loc,{#10000},4,18,4,20" +locations_default(#20125,#10000,4,18,4,20) +hasLocation(#20124,#20125) +#20126=* +tokeninfo(#20126,8,#20001,58,",") +#20127=@"loc,{#10000},4,21,4,21" +locations_default(#20127,#10000,4,21,4,21) +hasLocation(#20126,#20127) +#20128=* +tokeninfo(#20128,8,#20001,59,"}") +#20129=@"loc,{#10000},4,23,4,23" +locations_default(#20129,#10000,4,23,4,23) +hasLocation(#20128,#20129) +#20130=* +tokeninfo(#20130,8,#20001,60,";") +#20131=@"loc,{#10000},4,24,4,24" +locations_default(#20131,#10000,4,24,4,24) +hasLocation(#20130,#20131) +#20132=* +tokeninfo(#20132,0,#20001,61,"") +#20133=@"loc,{#10000},4,25,4,24" +locations_default(#20133,#10000,4,25,4,24) +hasLocation(#20132,#20133) toplevels(#20001,0) -#20050=@"loc,{#10000},1,1,2,29" -locations_default(#20050,#10000,1,1,2,29) -hasLocation(#20001,#20050) -#20051=* -entry_cfg_node(#20051,#20001) -#20052=@"loc,{#10000},1,1,1,0" -locations_default(#20052,#10000,1,1,1,0) -hasLocation(#20051,#20052) -#20053=* -exit_cfg_node(#20053,#20001) -hasLocation(#20053,#20049) -successor(#20051,#20053) -numlines(#10000,2,2,0) +#20134=@"loc,{#10000},1,1,4,24" +locations_default(#20134,#10000,1,1,4,24) +hasLocation(#20001,#20134) +#20135=* +entry_cfg_node(#20135,#20001) +#20136=@"loc,{#10000},1,1,1,0" +locations_default(#20136,#10000,1,1,1,0) +hasLocation(#20135,#20136) +#20137=* +exit_cfg_node(#20137,#20001) +hasLocation(#20137,#20133) +successor(#20135,#20137) +numlines(#10000,4,4,0) filetype(#10000,"javascript")