Merge remote-tracking branch 'upstream/main' into post-release-prep/codeql-cli-2.13.3

javascript/ql/lib/change-notes/2023-04-30-npm-submodule.md

@@ -0,0 +1,5 @@
+---
+category: minorAnalysis
+---
+
+- Added a support of sub modules in `node_modules`.

javascript/ql/lib/semmle/javascript/NPM.qll

@@ -12,8 +12,26 @@ class PackageJson extends JsonObject {
     this.isTopLevel()
   }
 
-  /** Gets the name of this package. */
-  string getPackageName() { result = this.getPropStringValue("name") }
+  /**
+   * Gets the name of this package.
+   * If the package is located under the package `pkg1` and its relative path is `foo/bar`, then the resulting package name will be `pkg1/foo/bar`.
+   */
+  string getPackageName() {
+    result = this.getPropStringValue("name")
+    or
+    exists(
+      PackageJson parentPkg, Container currentDir, Container parentDir, string parentPkgName,
+      string pkgNameDiff
+    |
+      currentDir = this.getJsonFile().getParentContainer() and
+      parentDir = parentPkg.getJsonFile().getParentContainer() and
+      parentPkgName = parentPkg.getPropStringValue("name") and
+      parentDir.getAChildContainer+() = currentDir and
+      pkgNameDiff = currentDir.getAbsolutePath().suffix(parentDir.getAbsolutePath().length()) and
+      not exists(pkgNameDiff.indexOf("/node_modules/")) and
+      result = parentPkgName + pkgNameDiff
+    )
+  }
 
   /** Gets the version of this package. */
   string getVersion() { result = this.getPropStringValue("version") }

javascript/ql/lib/semmlecode.javascript.dbscheme

@@ -1,103 +1,75 @@
 /*** Standard fragments ***/
 
-/** Files and folders **/
+/*- Files and folders -*/
 
-@location = @location_default;
+/**
+ * The location of an element.
+ * The location spans column `startcolumn` of line `startline` to
+ * column `endcolumn` of line `endline` in file `file`.
+ * For more information, see
+ * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
+ */
+locations_default(
+  unique int id: @location_default,
+  int file: @file ref,
+  int beginLine: int ref,
+  int beginColumn: int ref,
+  int endLine: int ref,
+  int endColumn: int ref
+);
 
-locations_default(unique int id: @location_default,
-          int file: @file ref,
-          int beginLine: int ref,
-          int beginColumn: int ref,
-          int endLine: int ref,
-          int endColumn: int ref
-         );
+files(
+  unique int id: @file,
+  string name: string ref
+);
 
-@sourceline = @locatable;
+folders(
+  unique int id: @folder,
+  string name: string ref
+);
 
-numlines(int element_id: @sourceline ref,
-         int num_lines: int ref,
-         int num_code: int ref,
-         int num_comment: int ref
-        );
+@container = @file | @folder
 
-files(unique int id: @file,
-      varchar(900) name: string ref);
+containerparent(
+  int parent: @container ref,
+  unique int child: @container ref
+);
 
-folders(unique int id: @folder,
-        varchar(900) name: string ref);
+/*- Lines of code -*/
 
+numlines(
+  int element_id: @sourceline ref,
+  int num_lines: int ref,
+  int num_code: int ref,
+  int num_comment: int ref
+);
 
-@container = @folder | @file ;
-
-
-containerparent(int parent: @container ref,
-                unique int child: @container ref);
-
-/** Duplicate code **/
-
-duplicateCode(
-  unique int id : @duplication,
-  varchar(900) relativePath : string ref,
-  int equivClass : int ref);
-
-similarCode(
-  unique int id : @similarity,
-  varchar(900) relativePath : string ref,
-  int equivClass : int ref);
-
-@duplication_or_similarity = @duplication | @similarity;
-
-tokens(
-  int id : @duplication_or_similarity ref,
-  int offset : int ref,
-  int beginLine : int ref,
-  int beginColumn : int ref,
-  int endLine : int ref,
-  int endColumn : int ref);
-
-/** External data **/
+/*- External data -*/
 
+/**
+ * External data, loaded from CSV files during snapshot creation. See
+ * [Tutorial: Incorporating external data](https://help.semmle.com/wiki/display/SD/Tutorial%3A+Incorporating+external+data)
+ * for more information.
+ */
 externalData(
   int id : @externalDataElement,
-  varchar(900) path : string ref,
+  string path : string ref,
   int column: int ref,
-  varchar(900) value : string ref
+  string value : string ref
 );
 
-snapshotDate(unique date snapshotDate : date ref);
+/*- Source location prefix -*/
 
-sourceLocationPrefix(varchar(900) prefix : string ref);
+/**
+ * The source location of the snapshot.
+ */
+sourceLocationPrefix(string prefix : string ref);
 
-/** Version control data **/
+/*- JavaScript-specific part -*/
 
-svnentries(
-  int id : @svnentry,
-  varchar(500) revision : string ref,
-  varchar(500) author : string ref,
-  date revisionDate : date ref,
-  int changeSize : int ref
-);
+@location = @location_default
 
-svnaffectedfiles(
-  int id : @svnentry ref,
-  int file : @file ref,
-  varchar(500) action : string ref
-);
-
-svnentrymsg(
-  int id : @svnentry ref,
-  varchar(500) message : string ref
-);
-
-svnchurn(
-  int commit : @svnentry ref,
-  int file : @file ref,
-  int addedLines : int ref,
-  int deletedLines : int ref
-);
-
-
-/*** JavaScript-specific part ***/
+@sourceline = @locatable;
 
 filetype(
   int file: @file ref,
@@ -1046,14 +1018,50 @@ jsdoc_has_new_parameter (int fn: @jsdoc_function_type_expr ref);
 
 jsdoc_errors (unique int id: @jsdoc_error, int tag: @jsdoc_tag ref, varchar(900) message: string ref, varchar(900) tostring: string ref);
 
-// YAML
+@dataflownode = @expr | @function_decl_stmt | @class_decl_stmt | @namespace_declaration | @enum_declaration | @property;
+
+@optionalchainable = @call_expr | @propaccess;
+
+isOptionalChaining(int id: @optionalchainable ref);
+
+/**
+ * The time taken for the extraction of a file.
+ * This table contains non-deterministic content.
+ *
+ * The sum of the `time` column for each (`file`, `timerKind`) pair
+ * is the total time taken for extraction of `file`.  The `extractionPhase`
+ * column provides a granular view of the extraction time of the file.
+ */
+extraction_time(
+  int file : @file ref,
+  // see `com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase`.
+  int extractionPhase: int ref,
+  // 0 for the elapsed CPU time in nanoseconds, 1 for the elapsed wallclock time in nanoseconds
+  int timerKind: int ref,
+  float time: float ref
+)
+
+/**
+* Non-timing related data for the extraction of a single file.
+* This table contains non-deterministic content.
+*/
+extraction_data(
+  int file : @file ref,
+  // the absolute path to the cache file
+  varchar(900) cacheFile: string ref,
+  boolean fromCache: boolean ref,
+  int length: int ref
+)
+
+/*- YAML -*/
+
 #keyset[parent, idx]
 yaml (unique int id: @yaml_node,
       int kind: int ref,
       int parent: @yaml_node_parent ref,
       int idx: int ref,
-      varchar(900) tag: string ref,
-      varchar(900) tostring: string ref);
+      string tag: string ref,
+      string tostring: string ref);
 
 case @yaml_node.kind of
   0 = @yaml_scalar_node
@@ -1067,41 +1075,41 @@ case @yaml_node.kind of
 @yaml_node_parent = @yaml_collection_node | @file;
 
 yaml_anchors (unique int node: @yaml_node ref,
-              varchar(900) anchor: string ref);
+              string anchor: string ref);
 
 yaml_aliases (unique int alias: @yaml_alias_node ref,
-              varchar(900) target: string ref);
+              string target: string ref);
 
 yaml_scalars (unique int scalar: @yaml_scalar_node ref,
               int style: int ref,
-              varchar(900) value: string ref);
+              string value: string ref);
 
 yaml_errors (unique int id: @yaml_error,
-             varchar(900) message: string ref);
+             string message: string ref);
 
 yaml_locations(unique int locatable: @yaml_locatable ref,
              int location: @location_default ref);
 
 @yaml_locatable = @yaml_node | @yaml_error;
 
-/* XML Files */
+/*- XML Files -*/
 
 xmlEncoding(
   unique int id: @file ref,
-  varchar(900) encoding: string ref
+  string encoding: string ref
 );
 
 xmlDTDs(
   unique int id: @xmldtd,
-  varchar(900) root: string ref,
-  varchar(900) publicId: string ref,
-  varchar(900) systemId: string ref,
+  string root: string ref,
+  string publicId: string ref,
+  string systemId: string ref,
   int fileid: @file ref
 );
 
 xmlElements(
   unique int id: @xmlelement,
-  varchar(900) name: string ref,
+  string name: string ref,
   int parentid: @xmlparent ref,
   int idx: int ref,
   int fileid: @file ref
@@ -1110,16 +1118,16 @@ xmlElements(
 xmlAttrs(
   unique int id: @xmlattribute,
   int elementid: @xmlelement ref,
-  varchar(900) name: string ref,
-  varchar(3600) value: string ref,
+  string name: string ref,
+  string value: string ref,
   int idx: int ref,
   int fileid: @file ref
 );
 
 xmlNs(
   int id: @xmlnamespace,
-  varchar(900) prefixName: string ref,
-  varchar(900) URI: string ref,
+  string prefixName: string ref,
+  string URI: string ref,
   int fileid: @file ref
 );
 
@@ -1131,14 +1139,14 @@ xmlHasNs(
 
 xmlComments(
   unique int id: @xmlcomment,
-  varchar(3600) text: string ref,
+  string text: string ref,
   int parentid: @xmlparent ref,
   int fileid: @file ref
 );
 
 xmlChars(
   unique int id: @xmlcharacters,
-  varchar(3600) text: string ref,
+  string text: string ref,
   int parentid: @xmlparent ref,
   int idx: int ref,
   int isCDATA: int ref,
@@ -1155,15 +1163,7 @@ xmllocations(
 
 @xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace;
 
-@dataflownode = @expr | @function_decl_stmt | @class_decl_stmt | @namespace_declaration | @enum_declaration | @property;
-
-@optionalchainable = @call_expr | @propaccess;
-
-isOptionalChaining(int id: @optionalchainable ref);
-
-/*
- * configuration files with key value pairs
- */
+/*- Configuration files with key value pairs -*/
 
 configs(
   unique int id: @config
@@ -1187,32 +1187,3 @@ configLocations(
 );
 
 @configLocatable = @config | @configName | @configValue;
-
-/**
- * The time taken for the extraction of a file.
- * This table contains non-deterministic content.
- *
- * The sum of the `time` column for each (`file`, `timerKind`) pair
- * is the total time taken for extraction of `file`.  The `extractionPhase`
- * column provides a granular view of the extraction time of the file.
- */
-extraction_time(
-   int file : @file ref,
-   // see `com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase`.
-   int extractionPhase: int ref,
-   // 0 for the elapsed CPU time in nanoseconds, 1 for the elapsed wallclock time in nanoseconds
-   int timerKind: int ref,
-   float time: float ref
-)
-
-/**
- * Non-timing related data for the extraction of a single file.
- * This table contains non-deterministic content.
- */
-extraction_data(
-   int file : @file ref,
-   // the absolute path to the cache file
-   varchar(900) cacheFile: string ref,
-   boolean fromCache: boolean ref,
-   int length: int ref
-)

javascript/ql/lib/upgrades/4d00210ca570d55c4833af11d3372b774dbc63f2/upgrade.properties

@@ -0,0 +1,11 @@
+description: Sync dbscheme fragments
+compatibility: full
+
+duplicateCode.rel: delete
+similarCode.rel: delete
+tokens.rel: delete
+snapshotDate.rel: delete
+svnentries.rel: delete
+svnaffectedfiles.rel: delete
+svnentrymsg.rel: delete
+svnchurn.rel: delete

javascript/ql/src/Performance/PolynomialReDoS.qhelp

@@ -15,8 +15,7 @@
 		</p>
 
 		<sample language="javascript">
-			text.replace(/^\s+|\s+$/g, ''); // BAD
-		</sample>
+text.replace(/^\s+|\s+$/g, ''); // BAD</sample>
 
 		<p>
 
@@ -71,8 +70,7 @@
 		</p>
 
 		<sample language="javascript">
-			/^0\.\d+E?\d+$/.test(str) // BAD
-		</sample>
+/^0\.\d+E?\d+$/.test(str) // BAD</sample>
 
 		<p>
 
@@ -103,6 +101,33 @@
 
 	</example>
 
+    <example>
+        <p>
+            Sometimes it is unclear how a regular expression can be rewritten to
+            avoid the problem. In such cases, it often suffices to limit the
+            length of the input string. For instance, the following 
+            regular expression is used to match numbers, and on some non-number 
+            inputs it can have quadratic time complexity:
+        </p>
+
+        <sample language="javascript">
+/^(\+|-)?(\d+|(\d*\.\d*))?(E|e)?([-+])?(\d+)?$/.test(str) // BAD</sample>
+
+        <p>
+            It is not immediately obvious how to rewrite this regular expression 
+            to avoid the problem. However, you can mitigate performance issues by limiting the length
+            to 1000 characters, which will always finish in a reasonable amount
+            of time.
+        </p>
+
+        <sample language="javascript">
+if (str.length &gt; 1000) {
+    throw new Error("Input too long");
+}
+
+/^(\+|-)?(\d+|(\d*\.\d*))?(E|e)?([-+])?(\d+)?$/.test(str)</sample>
+    </example>
+
 	<include src="ReDoSReferences.inc.qhelp"/>
 
 </qhelp>

javascript/ql/test/library-tests/NPM/src/node_modules/parent-module/main.js

@@ -0,0 +1 @@
+module.exports = "parent";

javascript/ql/test/library-tests/NPM/src/node_modules/parent-module/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "parent-module",
+  "main": "main.js"
+}

javascript/ql/test/library-tests/NPM/src/node_modules/parent-module/sub-module/main.js

@@ -0,0 +1 @@
+module.exports = "sub";

javascript/ql/test/library-tests/NPM/src/node_modules/parent-module/sub-module/package.json

@@ -0,0 +1,3 @@
+{
+  "main": "main.js"
+}

javascript/ql/test/library-tests/NPM/src/test-submodule.js

@@ -0,0 +1,2 @@
+require("parent-module");
+require("parent-module/sub-module");

javascript/ql/test/library-tests/NPM/tests.expected

@@ -8,6 +8,8 @@ importedFile
 | src/node_modules/nested/tst3.js:1:1:1:29 | require ... odule') | src/node_modules/third-party-module/fancy.js:0:0:0:0 | src/node_modules/third-party-module/fancy.js |
 | src/node_modules/nested/tst3.js:2:1:2:12 | require('a') | src/node_modules/nested/node_modules/a/index.js:0:0:0:0 | src/node_modules/nested/node_modules/a/index.js |
 | src/node_modules/tst2.js:1:1:1:38 | require ... cy.js') | src/node_modules/third-party-module/fancy.js:0:0:0:0 | src/node_modules/third-party-module/fancy.js |
+| src/test-submodule.js:1:1:1:24 | require ... odule") | src/node_modules/parent-module/main.js:0:0:0:0 | src/node_modules/parent-module/main.js |
+| src/test-submodule.js:2:1:2:35 | require ... odule") | src/node_modules/parent-module/sub-module/main.js:0:0:0:0 | src/node_modules/parent-module/sub-module/main.js |
 | src/tst2.js:1:1:1:12 | require(".") | src/index.js:0:0:0:0 | src/index.js |
 | src/tst.js:1:1:1:38 | require ... cy.js') | src/node_modules/third-party-module/fancy.js:0:0:0:0 | src/node_modules/third-party-module/fancy.js |
 | src/tst.js:2:1:2:37 | require ... ckage') | src/node_modules/third-party-module/package.json:0:0:0:0 | src/node_modules/third-party-module/package.json |
@@ -16,18 +18,24 @@ importedModule
 | src/node_modules/nested/tst3.js:1:1:1:29 | require ... odule') | src/node_modules/third-party-module/fancy.js:1:1:4:0 | <toplevel> |
 | src/node_modules/nested/tst3.js:2:1:2:12 | require('a') | src/node_modules/nested/node_modules/a/index.js:1:1:1:25 | <toplevel> |
 | src/node_modules/tst2.js:1:1:1:38 | require ... cy.js') | src/node_modules/third-party-module/fancy.js:1:1:4:0 | <toplevel> |
+| src/test-submodule.js:1:1:1:24 | require ... odule") | src/node_modules/parent-module/main.js:1:1:2:0 | <toplevel> |
+| src/test-submodule.js:2:1:2:35 | require ... odule") | src/node_modules/parent-module/sub-module/main.js:1:1:2:0 | <toplevel> |
 | src/tst2.js:1:1:1:12 | require(".") | src/index.js:1:1:4:0 | <toplevel> |
 | src/tst.js:1:1:1:38 | require ... cy.js') | src/node_modules/third-party-module/fancy.js:1:1:4:0 | <toplevel> |
 modules
 | src | test-package | src/index.js:1:1:4:0 | <toplevel> |
 | src | test-package | src/lib/tst2.js:1:1:1:14 | <toplevel> |
 | src | test-package | src/lib/tst.js:1:1:4:0 | <toplevel> |
+| src | test-package | src/test-submodule.js:1:1:3:0 | <toplevel> |
 | src | test-package | src/tst2.js:1:1:1:13 | <toplevel> |
 | src | test-package | src/tst.js:1:1:2:38 | <toplevel> |
 | src/node_modules/b | b | src/node_modules/b/lib/index.js:1:1:2:0 | <toplevel> |
 | src/node_modules/b | b | src/node_modules/b/lib/util.ts:1:1:2:0 | <toplevel> |
 | src/node_modules/c | c | src/node_modules/c/src/index.js:1:1:2:0 | <toplevel> |
 | src/node_modules/d | d | src/node_modules/d/main.js:1:1:2:0 | <toplevel> |
+| src/node_modules/parent-module | parent-module | src/node_modules/parent-module/main.js:1:1:2:0 | <toplevel> |
+| src/node_modules/parent-module | parent-module | src/node_modules/parent-module/sub-module/main.js:1:1:2:0 | <toplevel> |
+| src/node_modules/parent-module/sub-module | parent-module/sub-module | src/node_modules/parent-module/sub-module/main.js:1:1:2:0 | <toplevel> |
 | src/node_modules/third-party-module | third-party-module | src/node_modules/third-party-module/fancy.js:1:1:4:0 | <toplevel> |
 npm
 | src/node_modules/third-party-module/package.json:1:1:5:1 | {\\n  "na ... y.js"\\n} | third-party-module | 23.4.0 |
@@ -36,12 +44,16 @@ getMainModule
 | src/node_modules/b/package.json:1:1:4:1 | {\\n  "na ... "lib"\\n} | b | src/node_modules/b/lib/index.js:1:1:2:0 | <toplevel> |
 | src/node_modules/c/package.json:1:1:4:1 | {\\n  "na ... src/"\\n} | c | src/node_modules/c/src/index.js:1:1:2:0 | <toplevel> |
 | src/node_modules/d/package.json:1:1:4:1 | {\\n  "na ... main"\\n} | d | src/node_modules/d/main.js:1:1:2:0 | <toplevel> |
+| src/node_modules/parent-module/package.json:1:1:4:1 | {\\n  "na ... n.js"\\n} | parent-module | src/node_modules/parent-module/main.js:1:1:2:0 | <toplevel> |
+| src/node_modules/parent-module/sub-module/package.json:1:1:3:1 | {\\n  "ma ... n.js"\\n} | parent-module/sub-module | src/node_modules/parent-module/sub-module/main.js:1:1:2:0 | <toplevel> |
 | src/node_modules/third-party-module/package.json:1:1:5:1 | {\\n  "na ... y.js"\\n} | third-party-module | src/node_modules/third-party-module/fancy.js:1:1:4:0 | <toplevel> |
 | src/package.json:1:1:20:1 | {\\n  "na ... "\\n  }\\n} | test-package | src/index.js:1:1:4:0 | <toplevel> |
 packageJson
 | src/node_modules/b/package.json:1:1:4:1 | {\\n  "na ... "lib"\\n} |
 | src/node_modules/c/package.json:1:1:4:1 | {\\n  "na ... src/"\\n} |
 | src/node_modules/d/package.json:1:1:4:1 | {\\n  "na ... main"\\n} |
+| src/node_modules/parent-module/package.json:1:1:4:1 | {\\n  "na ... n.js"\\n} |
+| src/node_modules/parent-module/sub-module/package.json:1:1:3:1 | {\\n  "ma ... n.js"\\n} |
 | src/node_modules/third-party-module/package.json:1:1:5:1 | {\\n  "na ... y.js"\\n} |
 | src/package.json:1:1:20:1 | {\\n  "na ... "\\n  }\\n} |
 dependencyInfo
@@ -53,5 +65,6 @@ dependencyInfo
 | src/package.json:11:20:11:37 | "1.2.3-alpha.beta" | something | unknown |
 | src/package.json:12:14:12:57 | "! garb ... arse %" | foo | unknown |
 | src/package.json:15:16:15:20 | "1.0" | mocha | 1.0 |
+| src/test-submodule.js:1:1:3:0 | <toplevel> | test-package | 0.1.0 |
 | src/tst2.js:1:1:1:13 | <toplevel> | test-package | 0.1.0 |
 | src/tst.js:1:1:2:38 | <toplevel> | test-package | 0.1.0 |

javascript/ql/test/tutorials/Introducing the JavaScript libraries/query12.qll

@@ -4,7 +4,7 @@ query predicate test_query12(MethodCallExpr send) {
   exists(SimpleParameter res, DataFlow::Node resNode |
     res.getName() = "res" and
     resNode = DataFlow::parameterNode(res) and
-    resNode.getASuccessor() = DataFlow::valueNode(send.getReceiver()) and
+    resNode.getASuccessor+() = DataFlow::valueNode(send.getReceiver()) and
     send.getMethodName() = "send"
   |
     any()

javascript/ql/test/tutorials/Introducing the JavaScript libraries/tests.expected

@@ -21,6 +21,7 @@ test_query11
 | tst.js:31:18:31:18 | x | Dead store of local variable. |
 | tst.js:38:7:38:23 | password = "blah" | Dead store of local variable. |
 test_query12
+| tst.js:42:3:42:12 | res.send() |
 test_query20
 test_query3
 | tst.js:27:1:27:4 | <!-- | Do not use HTML comments. |

javascript/ql/test/tutorials/Introducing the JavaScript libraries/tst.js

@@ -37,3 +37,7 @@ var j, j;
 function foo() {
   var password = "blah";
 }
+
+function m(res) {
+  res.send()
+}