Merge pull request #3832 from asger-semmle/js/typescript-in-html-files3

Approved by erik-krogh

change-notes/1.25/analysis-javascript.md

@@ -29,6 +29,8 @@
 
 * TypeScript 3.9 is now supported.
 
+* TypeScript code embedded in HTML and Vue files is now extracted and analyzed.
+
 * The analysis of sanitizers has improved, leading to more accurate
   results from the security queries.
 

javascript/extractor/lib/typescript/src/common.ts

@@ -48,7 +48,7 @@ export class Project {
   public load(): void {
     const { config, host } = this;
     this.program = ts.createProgram(config.fileNames, config.options, host);
-    this.typeTable.setProgram(this.program);
+    this.typeTable.setProgram(this.program, this.virtualSourceRoot);
   }
 
   /**
@@ -71,10 +71,19 @@ export class Project {
         redirectedReference: ts.ResolvedProjectReference,
         options: ts.CompilerOptions) {
 
+    let oppositePath =
+      this.virtualSourceRoot.toVirtualPath(containingFile) ||
+      this.virtualSourceRoot.fromVirtualPath(containingFile);
+
     const { host, resolutionCache } = this;
     return moduleNames.map((moduleName) => {
       let redirected = this.redirectModuleName(moduleName, containingFile, options);
       if (redirected != null) return redirected;
+      if (oppositePath != null) {
+        // If the containing file is in the virtual source root, try resolving from the real source root, and vice versa.
+        redirected = ts.resolveModuleName(moduleName, oppositePath, options,  host, resolutionCache).resolvedModule;
+        if (redirected != null) return redirected;
+      }
       return ts.resolveModuleName(moduleName, containingFile, options,  host, resolutionCache).resolvedModule;
     });
   }
@@ -90,15 +99,7 @@ export class Project {
 
     // Get the overridden location of this package, if one exists.
     let packageEntryPoint = this.packageEntryPoints.get(packageName);
-    if (packageEntryPoint == null) {
+    if (packageEntryPoint == null) return null;
-      // The package is not overridden, but we have established that it begins with a valid package name.
-      // Do a lookup in the virtual source root (where dependencies are installed) by changing the 'containing file'.
-      let virtualContainingFile = this.virtualSourceRoot.toVirtualPath(containingFile);
-      if (virtualContainingFile != null) {
-        return ts.resolveModuleName(moduleName, virtualContainingFile, options, this.host, this.resolutionCache).resolvedModule;
-      }
-      return null;
-    }
 
     // If the requested module name is exactly the overridden package name,
     // return the entry point file (it is not necessarily called `index.ts`).

javascript/extractor/lib/typescript/src/main.ts

@@ -414,7 +414,25 @@ function loadTsConfig(command: LoadCommand): LoadedConfig {
      */
     let parseConfigHost: ts.ParseConfigHost = {
         useCaseSensitiveFileNames: true,
-        readDirectory: ts.sys.readDirectory, // No need to override traversal/glob matching
+        readDirectory: (rootDir, extensions, excludes?, includes?, depth?) => {
+            // Perform the glob matching in both real and virtual source roots.
+            let exclusions = excludes == null ? [] : [...excludes];
+            if (virtualSourceRoot.virtualSourceRoot != null) {
+                // qltest puts the virtual source root inside the real source root (.testproj).
+                // Make sure we don't find files inside the virtual source root in this pass.
+                exclusions.push(virtualSourceRoot.virtualSourceRoot);
+            }
+            let originalResults = ts.sys.readDirectory(rootDir, extensions, exclusions, includes, depth)
+            let virtualDir = virtualSourceRoot.toVirtualPath(rootDir);
+            if (virtualDir == null) {
+                return originalResults;
+            }
+            // Make sure glob matching does not to discover anything in node_modules.
+            let virtualExclusions = excludes == null ? [] : [...excludes];
+            virtualExclusions.push('**/node_modules/**/*');
+            let virtualResults = ts.sys.readDirectory(virtualDir, extensions, virtualExclusions, includes, depth)
+            return [ ...originalResults, ...virtualResults ];
+        },
         fileExists: (path: string) => {
             return ts.sys.fileExists(path)
                 || virtualSourceRoot.toVirtualPathIfFileExists(path) != null

javascript/extractor/lib/typescript/src/type_table.ts

@@ -1,4 +1,5 @@
 import * as ts from "./typescript";
+import { VirtualSourceRoot } from "./virtual_source_root";
 
 interface AugmentedSymbol extends ts.Symbol {
   parent?: AugmentedSymbol;
@@ -379,12 +380,15 @@ export class TypeTable {
    */
   public restrictedExpansion = false;
 
+  private virtualSourceRoot: VirtualSourceRoot;
+
   /**
    * Called when a new compiler instance has started.
    */
-  public setProgram(program: ts.Program) {
+  public setProgram(program: ts.Program, virtualSourceRoot: VirtualSourceRoot) {
     this.typeChecker = program.getTypeChecker();
     this.arbitraryAstNode = program.getSourceFiles()[0];
+    this.virtualSourceRoot = virtualSourceRoot;
   }
 
   /**
@@ -703,14 +707,21 @@ export class TypeTable {
   private getSymbolString(symbol: AugmentedSymbol): string {
     let parent = symbol.parent;
     if (parent == null || parent.escapedName === ts.InternalSymbolName.Global) {
-      return "root;" + this.getSymbolDeclarationString(symbol) + ";;" + symbol.name;
+      return "root;" + this.getSymbolDeclarationString(symbol) + ";;" + this.rewriteSymbolName(symbol);
     } else if (parent.exports != null && parent.exports.get(symbol.escapedName) === symbol) {
-      return "member;;" + this.getSymbolId(parent) + ";" + symbol.name;
+      return "member;;" + this.getSymbolId(parent) + ";" + this.rewriteSymbolName(symbol);
     } else {
-      return "other;" + this.getSymbolDeclarationString(symbol) + ";" + this.getSymbolId(parent) + ";" + symbol.name;
+      return "other;" + this.getSymbolDeclarationString(symbol) + ";" + this.getSymbolId(parent) + ";" + this.rewriteSymbolName(symbol);
     }
   }
 
+  private rewriteSymbolName(symbol: AugmentedSymbol) {
+    let { virtualSourceRoot, sourceRoot } = this.virtualSourceRoot;
+    let { name } = symbol;
+    if (virtualSourceRoot == null || sourceRoot == null) return name;
+    return name.replace(virtualSourceRoot, sourceRoot);
+  }
+
   /**
    * Gets a string that distinguishes the given symbol from symbols with different
    * lexical roots, or an empty string if the symbol is not a lexical root.

javascript/extractor/lib/typescript/src/virtual_source_root.ts

@@ -7,23 +7,42 @@ import * as ts from "./typescript";
  */
 export class VirtualSourceRoot {
   constructor(
-    private sourceRoot: string | null,
+    public sourceRoot: string | null,
 
     /**
      * Directory whose folder structure mirrors the real source root, but with `node_modules` installed,
      * or undefined if no virtual source root exists.
      */
-    private virtualSourceRoot: string | null,
+    public virtualSourceRoot: string | null,
   ) {}
 
+  private static translate(oldRoot: string, newRoot: string, path: string) {
+    if (!oldRoot || !newRoot) return null;
+    let relative = pathlib.relative(oldRoot, path);
+    if (relative.startsWith('..') || pathlib.isAbsolute(relative)) return null;
+    return pathlib.join(newRoot, relative);
+  }
+
   /**
    * Maps a path under the real source root to the corresponding path in the virtual source root.
+   *
+   * Returns `null` for paths already in the virtual source root.
    */
   public toVirtualPath(path: string) {
-    if (!this.virtualSourceRoot || !this.sourceRoot) return null;
+    let { virtualSourceRoot } = this;
-    let relative = pathlib.relative(this.sourceRoot, path);
+    if (path.startsWith(virtualSourceRoot)) {
-    if (relative.startsWith('..') || pathlib.isAbsolute(relative)) return null;
+      // 'qltest' creates a virtual source root inside the real source root.
-    return pathlib.join(this.virtualSourceRoot, relative);
+      // Make sure such files don't appear to be inside the real source root.
+      return null;
+    }
+    return VirtualSourceRoot.translate(this.sourceRoot, virtualSourceRoot, path);
+  }
+
+  /**
+   * Maps a path under the virtual source root to the corresponding path in the real source root.
+   */
+  public fromVirtualPath(path: string) {
+    return VirtualSourceRoot.translate(this.virtualSourceRoot, this.sourceRoot, path);
   }
 
   /**

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -26,9 +26,11 @@ import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -210,6 +212,8 @@ public class AutoBuild {
   private volatile boolean seenCode = false;
   private boolean installDependencies = false;
   private int installDependenciesTimeout;
+  private final VirtualSourceRoot virtualSourceRoot;
+  private ExtractorState state;
 
   /** The default timeout when running <code>yarn</code>, in milliseconds. */
   public static final int INSTALL_DEPENDENCIES_DEFAULT_TIMEOUT = 10 * 60 * 1000; // 10 minutes
@@ -227,9 +231,15 @@ public class AutoBuild {
         Env.systemEnv()
             .getInt(
                 "LGTM_INDEX_TYPESCRIPT_INSTALL_DEPS_TIMEOUT", INSTALL_DEPENDENCIES_DEFAULT_TIMEOUT);
+    this.virtualSourceRoot = makeVirtualSourceRoot();
     setupFileTypes();
     setupXmlMode();
     setupMatchers();
+    this.state = new ExtractorState();
+  }
+
+  protected VirtualSourceRoot makeVirtualSourceRoot() {
+    return new VirtualSourceRoot(LGTM_SRC, toRealPath(Paths.get(EnvironmentVariables.getScratchDir())));
   }
 
   private String getEnvVar(String envVarName) {
@@ -530,7 +540,7 @@ public class AutoBuild {
           @Override
           public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
               throws IOException {
-            if (".js".equals(FileUtil.extension(file.toString()))) extract(extractor, file, null);
+            if (".js".equals(FileUtil.extension(file.toString()))) extract(extractor, file, true);
             return super.visitFile(file, attrs);
           }
         };
@@ -568,19 +578,37 @@ public class AutoBuild {
     }
   };
 
+  public class FileExtractors {
+    FileExtractor defaultExtractor;
+    Map<String, FileExtractor> customExtractors = new LinkedHashMap<>();
+
+    FileExtractors(FileExtractor defaultExtractor) {
+      this.defaultExtractor = defaultExtractor;
+    }
+
+    public FileExtractor forFile(Path f) {
+      return customExtractors.getOrDefault(FileUtil.extension(f), defaultExtractor);
+    }
+
+    public FileType fileType(Path f) {
+      return forFile(f).getFileType(f.toFile());
+    }
+  }
+
   /** Extract all supported candidate files that pass the filters. */
   private void extractSource() throws IOException {
     // default extractor
     FileExtractor defaultExtractor =
         new FileExtractor(mkExtractorConfig(), outputConfig, trapCache);
 
+    FileExtractors extractors = new FileExtractors(defaultExtractor);
+
     // custom extractor for explicitly specified file types
-    Map<String, FileExtractor> customExtractors = new LinkedHashMap<>();
     for (Map.Entry<String, FileType> spec : fileTypes.entrySet()) {
       String extension = spec.getKey();
       String fileType = spec.getValue().name();
       ExtractorConfig extractorConfig = mkExtractorConfig().withFileType(fileType);
-      customExtractors.put(extension, new FileExtractor(extractorConfig, outputConfig, trapCache));
+      extractors.customExtractors.put(extension, new FileExtractor(extractorConfig, outputConfig, trapCache));
     }
 
     Set<Path> filesToExtract = new LinkedHashSet<>();
@@ -599,29 +627,44 @@ public class AutoBuild {
     if (!tsconfigFiles.isEmpty()) {
       dependencyInstallationResult = this.preparePackagesAndDependencies(filesToExtract);
     }
+    Set<Path> extractedFiles = new LinkedHashSet<>();
+    
+    // Extract HTML files as they may contain TypeScript
+    CompletableFuture<?> htmlFuture = extractFiles(
+        filesToExtract, extractedFiles, extractors,
+        f -> extractors.fileType(f) == FileType.HTML);
+    
+    htmlFuture.join(); // Wait for HTML extraction to be finished.
 
     // extract TypeScript projects and files
-    Set<Path> extractedFiles =
+    extractTypeScript(filesToExtract, extractedFiles,
-          extractTypeScript(
+              extractors, tsconfigFiles, dependencyInstallationResult);
-              defaultExtractor, filesToExtract, tsconfigFiles, dependencyInstallationResult);
 
     boolean hasTypeScriptFiles = extractedFiles.size() > 0;
 
     // extract remaining files
+    extractFiles(
+        filesToExtract, extractedFiles, extractors,
+        f -> !(hasTypeScriptFiles && isFileDerivedFromTypeScriptFile(f, extractedFiles)));
+  }
+
+  private CompletableFuture<?> extractFiles(
+      Set<Path> filesToExtract,
+      Set<Path> extractedFiles,
+      FileExtractors extractors,
+      Predicate<Path> shouldExtract) {
+
+    List<CompletableFuture<?>> futures = new ArrayList<>();
     for (Path f : filesToExtract) {
       if (extractedFiles.contains(f))
         continue;
-      if (hasTypeScriptFiles && isFileDerivedFromTypeScriptFile(f, extractedFiles)) {
+      if (!shouldExtract.test(f)) {
         continue;
       }
       extractedFiles.add(f);
-      FileExtractor extractor = defaultExtractor;
+      futures.add(extract(extractors.forFile(f), f, true));
-      if (!fileTypes.isEmpty()) {
-        String extension = FileUtil.extension(f);
-        if (customExtractors.containsKey(extension)) extractor = customExtractors.get(extension);
-      }
-      extract(extractor, f, null);
     }
+    return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
   }
 
   /**
@@ -733,7 +776,6 @@ public class AutoBuild {
    */
 protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path> filesToExtract) {
     final Path sourceRoot = LGTM_SRC;
-    final Path virtualSourceRoot = toRealPath(Paths.get(EnvironmentVariables.getScratchDir()));
 
     // Read all package.json files and index them by name.
     Map<Path, JsonObject> packageJsonFiles = new LinkedHashMap<>();
@@ -820,8 +862,7 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
 
     // Write the new package.json files to disk
     for (Path file : packageJsonFiles.keySet()) {
-      Path relativePath = sourceRoot.relativize(file);
+      Path virtualFile = virtualSourceRoot.toVirtualFile(file);
-      Path virtualFile = virtualSourceRoot.resolve(relativePath);
 
       try {
         Files.createDirectories(virtualFile.getParent());
@@ -836,7 +877,7 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
     // Install dependencies
     if (this.installDependencies && verifyYarnInstallation()) {
       for (Path file : packageJsonFiles.keySet()) {
-        Path virtualFile = virtualSourceRoot.resolve(sourceRoot.relativize(file));
+        Path virtualFile = virtualSourceRoot.toVirtualFile(file);
         System.out.println("Installing dependencies from " + virtualFile);
         ProcessBuilder pb =
             new ProcessBuilder(
@@ -862,7 +903,7 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
       }
     }
 
-    return new DependencyInstallationResult(sourceRoot, virtualSourceRoot, packageMainFile, packagesInRepo);
+    return new DependencyInstallationResult(packageMainFile, packagesInRepo);
   }
 
   /**
@@ -933,21 +974,20 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
     ExtractorConfig config = new ExtractorConfig(true);
     config = config.withSourceType(getSourceType());
     config = config.withTypeScriptMode(typeScriptMode);
+    config = config.withVirtualSourceRoot(virtualSourceRoot);
     if (defaultEncoding != null) config = config.withDefaultEncoding(defaultEncoding);
     return config;
   }
 
   private Set<Path> extractTypeScript(
-      FileExtractor extractor,
       Set<Path> files,
+      Set<Path> extractedFiles,
+      FileExtractors extractors,
       List<Path> tsconfig,
       DependencyInstallationResult deps) {
-    Set<Path> extractedFiles = new LinkedHashSet<>();
-
     if (hasTypeScriptFiles(files) || !tsconfig.isEmpty()) {
-      ExtractorState extractorState = new ExtractorState();
+      TypeScriptParser tsParser = state.getTypeScriptParser();
-      TypeScriptParser tsParser = extractorState.getTypeScriptParser();
+      verifyTypeScriptInstallation(state);
-      verifyTypeScriptInstallation(extractorState);
 
       // Collect all files included in a tsconfig.json inclusion pattern.
       // If a given file is referenced by multiple tsconfig files, we prefer to extract it using
@@ -955,7 +995,7 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
       Set<File> explicitlyIncludedFiles = new LinkedHashSet<>();
       if (tsconfig.size() > 1) { // No prioritization needed if there's only one tsconfig.
         for (Path projectPath : tsconfig) {
-          explicitlyIncludedFiles.addAll(tsParser.getOwnFiles(projectPath.toFile(), deps));
+          explicitlyIncludedFiles.addAll(tsParser.getOwnFiles(projectPath.toFile(), deps, virtualSourceRoot));
         }
       }
 
@@ -963,16 +1003,19 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
       for (Path projectPath : tsconfig) {
         File projectFile = projectPath.toFile();
         long start = logBeginProcess("Opening project " + projectFile);
-        ParsedProject project = tsParser.openProject(projectFile, deps);
+        ParsedProject project = tsParser.openProject(projectFile, deps, virtualSourceRoot);
         logEndProcess(start, "Done opening project " + projectFile);
         // Extract all files belonging to this project which are also matched
         // by our include/exclude filters.
         List<Path> typeScriptFiles = new ArrayList<Path>();
         for (File sourceFile : project.getAllFiles()) {
           Path sourcePath = sourceFile.toPath();
-          if (!files.contains(normalizePath(sourcePath))) continue;
+          Path normalizedFile = normalizePath(sourcePath);
+          if (!files.contains(normalizedFile) && !state.getSnippets().containsKey(normalizedFile)) {
+            continue;
+          }
           if (!project.getOwnFiles().contains(sourceFile) && explicitlyIncludedFiles.contains(sourceFile)) continue;
-          if (!FileType.TYPESCRIPT.getExtensions().contains(FileUtil.extension(sourcePath))) {
+          if (extractors.fileType(sourcePath) != FileType.TYPESCRIPT) {
             // For the time being, skip non-TypeScript files, even if the TypeScript
             // compiler can parse them for us.
             continue;
@@ -982,7 +1025,7 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
           }
         }
         typeScriptFiles.sort(PATH_ORDERING);
-        extractTypeScriptFiles(typeScriptFiles, extractedFiles, extractor, extractorState);
+        extractTypeScriptFiles(typeScriptFiles, extractedFiles, extractors);
         tsParser.closeProject(projectFile);
       }
 
@@ -996,12 +1039,12 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
       List<Path> remainingTypeScriptFiles = new ArrayList<>();
       for (Path f : files) {
         if (!extractedFiles.contains(f)
-            && FileType.forFileExtension(f.toFile()) == FileType.TYPESCRIPT) {
+            && extractors.fileType(f) == FileType.TYPESCRIPT) {
           remainingTypeScriptFiles.add(f);
         }
       }
       if (!remainingTypeScriptFiles.isEmpty()) {
-        extractTypeScriptFiles(remainingTypeScriptFiles, extractedFiles, extractor, extractorState);
+        extractTypeScriptFiles(remainingTypeScriptFiles, extractedFiles, extractors);
       }
 
       // The TypeScript compiler instance is no longer needed.
@@ -1087,16 +1130,15 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
   public void extractTypeScriptFiles(
       List<Path> files,
       Set<Path> extractedFiles,
-      FileExtractor extractor,
+      FileExtractors extractors) {
-      ExtractorState extractorState) {
     List<File> list = files
         .stream()
         .sorted(PATH_ORDERING)
         .map(p -> p.toFile()).collect(Collectors.toList());
-    extractorState.getTypeScriptParser().prepareFiles(list);
+    state.getTypeScriptParser().prepareFiles(list);
     for (Path path : files) {
       extractedFiles.add(path);
-      extract(extractor, path, extractorState);
+      extract(extractors.forFile(path), path, false);
     }
   }
 
@@ -1139,10 +1181,13 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
    * <p>If the state is {@code null}, the extraction job will be submitted to the {@link
    * #threadPool}, otherwise extraction will happen on the main thread.
    */
-  protected void extract(FileExtractor extractor, Path file, ExtractorState state) {
+  protected CompletableFuture<?> extract(FileExtractor extractor, Path file, boolean concurrent) {
-    if (state == null && threadPool != null)
+    if (concurrent && threadPool != null) {
-      threadPool.submit(() -> doExtract(extractor, file, state));
+      return CompletableFuture.runAsync(() -> doExtract(extractor, file, state), threadPool);
-    else doExtract(extractor, file, state);
+    } else {
+      doExtract(extractor, file, state);
+      return CompletableFuture.completedFuture(null);
+    }
   }
 
   private void doExtract(FileExtractor extractor, Path file, ExtractorState state) {

javascript/extractor/src/com/semmle/js/extractor/DependencyInstallationResult.java

@@ -6,46 +6,19 @@ import java.util.Map;
 
 /** Contains the results of installing dependencies. */
 public class DependencyInstallationResult {
-  private Path sourceRoot;
-  private Path virtualSourceRoot;
   private Map<String, Path> packageEntryPoints;
   private Map<String, Path> packageJsonFiles;
 
   public static final DependencyInstallationResult empty =
-      new DependencyInstallationResult(null, null, Collections.emptyMap(), Collections.emptyMap());
+      new DependencyInstallationResult(Collections.emptyMap(), Collections.emptyMap());
 
   public DependencyInstallationResult(
-      Path sourceRoot,
-      Path virtualSourceRoot,
       Map<String, Path> packageEntryPoints,
       Map<String, Path> packageJsonFiles) {
-    this.sourceRoot = sourceRoot;
-    this.virtualSourceRoot = virtualSourceRoot;
     this.packageEntryPoints = packageEntryPoints;
     this.packageJsonFiles = packageJsonFiles;
   }
 
-  /**
-   * Returns the source root mirrored by {@link #getVirtualSourceRoot()} or <code>null</code>
-   * if no virtual source root exists.
-   * <p>
-   * When invoked from the AutoBuilder, this corresponds to the source root. When invoked
-   * from ODASA, there is no notion of source root, so this is always <code>null</code> in that context.
-   */
-  public Path getSourceRoot() {
-    return sourceRoot;
-  }
-
-  /**
-   * Returns the virtual source root or <code>null</code> if no virtual source root exists.
-   * <p>
-   * The virtual source root is a directory hierarchy that mirrors the real source
-   * root, where dependencies are installed.
-   */
-  public Path getVirtualSourceRoot() {
-    return virtualSourceRoot;
-  }
-
   /**
    * Returns the mapping from package names to the TypeScript file that should
    * act as its main entry point.

javascript/extractor/src/com/semmle/js/extractor/ExtractorConfig.java

@@ -1,8 +1,5 @@
 package com.semmle.js.extractor;
 
-import com.semmle.js.parser.JcornWrapper;
-import com.semmle.util.data.StringUtil;
-import com.semmle.util.exception.UserError;
 import java.nio.charset.Charset;
 import java.nio.charset.IllegalCharsetNameException;
 import java.nio.charset.StandardCharsets;
@@ -12,6 +9,10 @@ import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.Set;
 
+import com.semmle.js.parser.JcornWrapper;
+import com.semmle.util.data.StringUtil;
+import com.semmle.util.exception.UserError;
+
 /**
  * Configuration options that affect the behaviour of the extractor.
  *
@@ -236,6 +237,8 @@ public class ExtractorConfig {
   /** The default character encoding to use for parsing source files. */
   private String defaultEncoding;
 
+  private VirtualSourceRoot virtualSourceRoot;
+
   public ExtractorConfig(boolean experimental) {
     this.ecmaVersion = experimental ? ECMAVersion.ECMA2020 : ECMAVersion.ECMA2019;
     this.platform = Platform.AUTO;
@@ -252,6 +255,7 @@ public class ExtractorConfig {
     this.typescriptMode = TypeScriptMode.NONE;
     this.e4x = experimental;
     this.defaultEncoding = StandardCharsets.UTF_8.name();
+    this.virtualSourceRoot = VirtualSourceRoot.none;
   }
 
   public ExtractorConfig(ExtractorConfig that) {
@@ -272,6 +276,7 @@ public class ExtractorConfig {
     this.typescriptMode = that.typescriptMode;
     this.typescriptRam = that.typescriptRam;
     this.defaultEncoding = that.defaultEncoding;
+    this.virtualSourceRoot = that.virtualSourceRoot;
   }
 
   public ECMAVersion getEcmaVersion() {
@@ -452,6 +457,16 @@ public class ExtractorConfig {
     return res;
   }
 
+  public VirtualSourceRoot getVirtualSourceRoot() {
+    return virtualSourceRoot;
+  }
+
+  public ExtractorConfig withVirtualSourceRoot(VirtualSourceRoot virtualSourceRoot) {
+    ExtractorConfig res = new ExtractorConfig(this);
+    res.virtualSourceRoot = virtualSourceRoot;
+    return res;
+  }
+
   @Override
   public String toString() {
     return "ExtractorConfig [ecmaVersion="
@@ -486,6 +501,8 @@ public class ExtractorConfig {
         + typescriptMode
         + ", defaultEncoding="
         + defaultEncoding
+        + ", virtualSourceRoot="
+        + virtualSourceRoot
         + "]";
   }
 }

javascript/extractor/src/com/semmle/js/extractor/ExtractorState.java

@@ -1,5 +1,8 @@
 package com.semmle.js.extractor;
 
+import java.nio.file.Path;
+import java.util.concurrent.ConcurrentHashMap;
+
 import com.semmle.js.parser.TypeScriptParser;
 
 /**
@@ -17,16 +20,28 @@ import com.semmle.js.parser.TypeScriptParser;
  */
 public class ExtractorState {
   private TypeScriptParser typeScriptParser = new TypeScriptParser();
+  
+  private final ConcurrentHashMap<Path, FileSnippet> snippets = new ConcurrentHashMap<>();
 
   public TypeScriptParser getTypeScriptParser() {
     return typeScriptParser;
   }
 
+  /**
+   * Returns the mapping that denotes where a snippet file originated from.
+   *
+   * <p>The map is thread-safe and may be mutated by the caller.
+   */
+  public ConcurrentHashMap<Path, FileSnippet> getSnippets() {
+    return snippets;
+  }
+
   /**
    * Makes this semantically equivalent to a fresh state, but may internally retain shared resources
    * that are expensive to reacquire.
    */
   public void reset() {
     typeScriptParser.reset();
+    snippets.clear();
   }
 }

javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java

@@ -1,5 +1,17 @@
 package com.semmle.js.extractor;
 
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.regex.Pattern;
+
 import com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase;
 import com.semmle.js.extractor.trapcache.CachingTrapWriter;
 import com.semmle.js.extractor.trapcache.ITrapCache;
@@ -10,16 +22,6 @@ import com.semmle.util.files.FileUtil;
 import com.semmle.util.io.WholeIO;
 import com.semmle.util.trap.TrapWriter;
 import com.semmle.util.trap.TrapWriter.Label;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.util.LinkedHashSet;
-import java.util.Set;
-import java.util.regex.Pattern;
 
 /**
  * The file extractor extracts a single file and handles source archive population and TRAP caching;
@@ -47,7 +49,7 @@ public class FileExtractor {
     HTML(".htm", ".html", ".xhtm", ".xhtml", ".vue") {
       @Override
       public IExtractor mkExtractor(ExtractorConfig config, ExtractorState state) {
-        return new HTMLExtractor(config);
+        return new HTMLExtractor(config, state);
       }
 
       @Override
@@ -293,7 +295,7 @@ public class FileExtractor {
 
       @Override
       public IExtractor mkExtractor(ExtractorConfig config, ExtractorState state) {
-        return new TypeScriptExtractor(config, state.getTypeScriptParser());
+        return new TypeScriptExtractor(config, state);
       }
 
       @Override
@@ -398,6 +400,10 @@ public class FileExtractor {
 
   /** @return the number of lines of code extracted, or {@code null} if the file was cached */
   public Integer extract(File f, ExtractorState state) throws IOException {
+    FileSnippet snippet = state.getSnippets().get(f.toPath());
+    if (snippet != null) {
+      return this.extractSnippet(f.toPath(), snippet, state);
+    }
 
     // populate source archive
     String source = new WholeIO(config.getDefaultEncoding()).strictread(f);
@@ -414,6 +420,25 @@ public class FileExtractor {
     return extractContents(f, fileLabel, source, locationManager, state);
   }
 
+  /**
+   * Extract the contents of a file that is a snippet from another file.
+   *
+   * <p>A trap file will be derived from the snippet file, but its file label, source locations, and
+   * source archive entry are based on the original file.
+   */
+  private Integer extractSnippet(Path file, FileSnippet origin, ExtractorState state) throws IOException {
+    TrapWriter trapwriter = outputConfig.getTrapWriterFactory().mkTrapWriter(file.toFile());
+
+    File originalFile = origin.getOriginalFile().toFile();
+    Label fileLabel = trapwriter.populateFile(originalFile);
+    LocationManager locationManager = new LocationManager(originalFile, trapwriter, fileLabel);
+    locationManager.setStart(origin.getLine(), origin.getColumn());
+
+    String source = new WholeIO(config.getDefaultEncoding()).strictread(file);
+
+    return extractContents(file.toFile(), fileLabel, source, locationManager, state);
+  }
+
   /**
    * Extract the contents of a file, potentially making use of cached information.
    *
@@ -436,20 +461,20 @@ public class FileExtractor {
    * obviously, no caching is done in that scenario.
    */
   private Integer extractContents(
-      File f, Label fileLabel, String source, LocationManager locationManager, ExtractorState state)
+      File extractedFile, Label fileLabel, String source, LocationManager locationManager, ExtractorState state)
       throws IOException {
     ExtractionMetrics metrics = new ExtractionMetrics();
     metrics.startPhase(ExtractionPhase.FileExtractor_extractContents);
     metrics.setLength(source.length());
     metrics.setFileLabel(fileLabel);
     TrapWriter trapwriter = locationManager.getTrapWriter();
-    FileType fileType = getFileType(f);
+    FileType fileType = getFileType(extractedFile);
 
     File cacheFile = null, // the cache file for this extraction
         resultFile = null; // the final result TRAP file for this extraction
 
     if (bumpIdCounter(trapwriter)) {
-      resultFile = outputConfig.getTrapWriterFactory().getTrapFileFor(f);
+      resultFile = outputConfig.getTrapWriterFactory().getTrapFileFor(extractedFile);
     }
     // check whether we can perform caching
     if (resultFile != null && fileType.isTrapCachingAllowed()) {
@@ -475,7 +500,7 @@ public class FileExtractor {
       trapwriter = new CachingTrapWriter(cacheFile, resultFile);
       bumpIdCounter(trapwriter);
       // re-initialise the location manager, since it keeps a reference to the TRAP writer
-      locationManager = new LocationManager(f, trapwriter, locationManager.getFileLabel());
+      locationManager = new LocationManager(extractedFile, trapwriter, locationManager.getFileLabel());
     }
 
     // now do the extraction itself
@@ -484,9 +509,9 @@ public class FileExtractor {
       IExtractor extractor = fileType.mkExtractor(config, state);
       TextualExtractor textualExtractor =
           new TextualExtractor(
-              trapwriter, locationManager, source, config.getExtractLines(), metrics);
+              trapwriter, locationManager, source, config.getExtractLines(), metrics, extractedFile);
       LoCInfo loc = extractor.extract(textualExtractor);
-      int numLines = textualExtractor.getNumLines();
+      int numLines = textualExtractor.isSnippet() ? 0 : textualExtractor.getNumLines();
       int linesOfCode = loc.getLinesOfCode(), linesOfComments = loc.getLinesOfComments();
       trapwriter.addTuple("numlines", fileLabel, numLines, linesOfCode, linesOfComments);
       trapwriter.addTuple("filetype", fileLabel, fileType.toString());

javascript/extractor/src/com/semmle/js/extractor/FileSnippet.java

@@ -0,0 +1,44 @@
+package com.semmle.js.extractor;
+
+import java.nio.file.Path;
+
+import com.semmle.js.extractor.ExtractorConfig.SourceType;
+
+/**
+ * Denotes where a code snippet originated from within a file.
+ */
+public class FileSnippet {
+  private Path originalFile;
+  private int line;
+  private int column;
+  private int topLevelKind;
+  private SourceType sourceType;
+
+  public FileSnippet(Path originalFile, int line, int column, int topLevelKind, SourceType sourceType) {
+    this.originalFile = originalFile;
+    this.line = line;
+    this.column = column;
+    this.topLevelKind = topLevelKind;
+    this.sourceType = sourceType;
+  }
+
+  public Path getOriginalFile() {
+    return originalFile;
+  }
+
+  public int getLine() {
+    return line;
+  }
+
+  public int getColumn() {
+    return column;
+  }
+
+  public int getTopLevelKind() {
+    return topLevelKind;
+  }
+
+  public SourceType getSourceType() {
+    return sourceType;
+  }
+}

javascript/extractor/src/com/semmle/js/extractor/HTMLExtractor.java

@@ -1,12 +1,17 @@
 package com.semmle.js.extractor;
 
+import java.io.File;
+import java.nio.file.Path;
+import java.util.regex.Pattern;
+
 import com.semmle.js.extractor.ExtractorConfig.Platform;
 import com.semmle.js.extractor.ExtractorConfig.SourceType;
 import com.semmle.js.parser.ParseError;
 import com.semmle.util.data.StringUtil;
+import com.semmle.util.io.WholeIO;
 import com.semmle.util.trap.TrapWriter;
 import com.semmle.util.trap.TrapWriter.Label;
-import java.util.regex.Pattern;
+
 import net.htmlparser.jericho.Attribute;
 import net.htmlparser.jericho.Attributes;
 import net.htmlparser.jericho.CharacterReference;
@@ -26,9 +31,11 @@ public class HTMLExtractor implements IExtractor {
           Pattern.CASE_INSENSITIVE);
 
   private final ExtractorConfig config;
+  private final ExtractorState state;
 
-  public HTMLExtractor(ExtractorConfig config) {
+  public HTMLExtractor(ExtractorConfig config, ExtractorState state) {
     this.config = config.withPlatform(Platform.WEB);
+    this.state = state;
   }
 
   @Override
@@ -49,7 +56,7 @@ public class HTMLExtractor implements IExtractor {
     for (Element elt : src.getAllElements()) {
       LoCInfo snippetLoC = null;
       if (elt.getName().equals(HTMLElementName.SCRIPT)) {
-        SourceType sourceType = getScriptSourceType(elt);
+        SourceType sourceType = getScriptSourceType(elt, textualExtractor.getExtractedFile());
         if (sourceType != null) {
           // Jericho sometimes misparses empty elements, which will show up as start tags
           // ending in "/"; we manually exclude these cases to avoid spurious syntax errors
@@ -57,6 +64,7 @@ public class HTMLExtractor implements IExtractor {
 
           Segment content = elt.getContent();
           String source = content.toString();
+          boolean isTypeScript = isTypeScriptTag(elt);
 
           /*
            * Script blocks in XHTML files may wrap (parts of) their code inside CDATA sections.
@@ -79,7 +87,8 @@ public class HTMLExtractor implements IExtractor {
                     textualExtractor,
                     source,
                     contentStart.getRow(),
-                    contentStart.getColumn());
+                    contentStart.getColumn(),
+                    isTypeScript);
           }
         }
       } else {
@@ -101,7 +110,8 @@ public class HTMLExtractor implements IExtractor {
                       textualExtractor,
                       source,
                       valueStart.getRow(),
-                      valueStart.getColumn());
+                      valueStart.getColumn(),
+                      false /* isTypeScript */);
             } else if (source.startsWith("javascript:")) {
               source = source.substring(11);
               snippetLoC =
@@ -112,7 +122,8 @@ public class HTMLExtractor implements IExtractor {
                       textualExtractor,
                       source,
                       valueStart.getRow(),
-                      valueStart.getColumn() + 11);
+                      valueStart.getColumn() + 11,
+                      false /* isTypeScript */);
             }
           }
       }
@@ -139,16 +150,23 @@ public class HTMLExtractor implements IExtractor {
    * Deduce the {@link SourceType} with which the given <code>script</code> element should be
    * extracted, returning <code>null</code> if it cannot be determined.
    */
-  private SourceType getScriptSourceType(Element script) {
+  private SourceType getScriptSourceType(Element script, File file) {
     String scriptType = getAttributeValueLC(script, "type");
-    String scriptLanguage = getAttributeValueLC(script, "language");
+    String scriptLanguage = getScriptLanguage(script);
+    
+    SourceType fallbackSourceType = config.getSourceType();
+    if (file.getName().endsWith(".vue")) {
+      fallbackSourceType = SourceType.MODULE;
+    }
+
+    if (isTypeScriptTag(script)) return fallbackSourceType;
 
     // if `type` and `language` are both either missing, contain the
     // string "javascript", or if `type` is the string "text/jsx", this is a plain script
     if ((scriptType == null || scriptType.contains("javascript") || "text/jsx".equals(scriptType))
         && (scriptLanguage == null || scriptLanguage.contains("javascript")))
       // use default source type
-      return config.getSourceType();
+      return fallbackSourceType;
 
     // if `type` is "text/babel", the source type depends on the `data-plugins` attribute
     if ("text/babel".equals(scriptType)) {
@@ -156,7 +174,7 @@ public class HTMLExtractor implements IExtractor {
       if (plugins != null && plugins.contains("transform-es2015-modules-umd")) {
         return SourceType.MODULE;
       }
-      return config.getSourceType();
+      return fallbackSourceType;
     }
 
     // if `type` is "module", extract as module
@@ -165,6 +183,23 @@ public class HTMLExtractor implements IExtractor {
     return null;
   }
 
+  private String getScriptLanguage(Element script) {
+    String scriptLanguage = getAttributeValueLC(script, "language");
+
+    if (scriptLanguage == null) { // Vue templates use 'lang' instead of 'language'.
+      scriptLanguage = getAttributeValueLC(script, "lang");
+    }
+    return scriptLanguage;
+  }
+
+  private boolean isTypeScriptTag(Element script) {
+    String language = getScriptLanguage(script);
+    if ("ts".equals(language) || "typescript".equals(language)) return true;
+    String type = getAttributeValueLC(script, "type");
+    if (type != null && type.contains("typescript")) return true;
+    return false;
+  }
+
   /**
    * Get the value of attribute <code>attr</code> of element <code>elt</code> in lower case; if the
    * attribute has no value, <code>null</code> is returned.
@@ -181,7 +216,27 @@ public class HTMLExtractor implements IExtractor {
       TextualExtractor textualExtractor,
       String source,
       int line,
-      int column) {
+      int column,
+      boolean isTypeScript) {
+    if (isTypeScript) {
+      Path file = textualExtractor.getExtractedFile().toPath();
+      FileSnippet snippet = new FileSnippet(file, line, column, toplevelKind, config.getSourceType());
+      VirtualSourceRoot vroot = config.getVirtualSourceRoot();
+      // Vue files are special in that they can be imported as modules, and may only contain one <script> tag.
+      // For .vue files we omit the usual snippet decoration to ensure the TypeScript compiler can find it.
+      Path virtualFile =
+          file.getFileName().toString().endsWith(".vue")
+          ? vroot.toVirtualFile(file.resolveSibling(file.getFileName() + ".ts"))
+          : vroot.getVirtualFileForSnippet(snippet, ".ts");
+      if (virtualFile != null) {
+        virtualFile = virtualFile.toAbsolutePath().normalize();
+        synchronized(vroot.getLock()) {
+          new WholeIO().strictwrite(virtualFile, source);
+        }
+        state.getSnippets().put(virtualFile, snippet);
+      }
+      return null; // LoC info is accounted for later
+    }
     TrapWriter trapwriter = textualExtractor.getTrapwriter();
     LocationManager locationManager = textualExtractor.getLocationManager();
     LocationManager scriptLocationManager =
@@ -196,7 +251,8 @@ public class HTMLExtractor implements IExtractor {
               scriptLocationManager,
               source,
               config.getExtractLines(),
-              textualExtractor.getMetrics());
+              textualExtractor.getMetrics(),
+              textualExtractor.getExtractedFile());
       return extractor.extract(tx, source, toplevelKind, scopeManager).snd();
     } catch (ParseError e) {
       e.setPosition(scriptLocationManager.translatePosition(e.getPosition()));

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -2,6 +2,8 @@ package com.semmle.js.extractor;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -31,6 +33,8 @@ import com.semmle.util.io.WholeIO;
 import com.semmle.util.language.LegacyLanguage;
 import com.semmle.util.process.ArgsParser;
 import com.semmle.util.process.ArgsParser.FileMode;
+import com.semmle.util.process.Env;
+import com.semmle.util.process.Env.Var;
 import com.semmle.util.trap.TrapWriter;
 
 /** The main entry point of the JavaScript extractor. */
@@ -134,12 +138,6 @@ public class Main {
       return;
     }
 
-    TypeScriptParser tsParser = extractorState.getTypeScriptParser();
-    tsParser.setTypescriptRam(extractorConfig.getTypeScriptRam());
-    if (containsTypeScriptFiles()) {
-      tsParser.verifyInstallation(!ap.has(P_QUIET));
-    }
-
     // Sort files for determinism
     projectFiles = projectFiles.stream()
           .sorted(AutoBuild.FILE_ORDERING)
@@ -149,16 +147,30 @@ public class Main {
         .sorted(AutoBuild.FILE_ORDERING)
         .collect(Collectors.toCollection(() -> new LinkedHashSet<>()));
 
+    // Extract HTML files first, as they may contain embedded TypeScript code
+    for (File file : files) {
+      if (FileType.forFile(file, extractorConfig) == FileType.HTML) {
+        ensureFileIsExtracted(file, ap);
+      }
+    }
+    
+    TypeScriptParser tsParser = extractorState.getTypeScriptParser();
+    tsParser.setTypescriptRam(extractorConfig.getTypeScriptRam());
+    if (containsTypeScriptFiles()) {
+      tsParser.verifyInstallation(!ap.has(P_QUIET));
+    }
+
     for (File projectFile : projectFiles) {
 
       long start = verboseLogStartTimer(ap, "Opening project " + projectFile);
-      ParsedProject project = tsParser.openProject(projectFile, DependencyInstallationResult.empty);
+      ParsedProject project = tsParser.openProject(projectFile, DependencyInstallationResult.empty, extractorConfig.getVirtualSourceRoot());
       verboseLogEndTimer(ap, start);
       // Extract all files belonging to this project which are also matched
       // by our include/exclude filters.
       List<File> filesToExtract = new ArrayList<>();
       for (File sourceFile : project.getOwnFiles()) {
-        if (files.contains(normalizeFile(sourceFile))
+        File normalizedFile = normalizeFile(sourceFile);
+        if ((files.contains(normalizedFile) || extractorState.getSnippets().containsKey(normalizedFile.toPath()))
             && !extractedFiles.contains(sourceFile.getAbsoluteFile())
             && FileType.TYPESCRIPT.getExtensions().contains(FileUtil.extension(sourceFile))) {
           filesToExtract.add(sourceFile);
@@ -287,10 +299,14 @@ public class Main {
   }
 
   public void collectFiles(ArgsParser ap) {
-    for (File f : ap.getOneOrMoreFiles("files", FileMode.FILE_OR_DIRECTORY_MUST_EXIST))
+    for (File f : getFilesArg(ap))
       collectFiles(f, true);
   }
 
+  private List<File> getFilesArg(ArgsParser ap) {
+    return ap.getOneOrMoreFiles("files", FileMode.FILE_OR_DIRECTORY_MUST_EXIST);
+  }
+
   public void setupMatchers(ArgsParser ap) {
     Set<String> includes = new LinkedHashSet<>();
 
@@ -444,6 +460,21 @@ public class Main {
     if (ap.has(P_TYPESCRIPT)) return TypeScriptMode.BASIC;
     return TypeScriptMode.NONE;
   }
+  
+  private Path inferSourceRoot(ArgsParser ap) {
+    List<File> files = getFilesArg(ap);
+    Path sourceRoot = files.iterator().next().toPath().toAbsolutePath().getParent();
+    for (File file : files) {
+      Path path = file.toPath().toAbsolutePath().getParent();
+      for (int i = 0; i < sourceRoot.getNameCount(); ++i) {
+        if (!(i < path.getNameCount() && path.getName(i).equals(sourceRoot.getName(i)))) {
+          sourceRoot = sourceRoot.subpath(0, i);
+          break;
+        }
+      }
+    }
+    return sourceRoot;
+  }
 
   private ExtractorConfig parseJSOptions(ArgsParser ap) {
     ExtractorConfig cfg =
@@ -466,6 +497,17 @@ public class Main {
                     ? UnitParser.parseOpt(ap.getString(P_TYPESCRIPT_RAM), UnitParser.MEGABYTES)
                     : 0);
     if (ap.has(P_DEFAULT_ENCODING)) cfg = cfg.withDefaultEncoding(ap.getString(P_DEFAULT_ENCODING));
+
+    // Make a usable virtual source root mapping.
+    // The concept of source root and scratch directory do not exist in the legacy extractor,
+    // so we construct these based on what we have.
+    String odasaDbDir = Env.systemEnv().getNonEmpty(Var.ODASA_DB);
+    VirtualSourceRoot virtualSourceRoot =
+        odasaDbDir == null
+            ? VirtualSourceRoot.none
+            : new VirtualSourceRoot(inferSourceRoot(ap), Paths.get(odasaDbDir, "working"));
+    cfg = cfg.withVirtualSourceRoot(virtualSourceRoot);
+
     return cfg;
   }
 

javascript/extractor/src/com/semmle/js/extractor/TextualExtractor.java

@@ -1,11 +1,13 @@
 package com.semmle.js.extractor;
 
+import java.io.File;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
 import com.semmle.js.ast.Position;
 import com.semmle.js.ast.SourceElement;
 import com.semmle.util.trap.TrapWriter;
 import com.semmle.util.trap.TrapWriter.Label;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 /**
  * Extractor for populating purely textual information about a file, namely its lines and their line
@@ -21,19 +23,40 @@ public class TextualExtractor {
   private final Label fileLabel;
   private final boolean extractLines;
   private final ExtractionMetrics metrics;
+  private final File extractedFile;
 
   public TextualExtractor(
       TrapWriter trapwriter,
       LocationManager locationManager,
       String source,
       boolean extractLines,
-      ExtractionMetrics metrics) {
+      ExtractionMetrics metrics,
+      File extractedFile) {
     this.trapwriter = trapwriter;
     this.locationManager = locationManager;
     this.source = source;
     this.fileLabel = locationManager.getFileLabel();
     this.extractLines = extractLines;
     this.metrics = metrics;
+    this.extractedFile = extractedFile;
+  }
+
+  /**
+   * Returns the file whose contents should be extracted, and is contained
+   * in {@link #source}.
+   *
+   * <p>This may differ from the source file of the location manager, which refers
+   * to the original file that this was derived from.
+   */
+  public File getExtractedFile() {
+    return extractedFile;
+  }
+
+  /**
+   * Returns true if the extracted file and the source location files are different.
+   */
+  public boolean isSnippet() {
+    return !extractedFile.equals(locationManager.getSourceFile());
   }
 
   public TrapWriter getTrapwriter() {

javascript/extractor/src/com/semmle/js/extractor/TypeScriptExtractor.java

@@ -1,32 +1,34 @@
 package com.semmle.js.extractor;
 
+import java.io.File;
+
 import com.semmle.js.extractor.ExtractorConfig.ECMAVersion;
 import com.semmle.js.extractor.ExtractorConfig.SourceType;
 import com.semmle.js.parser.JSParser.Result;
 import com.semmle.js.parser.ParseError;
-import com.semmle.js.parser.TypeScriptParser;
-import java.io.File;
 
 public class TypeScriptExtractor implements IExtractor {
   private final JSExtractor jsExtractor;
-  private final TypeScriptParser parser;
+  private final ExtractorState state;
 
-  public TypeScriptExtractor(ExtractorConfig config, TypeScriptParser parser) {
+  public TypeScriptExtractor(ExtractorConfig config, ExtractorState state) {
     this.jsExtractor = new JSExtractor(config);
-    this.parser = parser;
+    this.state = state;
   }
 
   @Override
   public LoCInfo extract(TextualExtractor textualExtractor) {
     LocationManager locationManager = textualExtractor.getLocationManager();
     String source = textualExtractor.getSource();
-    File sourceFile = locationManager.getSourceFile();
+    File sourceFile = textualExtractor.getExtractedFile();
-    Result res = parser.parse(sourceFile, source, textualExtractor.getMetrics());
+    Result res = state.getTypeScriptParser().parse(sourceFile, source, textualExtractor.getMetrics());
     ScopeManager scopeManager =
         new ScopeManager(textualExtractor.getTrapwriter(), ECMAVersion.ECMA2017);
     try {
-      SourceType sourceType = jsExtractor.establishSourceType(source, false);
+      FileSnippet snippet = state.getSnippets().get(sourceFile.toPath());
-      return jsExtractor.extract(textualExtractor, source, 0, scopeManager, sourceType, res).snd();
+      SourceType sourceType = snippet != null ? snippet.getSourceType() : jsExtractor.establishSourceType(source, false);
+      int toplevelKind = snippet != null ? snippet.getTopLevelKind() : 0;
+      return jsExtractor.extract(textualExtractor, source, toplevelKind, scopeManager, sourceType, res).snd();
     } catch (ParseError e) {
       e.setPosition(locationManager.translatePosition(e.getPosition()));
       throw e.asUserError();

javascript/extractor/src/com/semmle/js/extractor/VirtualSourceRoot.java

@@ -0,0 +1,80 @@
+package com.semmle.js.extractor;
+
+import java.nio.file.Path;
+
+public class VirtualSourceRoot {
+  private Path sourceRoot;
+  private Path virtualSourceRoot;
+  private Object lock = new Object();
+
+  public static final VirtualSourceRoot none = new VirtualSourceRoot(null, null);
+
+  public VirtualSourceRoot(Path sourceRoot, Path virtualSourceRoot) {
+    this.sourceRoot = sourceRoot;
+    this.virtualSourceRoot = virtualSourceRoot;
+  }
+
+  /**
+   * Returns the source root mirrored by {@link #getVirtualSourceRoot()} or <code>null</code> if no
+   * virtual source root exists.
+   *
+   * <p>When invoked from the AutoBuilder, this corresponds to the source root. When invoked from
+   * ODASA, there is no notion of source root, so this is always <code>null</code> in that context.
+   */
+  public Path getSourceRoot() {
+    return sourceRoot;
+  }
+
+  /**
+   * Returns the virtual source root or <code>null</code> if no virtual source root exists.
+   *
+   * <p>The virtual source root is a directory hierarchy that mirrors the real source root, where
+   * dependencies are installed.
+   */
+  public Path getVirtualSourceRoot() {
+    return virtualSourceRoot;
+  }
+
+  private static Path translate(Path oldRoot, Path newRoot, Path file) {
+    if (oldRoot == null || newRoot == null) return null;
+    Path relative = oldRoot.relativize(file);
+    if (relative.startsWith("..") || relative.isAbsolute()) return null;
+    return newRoot.resolve(relative);
+  }
+
+  public Path toVirtualFile(Path file) {
+    if (file.startsWith(virtualSourceRoot)) {
+      // 'qltest' creates a virtual source root inside the real source root.
+      // Make sure such files don't appear to be inside the real source root.
+      return null;
+    }
+    return translate(sourceRoot, virtualSourceRoot, file);
+  }
+
+  public Path fromVirtualFile(Path file) {
+    return translate(virtualSourceRoot, sourceRoot, file);
+  }
+
+  public Path getVirtualFileForSnippet(FileSnippet snippet, String extension) {
+    String basename =
+        snippet.getOriginalFile().getFileName()
+            + ".snippet."
+            + snippet.getLine()
+            + "."
+            + snippet.getColumn()
+            + extension;
+    return toVirtualFile(snippet.getOriginalFile().resolveSibling(basename));
+  }
+
+  @Override
+  public String toString() {
+    return "[sourceRoot=" + sourceRoot + ", virtualSourceRoot=" + virtualSourceRoot + "]";
+  }
+
+  /**
+   * Gets the lock to use when writing to the virtual source root in a multi-threaded context.
+   */
+  public Object getLock() {
+    return lock;
+  }
+}

javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java

@@ -15,6 +15,7 @@ import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
 
 import org.junit.After;
 import org.junit.Assert;
@@ -27,6 +28,7 @@ import com.semmle.js.extractor.DependencyInstallationResult;
 import com.semmle.js.extractor.ExtractorState;
 import com.semmle.js.extractor.FileExtractor;
 import com.semmle.js.extractor.FileExtractor.FileType;
+import com.semmle.js.extractor.VirtualSourceRoot;
 import com.semmle.util.data.StringUtil;
 import com.semmle.util.exception.UserError;
 import com.semmle.util.files.FileUtil;
@@ -109,11 +111,12 @@ public class AutoBuildTests {
       Set<String> actual = new LinkedHashSet<>();
       new AutoBuild() {
         @Override
-        protected void extract(FileExtractor extractor, Path file, ExtractorState state) {
+        protected CompletableFuture<?> extract(FileExtractor extractor, Path file, boolean concurrent) {
           String extracted = file.toString();
           if (extractor.getConfig().hasFileType())
             extracted += ":" + extractor.getFileType(file.toFile());
           actual.add(extracted);
+          return CompletableFuture.completedFuture(null);
         }
 
         @Override
@@ -123,8 +126,7 @@ public class AutoBuildTests {
         public void extractTypeScriptFiles(
             java.util.List<Path> files,
             java.util.Set<Path> extractedFiles,
-            FileExtractor extractor,
+            FileExtractors extractors) {
-            ExtractorState extractorState) {
           for (Path f : files) {
             actual.add(f.toString());
           }
@@ -136,6 +138,11 @@ public class AutoBuildTests {
           return DependencyInstallationResult.empty;
         }
 
+        @Override
+        protected VirtualSourceRoot makeVirtualSourceRoot() {
+          return VirtualSourceRoot.none; // not used in these tests
+        }
+
         @Override
         protected void extractXml() throws IOException {
           Files.walkFileTree(

javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java

@@ -31,6 +31,7 @@ import com.google.gson.JsonPrimitive;
 import com.semmle.js.extractor.DependencyInstallationResult;
 import com.semmle.js.extractor.EnvironmentVariables;
 import com.semmle.js.extractor.ExtractionMetrics;
+import com.semmle.js.extractor.VirtualSourceRoot;
 import com.semmle.js.parser.JSParser.Result;
 import com.semmle.ts.extractor.TypeTable;
 import com.semmle.util.data.StringUtil;
@@ -501,8 +502,8 @@ public class TypeScriptParser {
   /**
    * Returns the set of files included by the inclusion pattern in the given tsconfig.json file.
    */
-  public Set<File> getOwnFiles(File tsConfigFile, DependencyInstallationResult deps) {
+  public Set<File> getOwnFiles(File tsConfigFile, DependencyInstallationResult deps, VirtualSourceRoot vroot) {
-    JsonObject request = makeLoadCommand("get-own-files", tsConfigFile, deps);
+    JsonObject request = makeLoadCommand("get-own-files", tsConfigFile, deps, vroot);
     JsonObject response = talkToParserWrapper(request);
     try {
       checkResponseType(response, "file-list");
@@ -521,8 +522,8 @@ public class TypeScriptParser {
    *
    * <p>Only one project should be opened at once.
    */
-  public ParsedProject openProject(File tsConfigFile, DependencyInstallationResult deps) {
+  public ParsedProject openProject(File tsConfigFile, DependencyInstallationResult deps, VirtualSourceRoot vroot) {
-    JsonObject request = makeLoadCommand("open-project", tsConfigFile, deps);
+    JsonObject request = makeLoadCommand("open-project", tsConfigFile, deps, vroot);
     JsonObject response = talkToParserWrapper(request);
     try {
       checkResponseType(response, "project-opened");
@@ -536,18 +537,18 @@ public class TypeScriptParser {
     }
   }
 
-  private JsonObject makeLoadCommand(String command, File tsConfigFile, DependencyInstallationResult deps) {
+  private JsonObject makeLoadCommand(String command, File tsConfigFile, DependencyInstallationResult deps, VirtualSourceRoot vroot) {
     JsonObject request = new JsonObject();
     request.add("command", new JsonPrimitive(command));
     request.add("tsConfig", new JsonPrimitive(tsConfigFile.getPath()));
     request.add("packageEntryPoints", mapToArray(deps.getPackageEntryPoints()));
     request.add("packageJsonFiles", mapToArray(deps.getPackageJsonFiles()));
-    request.add("sourceRoot", deps.getSourceRoot() == null
+    request.add("sourceRoot", vroot.getSourceRoot() == null
         ? JsonNull.INSTANCE
-        : new JsonPrimitive(deps.getSourceRoot().toString()));
+        : new JsonPrimitive(vroot.getSourceRoot().toString()));
-    request.add("virtualSourceRoot", deps.getVirtualSourceRoot() == null
+    request.add("virtualSourceRoot", vroot.getVirtualSourceRoot() == null
         ? JsonNull.INSTANCE
-        : new JsonPrimitive(deps.getVirtualSourceRoot().toString()));
+        : new JsonPrimitive(vroot.getVirtualSourceRoot().toString()));
     return request;
   }
 

javascript/ql/src/semmle/javascript/Files.qll

@@ -206,13 +206,13 @@ class File extends Container, @file {
   override string getAbsolutePath() { files(this, result, _, _, _) }
 
   /** Gets the number of lines in this file. */
-  int getNumberOfLines() { numlines(this, result, _, _) }
+  int getNumberOfLines() { result = sum(int loc | numlines(this, loc, _, _) | loc) }
 
   /** Gets the number of lines containing code in this file. */
-  int getNumberOfLinesOfCode() { numlines(this, _, result, _) }
+  int getNumberOfLinesOfCode() { result = sum(int loc | numlines(this, _, loc, _) | loc) }
 
   /** Gets the number of lines containing comments in this file. */
-  int getNumberOfLinesOfComments() { numlines(this, _, _, result) }
+  int getNumberOfLinesOfComments() { result = sum(int loc | numlines(this, _, _, loc) | loc) }
 
   /** Gets a toplevel piece of JavaScript code in this file. */
   TopLevel getATopLevel() { result.getFile() = this }

javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/Test.expected

@@ -0,0 +1,26 @@
+classDeclaration
+| test.vue:3:18:5:3 | class M ... er;\\n  } |
+exprType
+| htmlfile.html:4:22:4:24 | foo | () => void |
+| htmlfile.html:4:22:4:24 | foo | () => void |
+| htmlfile.html:4:33:4:41 | "./other" | any |
+| htmlfile.html:5:17:5:22 | result | number[] |
+| htmlfile.html:5:26:5:28 | foo | () => void |
+| htmlfile.html:5:26:5:30 | foo() | void |
+| htmlfile.html:5:26:5:42 | foo() as number[] | number[] |
+| other.ts:1:8:1:16 | Component | typeof default in library-tests/TypeScript/EmbeddedInScript/test.vue |
+| other.ts:1:23:1:34 | "./test.vue" | any |
+| other.ts:3:1:3:15 | new Component() | MyComponent |
+| other.ts:3:5:3:13 | Component | typeof default in library-tests/TypeScript/EmbeddedInScript/test.vue |
+| other.ts:5:17:5:19 | foo | () => void |
+| test.vue:2:15:2:19 | other | typeof library-tests/TypeScript/EmbeddedInScript/other.ts |
+| test.vue:2:26:2:34 | "./other" | any |
+| test.vue:3:24:3:34 | MyComponent | MyComponent |
+| test.vue:4:7:4:7 | x | number |
+symbols
+| other.ts:1:1:6:0 | <toplevel> | library-tests/TypeScript/EmbeddedInScript/other.ts |
+| test.vue:2:3:6:0 | <toplevel> | library-tests/TypeScript/EmbeddedInScript/test.vue |
+importTarget
+| htmlfile.html:4:13:4:42 | import  ... other"; | other.ts:1:1:6:0 | <toplevel> |
+| other.ts:1:1:1:35 | import  ... t.vue"; | test.vue:2:3:6:0 | <toplevel> |
+| test.vue:2:3:2:35 | import  ... other"; | other.ts:1:1:6:0 | <toplevel> |

javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/Test.ql

@@ -0,0 +1,9 @@
+import javascript
+
+query ClassDefinition classDeclaration() { any() }
+
+query Type exprType(Expr e) { result = e.getType() }
+
+query predicate symbols(Module mod, CanonicalName name) { ast_node_symbol(mod, name) }
+
+query predicate importTarget(Import imprt, Module mod) { imprt.getImportedModule() = mod }

javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/htmlfile.html

@@ -0,0 +1,8 @@
+<html>
+    <body>
+        <script type="module" language="typescript">
+            import { foo } from "./other";
+            let result = foo() as number[];
+        </script>
+    </body>
+</html>

javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/other.ts

@@ -0,0 +1,5 @@
+import Component from "./test.vue";
+
+new Component();
+
+export function foo() {};

javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/test.vue

@@ -0,0 +1,6 @@
+<script lang='ts'>
+  import * as other from "./other";
+  export default class MyComponent {
+      x!: number;
+  }
+</script>

javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/tsconfig.json

@@ -0,0 +1,3 @@
+{
+    "include": ["."]
+}