Merge pull request #3832 from asger-semmle/js/typescript-in-html-files3

Approved by erik-krogh
2025-12-16 16:53:25 +01:00 · 2020-07-02 08:30:45 +01:00
parent 2bd84a3a5e 7a2c65f638
commit bfb734e1d7
25 changed files with 595 additions and 157 deletions
--- a/change-notes/1.25/analysis-javascript.md
+++ b/change-notes/1.25/analysis-javascript.md
@@ -29,6 +29,8 @@

 * TypeScript 3.9 is now supported.

+* TypeScript code embedded in HTML and Vue files is now extracted and analyzed.
+
 * The analysis of sanitizers has improved, leading to more accurate
  results from the security queries.

--- a/javascript/extractor/lib/typescript/src/common.ts
+++ b/javascript/extractor/lib/typescript/src/common.ts
@@ -48,7 +48,7 @@ export class Project {
  public load(): void {
    const { config, host } = this;
    this.program = ts.createProgram(config.fileNames, config.options, host);
-    this.typeTable.setProgram(this.program);
+    this.typeTable.setProgram(this.program, this.virtualSourceRoot);
  }

  /**
@@ -71,10 +71,19 @@ export class Project {
        redirectedReference: ts.ResolvedProjectReference,
        options: ts.CompilerOptions) {

+    let oppositePath =
+      this.virtualSourceRoot.toVirtualPath(containingFile) ||
+      this.virtualSourceRoot.fromVirtualPath(containingFile);
+
    const { host, resolutionCache } = this;
    return moduleNames.map((moduleName) => {
      let redirected = this.redirectModuleName(moduleName, containingFile, options);
      if (redirected != null) return redirected;
+      if (oppositePath != null) {
+        // If the containing file is in the virtual source root, try resolving from the real source root, and vice versa.
+        redirected = ts.resolveModuleName(moduleName, oppositePath, options,  host, resolutionCache).resolvedModule;
+        if (redirected != null) return redirected;
+      }
      return ts.resolveModuleName(moduleName, containingFile, options,  host, resolutionCache).resolvedModule;
    });
  }
@@ -90,15 +99,7 @@ export class Project {

    // Get the overridden location of this package, if one exists.
    let packageEntryPoint = this.packageEntryPoints.get(packageName);
-    if (packageEntryPoint == null) {
-      // The package is not overridden, but we have established that it begins with a valid package name.
-      // Do a lookup in the virtual source root (where dependencies are installed) by changing the 'containing file'.
-      let virtualContainingFile = this.virtualSourceRoot.toVirtualPath(containingFile);
-      if (virtualContainingFile != null) {
-        return ts.resolveModuleName(moduleName, virtualContainingFile, options, this.host, this.resolutionCache).resolvedModule;
-      }
-      return null;
-    }
+    if (packageEntryPoint == null) return null;

    // If the requested module name is exactly the overridden package name,
    // return the entry point file (it is not necessarily called `index.ts`).
--- a/javascript/extractor/lib/typescript/src/main.ts
+++ b/javascript/extractor/lib/typescript/src/main.ts
@@ -414,7 +414,25 @@ function loadTsConfig(command: LoadCommand): LoadedConfig {
     */
    let parseConfigHost: ts.ParseConfigHost = {
        useCaseSensitiveFileNames: true,
-        readDirectory: ts.sys.readDirectory, // No need to override traversal/glob matching
+        readDirectory: (rootDir, extensions, excludes?, includes?, depth?) => {
+            // Perform the glob matching in both real and virtual source roots.
+            let exclusions = excludes == null ? [] : [...excludes];
+            if (virtualSourceRoot.virtualSourceRoot != null) {
+                // qltest puts the virtual source root inside the real source root (.testproj).
+                // Make sure we don't find files inside the virtual source root in this pass.
+                exclusions.push(virtualSourceRoot.virtualSourceRoot);
+            }
+            let originalResults = ts.sys.readDirectory(rootDir, extensions, exclusions, includes, depth)
+            let virtualDir = virtualSourceRoot.toVirtualPath(rootDir);
+            if (virtualDir == null) {
+                return originalResults;
+            }
+            // Make sure glob matching does not to discover anything in node_modules.
+            let virtualExclusions = excludes == null ? [] : [...excludes];
+            virtualExclusions.push('**/node_modules/**/*');
+            let virtualResults = ts.sys.readDirectory(virtualDir, extensions, virtualExclusions, includes, depth)
+            return [ ...originalResults, ...virtualResults ];
+        },
        fileExists: (path: string) => {
            return ts.sys.fileExists(path)
                || virtualSourceRoot.toVirtualPathIfFileExists(path) != null
--- a/javascript/extractor/lib/typescript/src/type_table.ts
+++ b/javascript/extractor/lib/typescript/src/type_table.ts
@@ -1,4 +1,5 @@
 import * as ts from "./typescript";
+import { VirtualSourceRoot } from "./virtual_source_root";

 interface AugmentedSymbol extends ts.Symbol {
  parent?: AugmentedSymbol;
@@ -379,12 +380,15 @@ export class TypeTable {
   */
  public restrictedExpansion = false;

+  private virtualSourceRoot: VirtualSourceRoot;
+
  /**
   * Called when a new compiler instance has started.
   */
-  public setProgram(program: ts.Program) {
+  public setProgram(program: ts.Program, virtualSourceRoot: VirtualSourceRoot) {
    this.typeChecker = program.getTypeChecker();
    this.arbitraryAstNode = program.getSourceFiles()[0];
+    this.virtualSourceRoot = virtualSourceRoot;
  }

  /**
@@ -703,14 +707,21 @@ export class TypeTable {
  private getSymbolString(symbol: AugmentedSymbol): string {
    let parent = symbol.parent;
    if (parent == null || parent.escapedName === ts.InternalSymbolName.Global) {
-      return "root;" + this.getSymbolDeclarationString(symbol) + ";;" + symbol.name;
+      return "root;" + this.getSymbolDeclarationString(symbol) + ";;" + this.rewriteSymbolName(symbol);
    } else if (parent.exports != null && parent.exports.get(symbol.escapedName) === symbol) {
-      return "member;;" + this.getSymbolId(parent) + ";" + symbol.name;
+      return "member;;" + this.getSymbolId(parent) + ";" + this.rewriteSymbolName(symbol);
    } else {
-      return "other;" + this.getSymbolDeclarationString(symbol) + ";" + this.getSymbolId(parent) + ";" + symbol.name;
+      return "other;" + this.getSymbolDeclarationString(symbol) + ";" + this.getSymbolId(parent) + ";" + this.rewriteSymbolName(symbol);
    }
  }

+  private rewriteSymbolName(symbol: AugmentedSymbol) {
+    let { virtualSourceRoot, sourceRoot } = this.virtualSourceRoot;
+    let { name } = symbol;
+    if (virtualSourceRoot == null || sourceRoot == null) return name;
+    return name.replace(virtualSourceRoot, sourceRoot);
+  }
+
  /**
   * Gets a string that distinguishes the given symbol from symbols with different
   * lexical roots, or an empty string if the symbol is not a lexical root.
--- a/javascript/extractor/lib/typescript/src/virtual_source_root.ts
+++ b/javascript/extractor/lib/typescript/src/virtual_source_root.ts
@@ -7,23 +7,42 @@ import * as ts from "./typescript";
 */
 export class VirtualSourceRoot {
  constructor(
-    private sourceRoot: string | null,
+    public sourceRoot: string | null,

    /**
     * Directory whose folder structure mirrors the real source root, but with `node_modules` installed,
     * or undefined if no virtual source root exists.
     */
-    private virtualSourceRoot: string | null,
+    public virtualSourceRoot: string | null,
  ) {}

+  private static translate(oldRoot: string, newRoot: string, path: string) {
+    if (!oldRoot || !newRoot) return null;
+    let relative = pathlib.relative(oldRoot, path);
+    if (relative.startsWith('..') || pathlib.isAbsolute(relative)) return null;
+    return pathlib.join(newRoot, relative);
+  }
+
  /**
   * Maps a path under the real source root to the corresponding path in the virtual source root.
+   *
+   * Returns `null` for paths already in the virtual source root.
   */
  public toVirtualPath(path: string) {
-    if (!this.virtualSourceRoot || !this.sourceRoot) return null;
-    let relative = pathlib.relative(this.sourceRoot, path);
-    if (relative.startsWith('..') || pathlib.isAbsolute(relative)) return null;
-    return pathlib.join(this.virtualSourceRoot, relative);
+    let { virtualSourceRoot } = this;
+    if (path.startsWith(virtualSourceRoot)) {
+      // 'qltest' creates a virtual source root inside the real source root.
+      // Make sure such files don't appear to be inside the real source root.
+      return null;
+    }
+    return VirtualSourceRoot.translate(this.sourceRoot, virtualSourceRoot, path);
+  }
+
+  /**
+   * Maps a path under the virtual source root to the corresponding path in the real source root.
+   */
+  public fromVirtualPath(path: string) {
+    return VirtualSourceRoot.translate(this.virtualSourceRoot, this.sourceRoot, path);
  }

  /**
--- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java
@@ -26,9 +26,11 @@ import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;

@@ -210,6 +212,8 @@ public class AutoBuild {
  private volatile boolean seenCode = false;
  private boolean installDependencies = false;
  private int installDependenciesTimeout;
+  private final VirtualSourceRoot virtualSourceRoot;
+  private ExtractorState state;

  /** The default timeout when running <code>yarn</code>, in milliseconds. */
  public static final int INSTALL_DEPENDENCIES_DEFAULT_TIMEOUT = 10 * 60 * 1000; // 10 minutes
@@ -227,9 +231,15 @@ public class AutoBuild {
        Env.systemEnv()
            .getInt(
                "LGTM_INDEX_TYPESCRIPT_INSTALL_DEPS_TIMEOUT", INSTALL_DEPENDENCIES_DEFAULT_TIMEOUT);
+    this.virtualSourceRoot = makeVirtualSourceRoot();
    setupFileTypes();
    setupXmlMode();
    setupMatchers();
+    this.state = new ExtractorState();
+  }
+
+  protected VirtualSourceRoot makeVirtualSourceRoot() {
+    return new VirtualSourceRoot(LGTM_SRC, toRealPath(Paths.get(EnvironmentVariables.getScratchDir())));
  }

  private String getEnvVar(String envVarName) {
@@ -530,7 +540,7 @@ public class AutoBuild {
          @Override
          public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
              throws IOException {
-            if (".js".equals(FileUtil.extension(file.toString()))) extract(extractor, file, null);
+            if (".js".equals(FileUtil.extension(file.toString()))) extract(extractor, file, true);
            return super.visitFile(file, attrs);
          }
        };
@@ -568,19 +578,37 @@ public class AutoBuild {
    }
  };

+  public class FileExtractors {
+    FileExtractor defaultExtractor;
+    Map<String, FileExtractor> customExtractors = new LinkedHashMap<>();
+
+    FileExtractors(FileExtractor defaultExtractor) {
+      this.defaultExtractor = defaultExtractor;
+    }
+
+    public FileExtractor forFile(Path f) {
+      return customExtractors.getOrDefault(FileUtil.extension(f), defaultExtractor);
+    }
+
+    public FileType fileType(Path f) {
+      return forFile(f).getFileType(f.toFile());
+    }
+  }
+
  /** Extract all supported candidate files that pass the filters. */
  private void extractSource() throws IOException {
    // default extractor
    FileExtractor defaultExtractor =
        new FileExtractor(mkExtractorConfig(), outputConfig, trapCache);

+    FileExtractors extractors = new FileExtractors(defaultExtractor);
+
    // custom extractor for explicitly specified file types
-    Map<String, FileExtractor> customExtractors = new LinkedHashMap<>();
    for (Map.Entry<String, FileType> spec : fileTypes.entrySet()) {
      String extension = spec.getKey();
      String fileType = spec.getValue().name();
      ExtractorConfig extractorConfig = mkExtractorConfig().withFileType(fileType);
-      customExtractors.put(extension, new FileExtractor(extractorConfig, outputConfig, trapCache));
+      extractors.customExtractors.put(extension, new FileExtractor(extractorConfig, outputConfig, trapCache));
    }

    Set<Path> filesToExtract = new LinkedHashSet<>();
@@ -599,29 +627,44 @@ public class AutoBuild {
    if (!tsconfigFiles.isEmpty()) {
      dependencyInstallationResult = this.preparePackagesAndDependencies(filesToExtract);
    }
+    Set<Path> extractedFiles = new LinkedHashSet<>();
+    
+    // Extract HTML files as they may contain TypeScript
+    CompletableFuture<?> htmlFuture = extractFiles(
+        filesToExtract, extractedFiles, extractors,
+        f -> extractors.fileType(f) == FileType.HTML);
+    
+    htmlFuture.join(); // Wait for HTML extraction to be finished.

    // extract TypeScript projects and files
-    Set<Path> extractedFiles =
-          extractTypeScript(
-              defaultExtractor, filesToExtract, tsconfigFiles, dependencyInstallationResult);
+    extractTypeScript(filesToExtract, extractedFiles,
+              extractors, tsconfigFiles, dependencyInstallationResult);

    boolean hasTypeScriptFiles = extractedFiles.size() > 0;

    // extract remaining files
+    extractFiles(
+        filesToExtract, extractedFiles, extractors,
+        f -> !(hasTypeScriptFiles && isFileDerivedFromTypeScriptFile(f, extractedFiles)));
+  }
+
+  private CompletableFuture<?> extractFiles(
+      Set<Path> filesToExtract,
+      Set<Path> extractedFiles,
+      FileExtractors extractors,
+      Predicate<Path> shouldExtract) {
+
+    List<CompletableFuture<?>> futures = new ArrayList<>();
    for (Path f : filesToExtract) {
      if (extractedFiles.contains(f))
        continue;
-      if (hasTypeScriptFiles && isFileDerivedFromTypeScriptFile(f, extractedFiles)) {
+      if (!shouldExtract.test(f)) {
        continue;
      }
      extractedFiles.add(f);
-      FileExtractor extractor = defaultExtractor;
-      if (!fileTypes.isEmpty()) {
-        String extension = FileUtil.extension(f);
-        if (customExtractors.containsKey(extension)) extractor = customExtractors.get(extension);
-      }
-      extract(extractor, f, null);
+      futures.add(extract(extractors.forFile(f), f, true));
    }
+    return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
  }

  /**
@@ -733,7 +776,6 @@ public class AutoBuild {
   */
 protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path> filesToExtract) {
    final Path sourceRoot = LGTM_SRC;
-    final Path virtualSourceRoot = toRealPath(Paths.get(EnvironmentVariables.getScratchDir()));

    // Read all package.json files and index them by name.
    Map<Path, JsonObject> packageJsonFiles = new LinkedHashMap<>();
@@ -820,8 +862,7 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>

    // Write the new package.json files to disk
    for (Path file : packageJsonFiles.keySet()) {
-      Path relativePath = sourceRoot.relativize(file);
-      Path virtualFile = virtualSourceRoot.resolve(relativePath);
+      Path virtualFile = virtualSourceRoot.toVirtualFile(file);

      try {
        Files.createDirectories(virtualFile.getParent());
@@ -836,7 +877,7 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
    // Install dependencies
    if (this.installDependencies && verifyYarnInstallation()) {
      for (Path file : packageJsonFiles.keySet()) {
-        Path virtualFile = virtualSourceRoot.resolve(sourceRoot.relativize(file));
+        Path virtualFile = virtualSourceRoot.toVirtualFile(file);
        System.out.println("Installing dependencies from " + virtualFile);
        ProcessBuilder pb =
            new ProcessBuilder(
@@ -862,7 +903,7 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
      }
    }

-    return new DependencyInstallationResult(sourceRoot, virtualSourceRoot, packageMainFile, packagesInRepo);
+    return new DependencyInstallationResult(packageMainFile, packagesInRepo);
  }

  /**
@@ -933,21 +974,20 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
    ExtractorConfig config = new ExtractorConfig(true);
    config = config.withSourceType(getSourceType());
    config = config.withTypeScriptMode(typeScriptMode);
+    config = config.withVirtualSourceRoot(virtualSourceRoot);
    if (defaultEncoding != null) config = config.withDefaultEncoding(defaultEncoding);
    return config;
  }

  private Set<Path> extractTypeScript(
-      FileExtractor extractor,
      Set<Path> files,
+      Set<Path> extractedFiles,
+      FileExtractors extractors,
      List<Path> tsconfig,
      DependencyInstallationResult deps) {
-    Set<Path> extractedFiles = new LinkedHashSet<>();
-
    if (hasTypeScriptFiles(files) || !tsconfig.isEmpty()) {
-      ExtractorState extractorState = new ExtractorState();
-      TypeScriptParser tsParser = extractorState.getTypeScriptParser();
-      verifyTypeScriptInstallation(extractorState);
+      TypeScriptParser tsParser = state.getTypeScriptParser();
+      verifyTypeScriptInstallation(state);

      // Collect all files included in a tsconfig.json inclusion pattern.
      // If a given file is referenced by multiple tsconfig files, we prefer to extract it using
@@ -955,7 +995,7 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
      Set<File> explicitlyIncludedFiles = new LinkedHashSet<>();
      if (tsconfig.size() > 1) { // No prioritization needed if there's only one tsconfig.
        for (Path projectPath : tsconfig) {
-          explicitlyIncludedFiles.addAll(tsParser.getOwnFiles(projectPath.toFile(), deps));
+          explicitlyIncludedFiles.addAll(tsParser.getOwnFiles(projectPath.toFile(), deps, virtualSourceRoot));
        }
      }

@@ -963,16 +1003,19 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
      for (Path projectPath : tsconfig) {
        File projectFile = projectPath.toFile();
        long start = logBeginProcess("Opening project " + projectFile);
-        ParsedProject project = tsParser.openProject(projectFile, deps);
+        ParsedProject project = tsParser.openProject(projectFile, deps, virtualSourceRoot);
        logEndProcess(start, "Done opening project " + projectFile);
        // Extract all files belonging to this project which are also matched
        // by our include/exclude filters.
        List<Path> typeScriptFiles = new ArrayList<Path>();
        for (File sourceFile : project.getAllFiles()) {
          Path sourcePath = sourceFile.toPath();
-          if (!files.contains(normalizePath(sourcePath))) continue;
+          Path normalizedFile = normalizePath(sourcePath);
+          if (!files.contains(normalizedFile) && !state.getSnippets().containsKey(normalizedFile)) {
+            continue;
+          }
          if (!project.getOwnFiles().contains(sourceFile) && explicitlyIncludedFiles.contains(sourceFile)) continue;
-          if (!FileType.TYPESCRIPT.getExtensions().contains(FileUtil.extension(sourcePath))) {
+          if (extractors.fileType(sourcePath) != FileType.TYPESCRIPT) {
            // For the time being, skip non-TypeScript files, even if the TypeScript
            // compiler can parse them for us.
            continue;
@@ -982,7 +1025,7 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
          }
        }
        typeScriptFiles.sort(PATH_ORDERING);
-        extractTypeScriptFiles(typeScriptFiles, extractedFiles, extractor, extractorState);
+        extractTypeScriptFiles(typeScriptFiles, extractedFiles, extractors);
        tsParser.closeProject(projectFile);
      }

@@ -996,12 +1039,12 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
      List<Path> remainingTypeScriptFiles = new ArrayList<>();
      for (Path f : files) {
        if (!extractedFiles.contains(f)
-            && FileType.forFileExtension(f.toFile()) == FileType.TYPESCRIPT) {
+            && extractors.fileType(f) == FileType.TYPESCRIPT) {
          remainingTypeScriptFiles.add(f);
        }
      }
      if (!remainingTypeScriptFiles.isEmpty()) {
-        extractTypeScriptFiles(remainingTypeScriptFiles, extractedFiles, extractor, extractorState);
+        extractTypeScriptFiles(remainingTypeScriptFiles, extractedFiles, extractors);
      }

      // The TypeScript compiler instance is no longer needed.
@@ -1087,16 +1130,15 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
  public void extractTypeScriptFiles(
      List<Path> files,
      Set<Path> extractedFiles,
-      FileExtractor extractor,
-      ExtractorState extractorState) {
+      FileExtractors extractors) {
    List<File> list = files
        .stream()
        .sorted(PATH_ORDERING)
        .map(p -> p.toFile()).collect(Collectors.toList());
-    extractorState.getTypeScriptParser().prepareFiles(list);
+    state.getTypeScriptParser().prepareFiles(list);
    for (Path path : files) {
      extractedFiles.add(path);
-      extract(extractor, path, extractorState);
+      extract(extractors.forFile(path), path, false);
    }
  }

@@ -1139,10 +1181,13 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
   * <p>If the state is {@code null}, the extraction job will be submitted to the {@link
   * #threadPool}, otherwise extraction will happen on the main thread.
   */
-  protected void extract(FileExtractor extractor, Path file, ExtractorState state) {
-    if (state == null && threadPool != null)
-      threadPool.submit(() -> doExtract(extractor, file, state));
-    else doExtract(extractor, file, state);
+  protected CompletableFuture<?> extract(FileExtractor extractor, Path file, boolean concurrent) {
+    if (concurrent && threadPool != null) {
+      return CompletableFuture.runAsync(() -> doExtract(extractor, file, state), threadPool);
+    } else {
+      doExtract(extractor, file, state);
+      return CompletableFuture.completedFuture(null);
+    }
  }

  private void doExtract(FileExtractor extractor, Path file, ExtractorState state) {
--- a/javascript/extractor/src/com/semmle/js/extractor/DependencyInstallationResult.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/DependencyInstallationResult.java
@@ -6,46 +6,19 @@ import java.util.Map;

 /** Contains the results of installing dependencies. */
 public class DependencyInstallationResult {
-  private Path sourceRoot;
-  private Path virtualSourceRoot;
  private Map<String, Path> packageEntryPoints;
  private Map<String, Path> packageJsonFiles;

  public static final DependencyInstallationResult empty =
-      new DependencyInstallationResult(null, null, Collections.emptyMap(), Collections.emptyMap());
+      new DependencyInstallationResult(Collections.emptyMap(), Collections.emptyMap());

  public DependencyInstallationResult(
-      Path sourceRoot,
-      Path virtualSourceRoot,
      Map<String, Path> packageEntryPoints,
      Map<String, Path> packageJsonFiles) {
-    this.sourceRoot = sourceRoot;
-    this.virtualSourceRoot = virtualSourceRoot;
    this.packageEntryPoints = packageEntryPoints;
    this.packageJsonFiles = packageJsonFiles;
  }

-  /**
-   * Returns the source root mirrored by {@link #getVirtualSourceRoot()} or <code>null</code>
-   * if no virtual source root exists.
-   * <p>
-   * When invoked from the AutoBuilder, this corresponds to the source root. When invoked
-   * from ODASA, there is no notion of source root, so this is always <code>null</code> in that context.
-   */
-  public Path getSourceRoot() {
-    return sourceRoot;
-  }
-
-  /**
-   * Returns the virtual source root or <code>null</code> if no virtual source root exists.
-   * <p>
-   * The virtual source root is a directory hierarchy that mirrors the real source
-   * root, where dependencies are installed.
-   */
-  public Path getVirtualSourceRoot() {
-    return virtualSourceRoot;
-  }
-
  /**
   * Returns the mapping from package names to the TypeScript file that should
   * act as its main entry point.
--- a/javascript/extractor/src/com/semmle/js/extractor/ExtractorConfig.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/ExtractorConfig.java
@@ -1,8 +1,5 @@
 package com.semmle.js.extractor;

-import com.semmle.js.parser.JcornWrapper;
-import com.semmle.util.data.StringUtil;
-import com.semmle.util.exception.UserError;
 import java.nio.charset.Charset;
 import java.nio.charset.IllegalCharsetNameException;
 import java.nio.charset.StandardCharsets;
@@ -12,6 +9,10 @@ import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.Set;

+import com.semmle.js.parser.JcornWrapper;
+import com.semmle.util.data.StringUtil;
+import com.semmle.util.exception.UserError;
+
 /**
 * Configuration options that affect the behaviour of the extractor.
 *
@@ -236,6 +237,8 @@ public class ExtractorConfig {
  /** The default character encoding to use for parsing source files. */
  private String defaultEncoding;

+  private VirtualSourceRoot virtualSourceRoot;
+
  public ExtractorConfig(boolean experimental) {
    this.ecmaVersion = experimental ? ECMAVersion.ECMA2020 : ECMAVersion.ECMA2019;
    this.platform = Platform.AUTO;
@@ -252,6 +255,7 @@ public class ExtractorConfig {
    this.typescriptMode = TypeScriptMode.NONE;
    this.e4x = experimental;
    this.defaultEncoding = StandardCharsets.UTF_8.name();
+    this.virtualSourceRoot = VirtualSourceRoot.none;
  }

  public ExtractorConfig(ExtractorConfig that) {
@@ -272,6 +276,7 @@ public class ExtractorConfig {
    this.typescriptMode = that.typescriptMode;
    this.typescriptRam = that.typescriptRam;
    this.defaultEncoding = that.defaultEncoding;
+    this.virtualSourceRoot = that.virtualSourceRoot;
  }

  public ECMAVersion getEcmaVersion() {
@@ -452,6 +457,16 @@ public class ExtractorConfig {
    return res;
  }

+  public VirtualSourceRoot getVirtualSourceRoot() {
+    return virtualSourceRoot;
+  }
+
+  public ExtractorConfig withVirtualSourceRoot(VirtualSourceRoot virtualSourceRoot) {
+    ExtractorConfig res = new ExtractorConfig(this);
+    res.virtualSourceRoot = virtualSourceRoot;
+    return res;
+  }
+
  @Override
  public String toString() {
    return "ExtractorConfig [ecmaVersion="
@@ -486,6 +501,8 @@ public class ExtractorConfig {
        + typescriptMode
        + ", defaultEncoding="
        + defaultEncoding
+        + ", virtualSourceRoot="
+        + virtualSourceRoot
        + "]";
  }
 }
--- a/javascript/extractor/src/com/semmle/js/extractor/ExtractorState.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/ExtractorState.java
@@ -1,5 +1,8 @@
 package com.semmle.js.extractor;

+import java.nio.file.Path;
+import java.util.concurrent.ConcurrentHashMap;
+
 import com.semmle.js.parser.TypeScriptParser;

 /**
@@ -17,16 +20,28 @@ import com.semmle.js.parser.TypeScriptParser;
 */
 public class ExtractorState {
  private TypeScriptParser typeScriptParser = new TypeScriptParser();
+  
+  private final ConcurrentHashMap<Path, FileSnippet> snippets = new ConcurrentHashMap<>();

  public TypeScriptParser getTypeScriptParser() {
    return typeScriptParser;
  }

+  /**
+   * Returns the mapping that denotes where a snippet file originated from.
+   *
+   * <p>The map is thread-safe and may be mutated by the caller.
+   */
+  public ConcurrentHashMap<Path, FileSnippet> getSnippets() {
+    return snippets;
+  }
+
  /**
   * Makes this semantically equivalent to a fresh state, but may internally retain shared resources
   * that are expensive to reacquire.
   */
  public void reset() {
    typeScriptParser.reset();
+    snippets.clear();
  }
 }
--- a/javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java
@@ -1,5 +1,17 @@
 package com.semmle.js.extractor;

+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.regex.Pattern;
+
 import com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase;
 import com.semmle.js.extractor.trapcache.CachingTrapWriter;
 import com.semmle.js.extractor.trapcache.ITrapCache;
@@ -10,16 +22,6 @@ import com.semmle.util.files.FileUtil;
 import com.semmle.util.io.WholeIO;
 import com.semmle.util.trap.TrapWriter;
 import com.semmle.util.trap.TrapWriter.Label;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.util.LinkedHashSet;
-import java.util.Set;
-import java.util.regex.Pattern;

 /**
 * The file extractor extracts a single file and handles source archive population and TRAP caching;
@@ -47,7 +49,7 @@ public class FileExtractor {
    HTML(".htm", ".html", ".xhtm", ".xhtml", ".vue") {
      @Override
      public IExtractor mkExtractor(ExtractorConfig config, ExtractorState state) {
-        return new HTMLExtractor(config);
+        return new HTMLExtractor(config, state);
      }

      @Override
@@ -293,7 +295,7 @@ public class FileExtractor {

      @Override
      public IExtractor mkExtractor(ExtractorConfig config, ExtractorState state) {
-        return new TypeScriptExtractor(config, state.getTypeScriptParser());
+        return new TypeScriptExtractor(config, state);
      }

      @Override
@@ -398,6 +400,10 @@ public class FileExtractor {

  /** @return the number of lines of code extracted, or {@code null} if the file was cached */
  public Integer extract(File f, ExtractorState state) throws IOException {
+    FileSnippet snippet = state.getSnippets().get(f.toPath());
+    if (snippet != null) {
+      return this.extractSnippet(f.toPath(), snippet, state);
+    }

    // populate source archive
    String source = new WholeIO(config.getDefaultEncoding()).strictread(f);
@@ -414,6 +420,25 @@ public class FileExtractor {
    return extractContents(f, fileLabel, source, locationManager, state);
  }

+  /**
+   * Extract the contents of a file that is a snippet from another file.
+   *
+   * <p>A trap file will be derived from the snippet file, but its file label, source locations, and
+   * source archive entry are based on the original file.
+   */
+  private Integer extractSnippet(Path file, FileSnippet origin, ExtractorState state) throws IOException {
+    TrapWriter trapwriter = outputConfig.getTrapWriterFactory().mkTrapWriter(file.toFile());
+
+    File originalFile = origin.getOriginalFile().toFile();
+    Label fileLabel = trapwriter.populateFile(originalFile);
+    LocationManager locationManager = new LocationManager(originalFile, trapwriter, fileLabel);
+    locationManager.setStart(origin.getLine(), origin.getColumn());
+
+    String source = new WholeIO(config.getDefaultEncoding()).strictread(file);
+
+    return extractContents(file.toFile(), fileLabel, source, locationManager, state);
+  }
+
  /**
   * Extract the contents of a file, potentially making use of cached information.
   *
@@ -436,20 +461,20 @@ public class FileExtractor {
   * obviously, no caching is done in that scenario.
   */
  private Integer extractContents(
-      File f, Label fileLabel, String source, LocationManager locationManager, ExtractorState state)
+      File extractedFile, Label fileLabel, String source, LocationManager locationManager, ExtractorState state)
      throws IOException {
    ExtractionMetrics metrics = new ExtractionMetrics();
    metrics.startPhase(ExtractionPhase.FileExtractor_extractContents);
    metrics.setLength(source.length());
    metrics.setFileLabel(fileLabel);
    TrapWriter trapwriter = locationManager.getTrapWriter();
-    FileType fileType = getFileType(f);
+    FileType fileType = getFileType(extractedFile);

    File cacheFile = null, // the cache file for this extraction
        resultFile = null; // the final result TRAP file for this extraction

    if (bumpIdCounter(trapwriter)) {
-      resultFile = outputConfig.getTrapWriterFactory().getTrapFileFor(f);
+      resultFile = outputConfig.getTrapWriterFactory().getTrapFileFor(extractedFile);
    }
    // check whether we can perform caching
    if (resultFile != null && fileType.isTrapCachingAllowed()) {
@@ -475,7 +500,7 @@ public class FileExtractor {
      trapwriter = new CachingTrapWriter(cacheFile, resultFile);
      bumpIdCounter(trapwriter);
      // re-initialise the location manager, since it keeps a reference to the TRAP writer
-      locationManager = new LocationManager(f, trapwriter, locationManager.getFileLabel());
+      locationManager = new LocationManager(extractedFile, trapwriter, locationManager.getFileLabel());
    }

    // now do the extraction itself
@@ -484,9 +509,9 @@ public class FileExtractor {
      IExtractor extractor = fileType.mkExtractor(config, state);
      TextualExtractor textualExtractor =
          new TextualExtractor(
-              trapwriter, locationManager, source, config.getExtractLines(), metrics);
+              trapwriter, locationManager, source, config.getExtractLines(), metrics, extractedFile);
      LoCInfo loc = extractor.extract(textualExtractor);
-      int numLines = textualExtractor.getNumLines();
+      int numLines = textualExtractor.isSnippet() ? 0 : textualExtractor.getNumLines();
      int linesOfCode = loc.getLinesOfCode(), linesOfComments = loc.getLinesOfComments();
      trapwriter.addTuple("numlines", fileLabel, numLines, linesOfCode, linesOfComments);
      trapwriter.addTuple("filetype", fileLabel, fileType.toString());
--- a/javascript/extractor/src/com/semmle/js/extractor/FileSnippet.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/FileSnippet.java
@@ -0,0 +1,44 @@
+package com.semmle.js.extractor;
+
+import java.nio.file.Path;
+
+import com.semmle.js.extractor.ExtractorConfig.SourceType;
+
+/**
+ * Denotes where a code snippet originated from within a file.
+ */
+public class FileSnippet {
+  private Path originalFile;
+  private int line;
+  private int column;
+  private int topLevelKind;
+  private SourceType sourceType;
+
+  public FileSnippet(Path originalFile, int line, int column, int topLevelKind, SourceType sourceType) {
+    this.originalFile = originalFile;
+    this.line = line;
+    this.column = column;
+    this.topLevelKind = topLevelKind;
+    this.sourceType = sourceType;
+  }
+
+  public Path getOriginalFile() {
+    return originalFile;
+  }
+
+  public int getLine() {
+    return line;
+  }
+
+  public int getColumn() {
+    return column;
+  }
+
+  public int getTopLevelKind() {
+    return topLevelKind;
+  }
+
+  public SourceType getSourceType() {
+    return sourceType;
+  }
+}
--- a/javascript/extractor/src/com/semmle/js/extractor/HTMLExtractor.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/HTMLExtractor.java
@@ -1,12 +1,17 @@
 package com.semmle.js.extractor;

+import java.io.File;
+import java.nio.file.Path;
+import java.util.regex.Pattern;
+
 import com.semmle.js.extractor.ExtractorConfig.Platform;
 import com.semmle.js.extractor.ExtractorConfig.SourceType;
 import com.semmle.js.parser.ParseError;
 import com.semmle.util.data.StringUtil;
+import com.semmle.util.io.WholeIO;
 import com.semmle.util.trap.TrapWriter;
 import com.semmle.util.trap.TrapWriter.Label;
-import java.util.regex.Pattern;
+
 import net.htmlparser.jericho.Attribute;
 import net.htmlparser.jericho.Attributes;
 import net.htmlparser.jericho.CharacterReference;
@@ -26,9 +31,11 @@ public class HTMLExtractor implements IExtractor {
          Pattern.CASE_INSENSITIVE);

  private final ExtractorConfig config;
+  private final ExtractorState state;

-  public HTMLExtractor(ExtractorConfig config) {
+  public HTMLExtractor(ExtractorConfig config, ExtractorState state) {
    this.config = config.withPlatform(Platform.WEB);
+    this.state = state;
  }

  @Override
@@ -49,7 +56,7 @@ public class HTMLExtractor implements IExtractor {
    for (Element elt : src.getAllElements()) {
      LoCInfo snippetLoC = null;
      if (elt.getName().equals(HTMLElementName.SCRIPT)) {
-        SourceType sourceType = getScriptSourceType(elt);
+        SourceType sourceType = getScriptSourceType(elt, textualExtractor.getExtractedFile());
        if (sourceType != null) {
          // Jericho sometimes misparses empty elements, which will show up as start tags
          // ending in "/"; we manually exclude these cases to avoid spurious syntax errors
@@ -57,6 +64,7 @@ public class HTMLExtractor implements IExtractor {

          Segment content = elt.getContent();
          String source = content.toString();
+          boolean isTypeScript = isTypeScriptTag(elt);

          /*
           * Script blocks in XHTML files may wrap (parts of) their code inside CDATA sections.
@@ -79,7 +87,8 @@ public class HTMLExtractor implements IExtractor {
                    textualExtractor,
                    source,
                    contentStart.getRow(),
-                    contentStart.getColumn());
+                    contentStart.getColumn(),
+                    isTypeScript);
          }
        }
      } else {
@@ -101,7 +110,8 @@ public class HTMLExtractor implements IExtractor {
                      textualExtractor,
                      source,
                      valueStart.getRow(),
-                      valueStart.getColumn());
+                      valueStart.getColumn(),
+                      false /* isTypeScript */);
            } else if (source.startsWith("javascript:")) {
              source = source.substring(11);
              snippetLoC =
@@ -112,7 +122,8 @@ public class HTMLExtractor implements IExtractor {
                      textualExtractor,
                      source,
                      valueStart.getRow(),
-                      valueStart.getColumn() + 11);
+                      valueStart.getColumn() + 11,
+                      false /* isTypeScript */);
            }
          }
      }
@@ -139,16 +150,23 @@ public class HTMLExtractor implements IExtractor {
   * Deduce the {@link SourceType} with which the given <code>script</code> element should be
   * extracted, returning <code>null</code> if it cannot be determined.
   */
-  private SourceType getScriptSourceType(Element script) {
+  private SourceType getScriptSourceType(Element script, File file) {
    String scriptType = getAttributeValueLC(script, "type");
-    String scriptLanguage = getAttributeValueLC(script, "language");
+    String scriptLanguage = getScriptLanguage(script);
+    
+    SourceType fallbackSourceType = config.getSourceType();
+    if (file.getName().endsWith(".vue")) {
+      fallbackSourceType = SourceType.MODULE;
+    }
+
+    if (isTypeScriptTag(script)) return fallbackSourceType;

    // if `type` and `language` are both either missing, contain the
    // string "javascript", or if `type` is the string "text/jsx", this is a plain script
    if ((scriptType == null || scriptType.contains("javascript") || "text/jsx".equals(scriptType))
        && (scriptLanguage == null || scriptLanguage.contains("javascript")))
      // use default source type
-      return config.getSourceType();
+      return fallbackSourceType;

    // if `type` is "text/babel", the source type depends on the `data-plugins` attribute
    if ("text/babel".equals(scriptType)) {
@@ -156,7 +174,7 @@ public class HTMLExtractor implements IExtractor {
      if (plugins != null && plugins.contains("transform-es2015-modules-umd")) {
        return SourceType.MODULE;
      }
-      return config.getSourceType();
+      return fallbackSourceType;
    }

    // if `type` is "module", extract as module
@@ -165,6 +183,23 @@ public class HTMLExtractor implements IExtractor {
    return null;
  }

+  private String getScriptLanguage(Element script) {
+    String scriptLanguage = getAttributeValueLC(script, "language");
+
+    if (scriptLanguage == null) { // Vue templates use 'lang' instead of 'language'.
+      scriptLanguage = getAttributeValueLC(script, "lang");
+    }
+    return scriptLanguage;
+  }
+
+  private boolean isTypeScriptTag(Element script) {
+    String language = getScriptLanguage(script);
+    if ("ts".equals(language) || "typescript".equals(language)) return true;
+    String type = getAttributeValueLC(script, "type");
+    if (type != null && type.contains("typescript")) return true;
+    return false;
+  }
+
  /**
   * Get the value of attribute <code>attr</code> of element <code>elt</code> in lower case; if the
   * attribute has no value, <code>null</code> is returned.
@@ -181,7 +216,27 @@ public class HTMLExtractor implements IExtractor {
      TextualExtractor textualExtractor,
      String source,
      int line,
-      int column) {
+      int column,
+      boolean isTypeScript) {
+    if (isTypeScript) {
+      Path file = textualExtractor.getExtractedFile().toPath();
+      FileSnippet snippet = new FileSnippet(file, line, column, toplevelKind, config.getSourceType());
+      VirtualSourceRoot vroot = config.getVirtualSourceRoot();
+      // Vue files are special in that they can be imported as modules, and may only contain one <script> tag.
+      // For .vue files we omit the usual snippet decoration to ensure the TypeScript compiler can find it.
+      Path virtualFile =
+          file.getFileName().toString().endsWith(".vue")
+          ? vroot.toVirtualFile(file.resolveSibling(file.getFileName() + ".ts"))
+          : vroot.getVirtualFileForSnippet(snippet, ".ts");
+      if (virtualFile != null) {
+        virtualFile = virtualFile.toAbsolutePath().normalize();
+        synchronized(vroot.getLock()) {
+          new WholeIO().strictwrite(virtualFile, source);
+        }
+        state.getSnippets().put(virtualFile, snippet);
+      }
+      return null; // LoC info is accounted for later
+    }
    TrapWriter trapwriter = textualExtractor.getTrapwriter();
    LocationManager locationManager = textualExtractor.getLocationManager();
    LocationManager scriptLocationManager =
@@ -196,7 +251,8 @@ public class HTMLExtractor implements IExtractor {
              scriptLocationManager,
              source,
              config.getExtractLines(),
-              textualExtractor.getMetrics());
+              textualExtractor.getMetrics(),
+              textualExtractor.getExtractedFile());
      return extractor.extract(tx, source, toplevelKind, scopeManager).snd();
    } catch (ParseError e) {
      e.setPosition(scriptLocationManager.translatePosition(e.getPosition()));
--- a/javascript/extractor/src/com/semmle/js/extractor/Main.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/Main.java
@@ -2,6 +2,8 @@ package com.semmle.js.extractor;

 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -31,6 +33,8 @@ import com.semmle.util.io.WholeIO;
 import com.semmle.util.language.LegacyLanguage;
 import com.semmle.util.process.ArgsParser;
 import com.semmle.util.process.ArgsParser.FileMode;
+import com.semmle.util.process.Env;
+import com.semmle.util.process.Env.Var;
 import com.semmle.util.trap.TrapWriter;

 /** The main entry point of the JavaScript extractor. */
@@ -134,12 +138,6 @@ public class Main {
      return;
    }

-    TypeScriptParser tsParser = extractorState.getTypeScriptParser();
-    tsParser.setTypescriptRam(extractorConfig.getTypeScriptRam());
-    if (containsTypeScriptFiles()) {
-      tsParser.verifyInstallation(!ap.has(P_QUIET));
-    }
-
    // Sort files for determinism
    projectFiles = projectFiles.stream()
          .sorted(AutoBuild.FILE_ORDERING)
@@ -149,16 +147,30 @@ public class Main {
        .sorted(AutoBuild.FILE_ORDERING)
        .collect(Collectors.toCollection(() -> new LinkedHashSet<>()));

+    // Extract HTML files first, as they may contain embedded TypeScript code
+    for (File file : files) {
+      if (FileType.forFile(file, extractorConfig) == FileType.HTML) {
+        ensureFileIsExtracted(file, ap);
+      }
+    }
+    
+    TypeScriptParser tsParser = extractorState.getTypeScriptParser();
+    tsParser.setTypescriptRam(extractorConfig.getTypeScriptRam());
+    if (containsTypeScriptFiles()) {
+      tsParser.verifyInstallation(!ap.has(P_QUIET));
+    }
+
    for (File projectFile : projectFiles) {

      long start = verboseLogStartTimer(ap, "Opening project " + projectFile);
-      ParsedProject project = tsParser.openProject(projectFile, DependencyInstallationResult.empty);
+      ParsedProject project = tsParser.openProject(projectFile, DependencyInstallationResult.empty, extractorConfig.getVirtualSourceRoot());
      verboseLogEndTimer(ap, start);
      // Extract all files belonging to this project which are also matched
      // by our include/exclude filters.
      List<File> filesToExtract = new ArrayList<>();
      for (File sourceFile : project.getOwnFiles()) {
-        if (files.contains(normalizeFile(sourceFile))
+        File normalizedFile = normalizeFile(sourceFile);
+        if ((files.contains(normalizedFile) || extractorState.getSnippets().containsKey(normalizedFile.toPath()))
            && !extractedFiles.contains(sourceFile.getAbsoluteFile())
            && FileType.TYPESCRIPT.getExtensions().contains(FileUtil.extension(sourceFile))) {
          filesToExtract.add(sourceFile);
@@ -287,10 +299,14 @@ public class Main {
  }

  public void collectFiles(ArgsParser ap) {
-    for (File f : ap.getOneOrMoreFiles("files", FileMode.FILE_OR_DIRECTORY_MUST_EXIST))
+    for (File f : getFilesArg(ap))
      collectFiles(f, true);
  }

+  private List<File> getFilesArg(ArgsParser ap) {
+    return ap.getOneOrMoreFiles("files", FileMode.FILE_OR_DIRECTORY_MUST_EXIST);
+  }
+
  public void setupMatchers(ArgsParser ap) {
    Set<String> includes = new LinkedHashSet<>();

@@ -444,6 +460,21 @@ public class Main {
    if (ap.has(P_TYPESCRIPT)) return TypeScriptMode.BASIC;
    return TypeScriptMode.NONE;
  }
+  
+  private Path inferSourceRoot(ArgsParser ap) {
+    List<File> files = getFilesArg(ap);
+    Path sourceRoot = files.iterator().next().toPath().toAbsolutePath().getParent();
+    for (File file : files) {
+      Path path = file.toPath().toAbsolutePath().getParent();
+      for (int i = 0; i < sourceRoot.getNameCount(); ++i) {
+        if (!(i < path.getNameCount() && path.getName(i).equals(sourceRoot.getName(i)))) {
+          sourceRoot = sourceRoot.subpath(0, i);
+          break;
+        }
+      }
+    }
+    return sourceRoot;
+  }

  private ExtractorConfig parseJSOptions(ArgsParser ap) {
    ExtractorConfig cfg =
@@ -466,6 +497,17 @@ public class Main {
                    ? UnitParser.parseOpt(ap.getString(P_TYPESCRIPT_RAM), UnitParser.MEGABYTES)
                    : 0);
    if (ap.has(P_DEFAULT_ENCODING)) cfg = cfg.withDefaultEncoding(ap.getString(P_DEFAULT_ENCODING));
+
+    // Make a usable virtual source root mapping.
+    // The concept of source root and scratch directory do not exist in the legacy extractor,
+    // so we construct these based on what we have.
+    String odasaDbDir = Env.systemEnv().getNonEmpty(Var.ODASA_DB);
+    VirtualSourceRoot virtualSourceRoot =
+        odasaDbDir == null
+            ? VirtualSourceRoot.none
+            : new VirtualSourceRoot(inferSourceRoot(ap), Paths.get(odasaDbDir, "working"));
+    cfg = cfg.withVirtualSourceRoot(virtualSourceRoot);
+
    return cfg;
  }

--- a/javascript/extractor/src/com/semmle/js/extractor/TextualExtractor.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/TextualExtractor.java
@@ -1,11 +1,13 @@
 package com.semmle.js.extractor;

+import java.io.File;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
 import com.semmle.js.ast.Position;
 import com.semmle.js.ast.SourceElement;
 import com.semmle.util.trap.TrapWriter;
 import com.semmle.util.trap.TrapWriter.Label;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;

 /**
 * Extractor for populating purely textual information about a file, namely its lines and their line
@@ -21,19 +23,40 @@ public class TextualExtractor {
  private final Label fileLabel;
  private final boolean extractLines;
  private final ExtractionMetrics metrics;
+  private final File extractedFile;

  public TextualExtractor(
      TrapWriter trapwriter,
      LocationManager locationManager,
      String source,
      boolean extractLines,
-      ExtractionMetrics metrics) {
+      ExtractionMetrics metrics,
+      File extractedFile) {
    this.trapwriter = trapwriter;
    this.locationManager = locationManager;
    this.source = source;
    this.fileLabel = locationManager.getFileLabel();
    this.extractLines = extractLines;
    this.metrics = metrics;
+    this.extractedFile = extractedFile;
+  }
+
+  /**
+   * Returns the file whose contents should be extracted, and is contained
+   * in {@link #source}.
+   *
+   * <p>This may differ from the source file of the location manager, which refers
+   * to the original file that this was derived from.
+   */
+  public File getExtractedFile() {
+    return extractedFile;
+  }
+
+  /**
+   * Returns true if the extracted file and the source location files are different.
+   */
+  public boolean isSnippet() {
+    return !extractedFile.equals(locationManager.getSourceFile());
  }

  public TrapWriter getTrapwriter() {
--- a/javascript/extractor/src/com/semmle/js/extractor/TypeScriptExtractor.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/TypeScriptExtractor.java
@@ -1,32 +1,34 @@
 package com.semmle.js.extractor;

+import java.io.File;
+
 import com.semmle.js.extractor.ExtractorConfig.ECMAVersion;
 import com.semmle.js.extractor.ExtractorConfig.SourceType;
 import com.semmle.js.parser.JSParser.Result;
 import com.semmle.js.parser.ParseError;
-import com.semmle.js.parser.TypeScriptParser;
-import java.io.File;

 public class TypeScriptExtractor implements IExtractor {
  private final JSExtractor jsExtractor;
-  private final TypeScriptParser parser;
+  private final ExtractorState state;

-  public TypeScriptExtractor(ExtractorConfig config, TypeScriptParser parser) {
+  public TypeScriptExtractor(ExtractorConfig config, ExtractorState state) {
    this.jsExtractor = new JSExtractor(config);
-    this.parser = parser;
+    this.state = state;
  }

  @Override
  public LoCInfo extract(TextualExtractor textualExtractor) {
    LocationManager locationManager = textualExtractor.getLocationManager();
    String source = textualExtractor.getSource();
-    File sourceFile = locationManager.getSourceFile();
-    Result res = parser.parse(sourceFile, source, textualExtractor.getMetrics());
+    File sourceFile = textualExtractor.getExtractedFile();
+    Result res = state.getTypeScriptParser().parse(sourceFile, source, textualExtractor.getMetrics());
    ScopeManager scopeManager =
        new ScopeManager(textualExtractor.getTrapwriter(), ECMAVersion.ECMA2017);
    try {
-      SourceType sourceType = jsExtractor.establishSourceType(source, false);
-      return jsExtractor.extract(textualExtractor, source, 0, scopeManager, sourceType, res).snd();
+      FileSnippet snippet = state.getSnippets().get(sourceFile.toPath());
+      SourceType sourceType = snippet != null ? snippet.getSourceType() : jsExtractor.establishSourceType(source, false);
+      int toplevelKind = snippet != null ? snippet.getTopLevelKind() : 0;
+      return jsExtractor.extract(textualExtractor, source, toplevelKind, scopeManager, sourceType, res).snd();
    } catch (ParseError e) {
      e.setPosition(locationManager.translatePosition(e.getPosition()));
      throw e.asUserError();
--- a/javascript/extractor/src/com/semmle/js/extractor/VirtualSourceRoot.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/VirtualSourceRoot.java
@@ -0,0 +1,80 @@
+package com.semmle.js.extractor;
+
+import java.nio.file.Path;
+
+public class VirtualSourceRoot {
+  private Path sourceRoot;
+  private Path virtualSourceRoot;
+  private Object lock = new Object();
+
+  public static final VirtualSourceRoot none = new VirtualSourceRoot(null, null);
+
+  public VirtualSourceRoot(Path sourceRoot, Path virtualSourceRoot) {
+    this.sourceRoot = sourceRoot;
+    this.virtualSourceRoot = virtualSourceRoot;
+  }
+
+  /**
+   * Returns the source root mirrored by {@link #getVirtualSourceRoot()} or <code>null</code> if no
+   * virtual source root exists.
+   *
+   * <p>When invoked from the AutoBuilder, this corresponds to the source root. When invoked from
+   * ODASA, there is no notion of source root, so this is always <code>null</code> in that context.
+   */
+  public Path getSourceRoot() {
+    return sourceRoot;
+  }
+
+  /**
+   * Returns the virtual source root or <code>null</code> if no virtual source root exists.
+   *
+   * <p>The virtual source root is a directory hierarchy that mirrors the real source root, where
+   * dependencies are installed.
+   */
+  public Path getVirtualSourceRoot() {
+    return virtualSourceRoot;
+  }
+
+  private static Path translate(Path oldRoot, Path newRoot, Path file) {
+    if (oldRoot == null || newRoot == null) return null;
+    Path relative = oldRoot.relativize(file);
+    if (relative.startsWith("..") || relative.isAbsolute()) return null;
+    return newRoot.resolve(relative);
+  }
+
+  public Path toVirtualFile(Path file) {
+    if (file.startsWith(virtualSourceRoot)) {
+      // 'qltest' creates a virtual source root inside the real source root.
+      // Make sure such files don't appear to be inside the real source root.
+      return null;
+    }
+    return translate(sourceRoot, virtualSourceRoot, file);
+  }
+
+  public Path fromVirtualFile(Path file) {
+    return translate(virtualSourceRoot, sourceRoot, file);
+  }
+
+  public Path getVirtualFileForSnippet(FileSnippet snippet, String extension) {
+    String basename =
+        snippet.getOriginalFile().getFileName()
+            + ".snippet."
+            + snippet.getLine()
+            + "."
+            + snippet.getColumn()
+            + extension;
+    return toVirtualFile(snippet.getOriginalFile().resolveSibling(basename));
+  }
+
+  @Override
+  public String toString() {
+    return "[sourceRoot=" + sourceRoot + ", virtualSourceRoot=" + virtualSourceRoot + "]";
+  }
+
+  /**
+   * Gets the lock to use when writing to the virtual source root in a multi-threaded context.
+   */
+  public Object getLock() {
+    return lock;
+  }
+}
--- a/javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java
@@ -15,6 +15,7 @@ import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;

 import org.junit.After;
 import org.junit.Assert;
@@ -27,6 +28,7 @@ import com.semmle.js.extractor.DependencyInstallationResult;
 import com.semmle.js.extractor.ExtractorState;
 import com.semmle.js.extractor.FileExtractor;
 import com.semmle.js.extractor.FileExtractor.FileType;
+import com.semmle.js.extractor.VirtualSourceRoot;
 import com.semmle.util.data.StringUtil;
 import com.semmle.util.exception.UserError;
 import com.semmle.util.files.FileUtil;
@@ -109,11 +111,12 @@ public class AutoBuildTests {
      Set<String> actual = new LinkedHashSet<>();
      new AutoBuild() {
        @Override
-        protected void extract(FileExtractor extractor, Path file, ExtractorState state) {
+        protected CompletableFuture<?> extract(FileExtractor extractor, Path file, boolean concurrent) {
          String extracted = file.toString();
          if (extractor.getConfig().hasFileType())
            extracted += ":" + extractor.getFileType(file.toFile());
          actual.add(extracted);
+          return CompletableFuture.completedFuture(null);
        }

        @Override
@@ -123,8 +126,7 @@ public class AutoBuildTests {
        public void extractTypeScriptFiles(
            java.util.List<Path> files,
            java.util.Set<Path> extractedFiles,
-            FileExtractor extractor,
-            ExtractorState extractorState) {
+            FileExtractors extractors) {
          for (Path f : files) {
            actual.add(f.toString());
          }
@@ -136,6 +138,11 @@ public class AutoBuildTests {
          return DependencyInstallationResult.empty;
        }

+        @Override
+        protected VirtualSourceRoot makeVirtualSourceRoot() {
+          return VirtualSourceRoot.none; // not used in these tests
+        }
+
        @Override
        protected void extractXml() throws IOException {
          Files.walkFileTree(
--- a/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java
+++ b/javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java
@@ -31,6 +31,7 @@ import com.google.gson.JsonPrimitive;
 import com.semmle.js.extractor.DependencyInstallationResult;
 import com.semmle.js.extractor.EnvironmentVariables;
 import com.semmle.js.extractor.ExtractionMetrics;
+import com.semmle.js.extractor.VirtualSourceRoot;
 import com.semmle.js.parser.JSParser.Result;
 import com.semmle.ts.extractor.TypeTable;
 import com.semmle.util.data.StringUtil;
@@ -501,8 +502,8 @@ public class TypeScriptParser {
  /**
   * Returns the set of files included by the inclusion pattern in the given tsconfig.json file.
   */
-  public Set<File> getOwnFiles(File tsConfigFile, DependencyInstallationResult deps) {
-    JsonObject request = makeLoadCommand("get-own-files", tsConfigFile, deps);
+  public Set<File> getOwnFiles(File tsConfigFile, DependencyInstallationResult deps, VirtualSourceRoot vroot) {
+    JsonObject request = makeLoadCommand("get-own-files", tsConfigFile, deps, vroot);
    JsonObject response = talkToParserWrapper(request);
    try {
      checkResponseType(response, "file-list");
@@ -521,8 +522,8 @@ public class TypeScriptParser {
   *
   * <p>Only one project should be opened at once.
   */
-  public ParsedProject openProject(File tsConfigFile, DependencyInstallationResult deps) {
-    JsonObject request = makeLoadCommand("open-project", tsConfigFile, deps);
+  public ParsedProject openProject(File tsConfigFile, DependencyInstallationResult deps, VirtualSourceRoot vroot) {
+    JsonObject request = makeLoadCommand("open-project", tsConfigFile, deps, vroot);
    JsonObject response = talkToParserWrapper(request);
    try {
      checkResponseType(response, "project-opened");
@@ -536,18 +537,18 @@ public class TypeScriptParser {
    }
  }

-  private JsonObject makeLoadCommand(String command, File tsConfigFile, DependencyInstallationResult deps) {
+  private JsonObject makeLoadCommand(String command, File tsConfigFile, DependencyInstallationResult deps, VirtualSourceRoot vroot) {
    JsonObject request = new JsonObject();
    request.add("command", new JsonPrimitive(command));
    request.add("tsConfig", new JsonPrimitive(tsConfigFile.getPath()));
    request.add("packageEntryPoints", mapToArray(deps.getPackageEntryPoints()));
    request.add("packageJsonFiles", mapToArray(deps.getPackageJsonFiles()));
-    request.add("sourceRoot", deps.getSourceRoot() == null
+    request.add("sourceRoot", vroot.getSourceRoot() == null
        ? JsonNull.INSTANCE
-        : new JsonPrimitive(deps.getSourceRoot().toString()));
-    request.add("virtualSourceRoot", deps.getVirtualSourceRoot() == null
+        : new JsonPrimitive(vroot.getSourceRoot().toString()));
+    request.add("virtualSourceRoot", vroot.getVirtualSourceRoot() == null
        ? JsonNull.INSTANCE
-        : new JsonPrimitive(deps.getVirtualSourceRoot().toString()));
+        : new JsonPrimitive(vroot.getVirtualSourceRoot().toString()));
    return request;
  }

--- a/javascript/ql/src/semmle/javascript/Files.qll
+++ b/javascript/ql/src/semmle/javascript/Files.qll
@@ -206,13 +206,13 @@ class File extends Container, @file {
  override string getAbsolutePath() { files(this, result, _, _, _) }

  /** Gets the number of lines in this file. */
-  int getNumberOfLines() { numlines(this, result, _, _) }
+  int getNumberOfLines() { result = sum(int loc | numlines(this, loc, _, _) | loc) }

  /** Gets the number of lines containing code in this file. */
-  int getNumberOfLinesOfCode() { numlines(this, _, result, _) }
+  int getNumberOfLinesOfCode() { result = sum(int loc | numlines(this, _, loc, _) | loc) }

  /** Gets the number of lines containing comments in this file. */
-  int getNumberOfLinesOfComments() { numlines(this, _, _, result) }
+  int getNumberOfLinesOfComments() { result = sum(int loc | numlines(this, _, _, loc) | loc) }

  /** Gets a toplevel piece of JavaScript code in this file. */
  TopLevel getATopLevel() { result.getFile() = this }
--- a/javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/Test.expected
+++ b/javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/Test.expected
@@ -0,0 +1,26 @@
+classDeclaration
+| test.vue:3:18:5:3 | class M ... er;\\n  } |
+exprType
+| htmlfile.html:4:22:4:24 | foo | () => void |
+| htmlfile.html:4:22:4:24 | foo | () => void |
+| htmlfile.html:4:33:4:41 | "./other" | any |
+| htmlfile.html:5:17:5:22 | result | number[] |
+| htmlfile.html:5:26:5:28 | foo | () => void |
+| htmlfile.html:5:26:5:30 | foo() | void |
+| htmlfile.html:5:26:5:42 | foo() as number[] | number[] |
+| other.ts:1:8:1:16 | Component | typeof default in library-tests/TypeScript/EmbeddedInScript/test.vue |
+| other.ts:1:23:1:34 | "./test.vue" | any |
+| other.ts:3:1:3:15 | new Component() | MyComponent |
+| other.ts:3:5:3:13 | Component | typeof default in library-tests/TypeScript/EmbeddedInScript/test.vue |
+| other.ts:5:17:5:19 | foo | () => void |
+| test.vue:2:15:2:19 | other | typeof library-tests/TypeScript/EmbeddedInScript/other.ts |
+| test.vue:2:26:2:34 | "./other" | any |
+| test.vue:3:24:3:34 | MyComponent | MyComponent |
+| test.vue:4:7:4:7 | x | number |
+symbols
+| other.ts:1:1:6:0 | <toplevel> | library-tests/TypeScript/EmbeddedInScript/other.ts |
+| test.vue:2:3:6:0 | <toplevel> | library-tests/TypeScript/EmbeddedInScript/test.vue |
+importTarget
+| htmlfile.html:4:13:4:42 | import  ... other"; | other.ts:1:1:6:0 | <toplevel> |
+| other.ts:1:1:1:35 | import  ... t.vue"; | test.vue:2:3:6:0 | <toplevel> |
+| test.vue:2:3:2:35 | import  ... other"; | other.ts:1:1:6:0 | <toplevel> |
--- a/javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/Test.ql
+++ b/javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/Test.ql
@@ -0,0 +1,9 @@
+import javascript
+
+query ClassDefinition classDeclaration() { any() }
+
+query Type exprType(Expr e) { result = e.getType() }
+
+query predicate symbols(Module mod, CanonicalName name) { ast_node_symbol(mod, name) }
+
+query predicate importTarget(Import imprt, Module mod) { imprt.getImportedModule() = mod }
--- a/javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/htmlfile.html
+++ b/javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/htmlfile.html
@@ -0,0 +1,8 @@
+<html>
+    <body>
+        <script type="module" language="typescript">
+            import { foo } from "./other";
+            let result = foo() as number[];
+        </script>
+    </body>
+</html>
--- a/javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/other.ts
+++ b/javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/other.ts
@@ -0,0 +1,5 @@
+import Component from "./test.vue";
+
+new Component();
+
+export function foo() {};
--- a/javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/test.vue
+++ b/javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/test.vue
@@ -0,0 +1,6 @@
+<script lang='ts'>
+  import * as other from "./other";
+  export default class MyComponent {
+      x!: number;
+  }
+</script>
--- a/javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/tsconfig.json
+++ b/javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/tsconfig.json
@@ -0,0 +1,3 @@
+{
+    "include": ["."]
+}