Merge branch 'main' into aeisenberg/save-query-history

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,9 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## [UNRELEASED]
+
+- Fix a bug where database upgrades could not be resolved if some of the target pack's dependencies are outside of the workspace. [#1138](https://github.com/github/vscode-codeql/pull/1138)
+
 ## 1.5.11 - 10 February 2022
 
 - Fix a bug where invoking _View AST_ from the file explorer would not view the selected file. Instead it would view the active editor. Also, prevent the _View AST_ from appearing if the current selection includes a directory or multiple files. [#1113](https://github.com/github/vscode-codeql/pull/1113)

extensions/ql-vscode/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vscode-codeql",
-  "version": "1.5.11",
+  "version": "1.5.12",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "vscode-codeql",
-      "version": "1.5.11",
+      "version": "1.5.12",
       "license": "MIT",
       "dependencies": {
         "@octokit/rest": "^18.5.6",
@@ -9452,9 +9452,9 @@
       }
     },
     "node_modules/pathval": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz",
-      "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=",
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz",
+      "integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==",
       "dev": true,
       "engines": {
         "node": "*"
@@ -20617,9 +20617,9 @@
       }
     },
     "pathval": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz",
-      "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=",
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz",
+      "integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==",
       "dev": true
     },
     "pend": {

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.5.11",
+  "version": "1.5.12",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",

extensions/ql-vscode/src/config.ts

@@ -355,15 +355,3 @@ export function getRemoteControllerRepo(): string | undefined {
 export async function setRemoteControllerRepo(repo: string | undefined) {
   await REMOTE_CONTROLLER_REPO.updateValue(repo, ConfigurationTarget.Global);
 }
-
-/**
- * Whether to insecurely load ML models from CodeQL packs.
- *
- * This setting is for internal users only.
- */
-const SHOULD_INSECURELY_LOAD_MODELS_FROM_PACKS =
-  new Setting('shouldInsecurelyLoadModelsFromPacks', RUNNING_QUERIES_SETTING);
-
-export function shouldInsecurelyLoadMlModelsFromPacks(): boolean {
-  return SHOULD_INSECURELY_LOAD_MODELS_FROM_PACKS.getValue<boolean>();
-}

extensions/ql-vscode/src/remote-queries/run-remote-query.ts

@@ -23,12 +23,6 @@ import { RemoteQuery } from './remote-query';
 import { RemoteQuerySubmissionResult } from './remote-query-submission-result';
 import { QueryMetadata } from '../pure/interface-types';
 
-interface Config {
-  repositories: string[];
-  ref?: string;
-  language?: string;
-}
-
 export interface QlPack {
   name: string;
   version: string;
@@ -108,7 +102,7 @@ export async function getRepositories(): Promise<string[] | undefined> {
  *
  * @returns the entire qlpack as a base64 string.
  */
-async function generateQueryPack(cliServer: cli.CodeQLCliServer, queryFile: string, queryPackDir: string, fallbackLanguage?: string): Promise<{
+async function generateQueryPack(cliServer: cli.CodeQLCliServer, queryFile: string, queryPackDir: string): Promise<{
   base64Pack: string,
   language: string
 }> {
@@ -150,7 +144,7 @@ async function generateQueryPack(cliServer: cli.CodeQLCliServer, queryFile: stri
 
   } else {
     // open popup to ask for language if not already hardcoded
-    language = fallbackLanguage || await askForLanguage(cliServer);
+    language = await askForLanguage(cliServer);
 
     // copy only the query file to the query pack directory
     // and generate a synthetic query pack
@@ -238,47 +232,22 @@ export async function runRemoteQuery(
       throw new UserCancellationException('Not a CodeQL query file.');
     }
 
-    progress({
-      maxStep: 5,
-      step: 1,
-      message: 'Determining project list'
-    });
-
     const queryFile = uri.fsPath;
-    const repositoriesFile = queryFile.substring(0, queryFile.length - '.ql'.length) + '.repositories';
-    let ref: string | undefined;
-    // For the case of single file remote queries, use the language from the config in order to avoid the user having to select it.
-    let fallbackLanguage: string | undefined;
-    let repositories: string[] | undefined;
 
     progress({
-      maxStep: 5,
-      step: 2,
+      maxStep: 4,
+      step: 1,
       message: 'Determining query target language'
     });
 
-    // If the user has an explicit `.repositories` file, use that.
-    // Otherwise, prompt user to select repositories from the `codeQL.remoteQueries.repositoryLists` setting.
-    if (await fs.pathExists(repositoriesFile)) {
-      void logger.log(`Found '${repositoriesFile}'. Using information from that file to run ${queryFile}.`);
-
-      const config = yaml.safeLoad(await fs.readFile(repositoriesFile, 'utf8')) as Config;
-
-      ref = config.ref || 'main';
-      fallbackLanguage = config.language;
-      repositories = config.repositories;
-    } else {
-      ref = 'main';
-      repositories = await getRepositories();
-    }
-
+    const repositories = await getRepositories();
     if (!repositories || repositories.length === 0) {
       throw new UserCancellationException('No repositories to query.');
     }
 
     progress({
-      maxStep: 5,
-      step: 3,
+      maxStep: 4,
+      step: 2,
       message: 'Determining controller repo'
     });
 
@@ -309,8 +278,8 @@ export async function runRemoteQuery(
     const [owner, repo] = controllerRepo.split('/');
 
     progress({
-      maxStep: 5,
-      step: 4,
+      maxStep: 4,
+      step: 3,
       message: 'Bundling the query pack'
     });
 
@@ -318,19 +287,19 @@ export async function runRemoteQuery(
       throw new UserCancellationException('Cancelled');
     }
 
-    const { base64Pack, language } = await generateQueryPack(cliServer, queryFile, queryPackDir, fallbackLanguage);
+    const { base64Pack, language } = await generateQueryPack(cliServer, queryFile, queryPackDir);
 
     if (token.isCancellationRequested) {
       throw new UserCancellationException('Cancelled');
     }
 
     progress({
-      maxStep: 5,
-      step: 5,
+      maxStep: 4,
+      step: 4,
       message: 'Sending request'
     });
 
-    const workflowRunId = await runRemoteQueriesApiRequest(credentials, ref, language, repositories, owner, repo, base64Pack, dryRun);
+    const workflowRunId = await runRemoteQueriesApiRequest(credentials, 'main', language, repositories, owner, repo, base64Pack, dryRun);
     const queryStartTime = new Date();
     const queryMetadata = await tryGetQueryMetadata(cliServer, queryFile);
 

extensions/ql-vscode/src/run-queries.ts

@@ -10,8 +10,7 @@ import {
   TextDocument,
   TextEditor,
   Uri,
-  window,
-  workspace
+  window
 } from 'vscode';
 import { ErrorCodes, ResponseError } from 'vscode-languageclient';
 
@@ -447,12 +446,23 @@ async function compileNonDestructiveUpgrade(
   progress: ProgressCallback,
   token: CancellationToken,
 ): Promise<string> {
-  const searchPath = getOnDiskWorkspaceFolders();
 
   if (!dbItem?.contents?.dbSchemeUri) {
     throw new Error('Database is invalid, and cannot be upgraded.');
   }
-  const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades(dbItem.contents.dbSchemeUri.fsPath, searchPath, true, query.queryDbscheme);
+
+  // When packaging is used, dependencies may exist outside of the workspace and they are always on the resolved search path.
+  // When packaging is not used, all dependencies are in the workspace.
+  const upgradesPath = (await qs.cliServer.cliConstraints.supportsPackaging())
+    ? qlProgram.libraryPath
+    : getOnDiskWorkspaceFolders();
+
+  const { scripts, matchesTarget } = await qs.cliServer.resolveUpgrades(
+    dbItem.contents.dbSchemeUri.fsPath,
+    upgradesPath,
+    true,
+    query.queryDbscheme
+  );
 
   if (!matchesTarget) {
     reportNoUpgradePath(qlProgram, query);
@@ -643,15 +653,7 @@ export async function compileAndRunQueryAgainstDatabase(
   const metadata = await tryGetQueryMetadata(cliServer, qlProgram.queryPath);
 
   let availableMlModels: cli.MlModelInfo[] = [];
-  // The `capabilities.untrustedWorkspaces.restrictedConfigurations` entry in package.json doesn't
-  // work with hidden settings, so we manually check that the workspace is trusted before looking at
-  // whether the `shouldInsecurelyLoadMlModelsFromPacks` setting is enabled.
-  if (
-    workspace.isTrusted &&
-    config.isCanary() &&
-    config.shouldInsecurelyLoadMlModelsFromPacks() &&
-    await cliServer.cliConstraints.supportsResolveMlModels()
-  ) {
+  if (await cliServer.cliConstraints.supportsResolveMlModels()) {
     try {
       availableMlModels = (await cliServer.resolveMlModels(diskWorkspaceFolders)).models;
       void logger.log(`Found available ML models at the following paths: ${availableMlModels.map(x => `'${x.path}'`).join(', ')}.`);