Merge pull request #2847 from github/dependabot/npm_and_yarn/extensions/ql-vscode/types/js-yaml-4.0.6

Bump @types/js-yaml from 3.12.5 to 4.0.6 in /extensions/ql-vscode
This commit is contained in:
Koen Vlaswinkel
2023-09-26 12:41:11 +02:00
committed by GitHub
21 changed files with 442 additions and 90 deletions

View File

@@ -99,6 +99,42 @@ jobs:
run: |
npm run find-deadcode
generated:
name: Check generated code
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- uses: actions/setup-node@v3
with:
node-version: '16.17.1'
cache: 'npm'
cache-dependency-path: extensions/ql-vscode/package-lock.json
- name: Install dependencies
working-directory: extensions/ql-vscode
run: |
npm ci
shell: bash
- name: Check that repo is clean
run: |
git diff --exit-code
git diff --exit-code --cached
- name: Generate code
working-directory: extensions/ql-vscode
run: |
npm run generate
- name: Check for changes
run: |
git diff --exit-code
git diff --exit-code --cached
unit-test:
name: Unit Test
runs-on: ${{ matrix.os }}

View File

@@ -79,7 +79,7 @@
"@types/gulp": "^4.0.9",
"@types/gulp-replace": "^1.1.0",
"@types/jest": "^29.0.2",
"@types/js-yaml": "^3.12.5",
"@types/js-yaml": "^4.0.6",
"@types/nanoid": "^3.0.0",
"@types/node": "^16.11.25",
"@types/node-fetch": "^2.5.2",
@@ -9150,8 +9150,9 @@
"integrity": "sha512-qC4bCqYGy1y/NP7dDVr7KJarn+PbX1nSpwA7JXdu0HxT3QYjO8MJ+cntENtHFVy2dRAyBV23OZ6MxsW1AM1L8g=="
},
"node_modules/@types/js-yaml": {
"version": "3.12.5",
"integrity": "sha512-JCcp6J0GV66Y4ZMDAQCXot4xprYB+Zfd3meK9+INSJeVZwJmHAW30BBEEkPzXswMXuiyReUGOP3GxrADc9wPww==",
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.6.tgz",
"integrity": "sha512-ACTuifTSIIbyksx2HTon3aFtCKWcID7/h3XEmRpDYdMCXxPbl+m9GteOJeaAkiAta/NJaSFuA7ahZ0NkwajDSw==",
"dev": true
},
"node_modules/@types/jsdom": {
@@ -38478,8 +38479,9 @@
"integrity": "sha512-qC4bCqYGy1y/NP7dDVr7KJarn+PbX1nSpwA7JXdu0HxT3QYjO8MJ+cntENtHFVy2dRAyBV23OZ6MxsW1AM1L8g=="
},
"@types/js-yaml": {
"version": "3.12.5",
"integrity": "sha512-JCcp6J0GV66Y4ZMDAQCXot4xprYB+Zfd3meK9+INSJeVZwJmHAW30BBEEkPzXswMXuiyReUGOP3GxrADc9wPww==",
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.6.tgz",
"integrity": "sha512-ACTuifTSIIbyksx2HTon3aFtCKWcID7/h3XEmRpDYdMCXxPbl+m9GteOJeaAkiAta/NJaSFuA7ahZ0NkwajDSw==",
"dev": true
},
"@types/jsdom": {

View File

@@ -1946,6 +1946,8 @@
"storybook": "storybook dev -p 6006",
"build-storybook": "storybook build",
"lint:scenarios": "ts-node scripts/lint-scenarios.ts",
"generate": "npm-run-all -p generate:*",
"generate:schemas": "ts-node scripts/generate-schemas.ts",
"check-types": "find . -type f -name \"tsconfig.json\" -not -path \"./node_modules/*\" | sed -r 's|/[^/]+$||' | sort | uniq | xargs -I {} sh -c \"echo Checking types in {} && cd {} && npx tsc --noEmit\"",
"postinstall": "patch-package",
"prepare": "cd ../.. && husky install"
@@ -2020,7 +2022,7 @@
"@types/gulp": "^4.0.9",
"@types/gulp-replace": "^1.1.0",
"@types/jest": "^29.0.2",
"@types/js-yaml": "^3.12.5",
"@types/js-yaml": "^4.0.6",
"@types/nanoid": "^3.0.0",
"@types/node": "^16.11.25",
"@types/node-fetch": "^2.5.2",

View File

@@ -0,0 +1,72 @@
import { createGenerator } from "ts-json-schema-generator";
import { join, resolve } from "path";
import { outputFile } from "fs-extra";
import { format, resolveConfig } from "prettier";
const extensionDirectory = resolve(__dirname, "..");
const schemas = [
{
path: join(
extensionDirectory,
"src",
"model-editor",
"extension-pack-metadata.ts",
),
type: "ExtensionPackMetadata",
schemaPath: join(
extensionDirectory,
"src",
"model-editor",
"extension-pack-metadata.schema.json",
),
},
{
path: join(
extensionDirectory,
"src",
"model-editor",
"model-extension-file.ts",
),
type: "ModelExtensionFile",
schemaPath: join(
extensionDirectory,
"src",
"model-editor",
"model-extension-file.schema.json",
),
},
];
async function generateSchema(
schemaDefinition: (typeof schemas)[number],
): Promise<void> {
const schema = createGenerator({
path: schemaDefinition.path,
tsconfig: resolve(extensionDirectory, "tsconfig.json"),
type: schemaDefinition.type,
skipTypeCheck: true,
topRef: true,
additionalProperties: true,
}).createSchema(schemaDefinition.type);
const schemaJson = JSON.stringify(schema, null, 2);
const prettierOptions = await resolveConfig(schemaDefinition.schemaPath);
const formattedSchemaJson = await format(schemaJson, {
...prettierOptions,
filepath: schemaDefinition.schemaPath,
});
await outputFile(schemaDefinition.schemaPath, formattedSchemaJson);
}
async function generateSchemas() {
await Promise.all(schemas.map(generateSchema));
}
generateSchemas().catch((e: unknown) => {
console.error(e);
process.exit(2);
});

View File

@@ -13,6 +13,7 @@ import { redactableError } from "../common/errors";
import { showAndLogExceptionWithTelemetry } from "../common/logging";
import { extLogger } from "../common/logging/vscode";
import { telemetryListener } from "../common/vscode/telemetry";
import { SuiteInstruction } from "../packaging/suite-instruction";
export async function qlpackOfDatabase(
cli: Pick<CodeQLCliServer, "resolveQlpacks">,
@@ -50,12 +51,12 @@ async function resolveQueriesFromPacks(
postfix: ".qls",
})
).path;
const suiteYaml = [];
const suiteYaml: SuiteInstruction[] = [];
for (const qlpack of qlpacks) {
suiteYaml.push({
from: qlpack,
queries: ".",
include: constraints,
include: constraints as Record<string, string[]>,
});
}
await writeFile(

View File

@@ -1,45 +0,0 @@
{
"type": "object",
"properties": {
"extensions": {
"type": "array",
"items": {
"type": "object",
"required": ["addsTo", "data"],
"properties": {
"addsTo": {
"type": "object",
"required": ["pack", "extensible"],
"properties": {
"pack": {
"type": "string"
},
"extensible": {
"type": "string"
}
}
},
"data": {
"type": "array",
"items": {
"type": "array",
"items": {
"oneOf": [
{
"type": "string"
},
{
"type": "boolean"
},
{
"type": "number"
}
]
}
}
}
}
}
}
}
}

View File

@@ -0,0 +1,97 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$ref": "#/definitions/ExtensionPackMetadata",
"definitions": {
"ExtensionPackMetadata": {
"type": "object",
"properties": {
"extensionTargets": {
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"dataExtensions": {
"anyOf": [
{
"type": "array",
"items": {
"type": "string"
}
},
{
"type": "string"
}
]
},
"name": {
"type": "string"
},
"version": {
"type": "string"
},
"dependencies": {
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"dbscheme": {
"type": "string"
},
"library": {
"type": "boolean"
},
"defaultSuite": {
"type": "array",
"items": {
"$ref": "#/definitions/SuiteInstruction"
}
},
"defaultSuiteFile": {
"type": "string"
}
},
"required": ["dataExtensions", "extensionTargets", "name", "version"]
},
"SuiteInstruction": {
"type": "object",
"properties": {
"qlpack": {
"type": "string"
},
"query": {
"type": "string"
},
"queries": {
"type": "string"
},
"include": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
},
"exclude": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
},
"description": {
"type": "string"
},
"from": {
"type": "string"
}
},
"description": "A single entry in a .qls file."
}
}
}

View File

@@ -0,0 +1,7 @@
import { QlPackFile } from "../packaging/qlpack-file";
export type ExtensionPackMetadata = QlPackFile & {
// Make both extensionTargets and dataExtensions required
extensionTargets: Record<string, string>;
dataExtensions: string[] | string;
};

View File

@@ -2,6 +2,7 @@ import { join } from "path";
import { outputFile, pathExists, readFile } from "fs-extra";
import { dump as dumpYaml, load as loadYaml } from "js-yaml";
import { Uri } from "vscode";
import Ajv from "ajv";
import { CodeQLCliServer } from "../codeql-cli/cli";
import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
import { ProgressCallback } from "../common/vscode/progress";
@@ -18,6 +19,12 @@ import {
} from "./extension-pack-name";
import { autoPickExtensionsDirectory } from "./extensions-workspace-folder";
import { ExtensionPackMetadata } from "./extension-pack-metadata";
import * as extensionPackMetadataSchemaJson from "./extension-pack-metadata.schema.json";
const ajv = new Ajv({ allErrors: true });
const extensionPackValidate = ajv.compile(extensionPackMetadataSchemaJson);
export async function pickExtensionPack(
cliServer: Pick<CodeQLCliServer, "resolveQlpacks">,
databaseItem: Pick<DatabaseItem, "name" | "language">,
@@ -170,6 +177,22 @@ async function writeExtensionPack(
return extensionPack;
}
function validateExtensionPack(
extensionPack: unknown,
): extensionPack is ExtensionPackMetadata {
extensionPackValidate(extensionPack);
if (extensionPackValidate.errors) {
throw new Error(
`Invalid extension pack YAML: ${extensionPackValidate.errors
.map((error) => `${error.instancePath} ${error.message}`)
.join(", ")}`,
);
}
return true;
}
async function readExtensionPack(
path: string,
language: string,
@@ -188,6 +211,10 @@ async function readExtensionPack(
throw new Error(`Could not parse ${qlpackPath}`);
}
if (!validateExtensionPack(qlpack)) {
throw new Error(`Could not validate ${qlpackPath}`);
}
const dataExtensionValue = qlpack.dataExtensions;
if (
!(

View File

@@ -0,0 +1,45 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$ref": "#/definitions/ModelExtensionFile",
"definitions": {
"ModelExtensionFile": {
"type": "object",
"properties": {
"extensions": {
"type": "array",
"items": {
"type": "object",
"properties": {
"addsTo": {
"type": "object",
"properties": {
"pack": {
"type": "string"
},
"extensible": {
"type": "string"
}
},
"required": ["pack", "extensible"]
},
"data": {
"type": "array",
"items": {
"type": "array",
"items": {
"$ref": "#/definitions/DataTuple"
}
}
}
},
"required": ["addsTo", "data"]
}
}
},
"required": ["extensions"]
},
"DataTuple": {
"type": ["boolean", "number", "string"]
}
}
}

View File

@@ -0,0 +1,17 @@
type ExtensibleReference = {
pack: string;
extensible: string;
};
export type DataTuple = boolean | number | string;
type DataRow = DataTuple[];
type ModelExtension = {
addsTo: ExtensibleReference;
data: DataRow[];
};
export type ModelExtensionFile = {
extensions: ModelExtension[];
};

View File

@@ -1,16 +1,15 @@
import { ModeledMethod, ModeledMethodType, Provenance } from "./modeled-method";
import { DataTuple } from "./model-extension-file";
export type ExtensiblePredicateDefinition = {
extensiblePredicate: string;
generateMethodDefinition: (method: ModeledMethod) => Tuple[];
readModeledMethod: (row: Tuple[]) => ModeledMethod;
generateMethodDefinition: (method: ModeledMethod) => DataTuple[];
readModeledMethod: (row: DataTuple[]) => ModeledMethod;
supportedKinds?: string[];
};
type Tuple = boolean | number | string;
function readRowToMethod(row: Tuple[]): string {
function readRowToMethod(row: DataTuple[]): string {
return `${row[0]}.${row[1]}#${row[3]}${row[4]}`;
}

View File

@@ -7,12 +7,13 @@ import {
extensiblePredicateDefinitions,
} from "./predicates";
import * as dataSchemaJson from "./data-schema.json";
import * as modelExtensionFileSchema from "./model-extension-file.schema.json";
import { Mode } from "./shared/mode";
import { assertNever } from "../common/helpers-pure";
import { ModelExtensionFile } from "./model-extension-file";
const ajv = new Ajv({ allErrors: true });
const dataSchemaValidate = ajv.compile(dataSchemaJson);
const ajv = new Ajv({ allErrors: true, allowUnionTypes: true });
const modelExtensionFileSchemaValidate = ajv.compile(modelExtensionFileSchema);
function createDataProperty(
methods: ModeledMethod[],
@@ -211,24 +212,29 @@ export function createFilenameForPackage(
return `${prefix}${packageName}${suffix}.yml`;
}
export function loadDataExtensionYaml(
data: any,
): Record<string, ModeledMethod> | undefined {
dataSchemaValidate(data);
function validateModelExtensionFile(data: unknown): data is ModelExtensionFile {
modelExtensionFileSchemaValidate(data);
if (dataSchemaValidate.errors) {
if (modelExtensionFileSchemaValidate.errors) {
throw new Error(
`Invalid data extension YAML: ${dataSchemaValidate.errors
`Invalid data extension YAML: ${modelExtensionFileSchemaValidate.errors
.map((error) => `${error.instancePath} ${error.message}`)
.join(", ")}`,
);
}
const extensions = data.extensions;
if (!Array.isArray(extensions)) {
return true;
}
export function loadDataExtensionYaml(
data: unknown,
): Record<string, ModeledMethod> | undefined {
if (!validateModelExtensionFile(data)) {
return undefined;
}
const extensions = data.extensions;
const modeledMethods: Record<string, ModeledMethod> = {};
for (const extension of extensions) {

View File

@@ -0,0 +1,16 @@
import { SuiteInstruction } from "./suite-instruction";
/**
* The qlpack pack file, either in qlpack.yml or in codeql-pack.yml.
*/
export interface QlPackFile {
name: string;
version: string;
dependencies?: Record<string, string>;
extensionTargets?: Record<string, string>;
dbscheme?: string;
library?: boolean;
defaultSuite?: SuiteInstruction[];
defaultSuiteFile?: string;
dataExtensions?: string[] | string;
}

View File

@@ -0,0 +1,8 @@
/**
* The qlpack lock file, either in qlpack.lock.yml or in codeql-pack.lock.yml.
*/
export interface QlPackLockFile {
lockVersion: string;
dependencies?: Record<string, string>;
compiled?: boolean;
}

View File

@@ -0,0 +1,12 @@
/**
* A single entry in a .qls file.
*/
export interface SuiteInstruction {
qlpack?: string;
query?: string;
queries?: string;
include?: Record<string, string[]>;
exclude?: Record<string, string[]>;
description?: string;
from?: string;
}

View File

@@ -37,15 +37,7 @@ import {
import { QueryLanguage } from "../common/query-language";
import { tryGetQueryMetadata } from "../codeql-cli/query-metadata";
import { askForLanguage, findLanguage } from "../codeql-cli/query-language";
interface QlPack {
name: string;
version: string;
library?: boolean;
dependencies: { [key: string]: string };
defaultSuite?: Array<Record<string, unknown>>;
defaultSuiteFile?: string;
}
import { QlPackFile } from "../packaging/qlpack-file";
/**
* Well-known names for the query pack used by the server.
@@ -395,7 +387,7 @@ async function fixPackFile(
)} file in '${queryPackDir}'`,
);
}
const qlpack = load(await readFile(packPath, "utf8")) as QlPack;
const qlpack = load(await readFile(packPath, "utf8")) as QlPackFile;
updateDefaultSuite(qlpack, packRelativePath);
removeWorkspaceRefs(qlpack);
@@ -416,7 +408,11 @@ async function injectExtensionPacks(
)} file in '${queryPackDir}'`,
);
}
const syntheticQueryPack = load(await readFile(qlpackFile, "utf8")) as QlPack;
const syntheticQueryPack = load(
await readFile(qlpackFile, "utf8"),
) as QlPackFile;
const dependencies = syntheticQueryPack.dependencies ?? {};
const extensionPacks = await cliServer.resolveQlpacks(workspaceFolders, true);
Object.entries(extensionPacks).forEach(([name, paths]) => {
@@ -433,13 +429,16 @@ async function injectExtensionPacks(
// Add this extension pack as a dependency. It doesn't matter which
// version we specify, since we are guaranteed that the extension pack
// is resolved from source at the given path.
syntheticQueryPack.dependencies[name] = "*";
dependencies[name] = "*";
});
syntheticQueryPack.dependencies = dependencies;
await writeFile(qlpackFile, dump(syntheticQueryPack));
await cliServer.clearCache();
}
function updateDefaultSuite(qlpack: QlPack, packRelativePath: string) {
function updateDefaultSuite(qlpack: QlPackFile, packRelativePath: string) {
delete qlpack.defaultSuiteFile;
qlpack.defaultSuite = generateDefaultSuite(packRelativePath);
}
@@ -541,8 +540,12 @@ async function getControllerRepoFromApi(
}
}
export function removeWorkspaceRefs(qlpack: QlPack) {
for (const [key, value] of Object.entries(qlpack.dependencies || {})) {
export function removeWorkspaceRefs(qlpack: QlPackFile) {
if (!qlpack.dependencies) {
return;
}
for (const [key, value] of Object.entries(qlpack.dependencies)) {
if (value === "${workspace}") {
qlpack.dependencies[key] = "*";
}

View File

@@ -31,6 +31,8 @@ import { mockedQuickPickItem } from "../../utils/mocking.helpers";
import { QueryLanguage } from "../../../../src/common/query-language";
import { readBundledPack } from "../../utils/bundled-pack-helpers";
import { load } from "js-yaml";
import { ExtensionPackMetadata } from "../../../../src/model-editor/extension-pack-metadata";
import { QlPackLockFile } from "../../../../src/packaging/qlpack-lock-file";
describe("Variant Analysis Manager", () => {
let cli: CodeQLCliServer;
@@ -379,7 +381,7 @@ describe("Variant Analysis Manager", () => {
: "codeql-pack.yml";
const qlpackContents = load(
packFS.fileContents(packFileName).toString("utf-8"),
);
) as ExtensionPackMetadata;
expect(qlpackContents.name).toEqual(expectedPackName);
if (checkVersion) {
expect(qlpackContents.version).toEqual("0.0.0");
@@ -393,9 +395,9 @@ describe("Variant Analysis Manager", () => {
}
const qlpackLockContents = load(
packFS.fileContents("codeql-pack.lock.yml").toString("utf-8"),
);
) as QlPackLockFile;
const actualLockKeys = Object.keys(qlpackLockContents.dependencies);
const actualLockKeys = Object.keys(qlpackLockContents.dependencies ?? {});
// The lock file should contain at least the specified dependencies.
dependenciesToCheck.forEach((dep) =>

View File

@@ -11,6 +11,7 @@ import { removeWorkspaceRefs } from "../../src/variant-analysis/run-remote-query
import { CodeQLExtensionInterface } from "../../src/extension";
import { importArchiveDatabase } from "../../src/databases/database-fetcher";
import { createMockCommandManager } from "../__mocks__/commandsMock";
import { QlPackFile } from "../../src/packaging/qlpack-file";
// This file contains helpers shared between tests that work with an activated extension.
@@ -89,7 +90,9 @@ export async function fixWorkspaceReferences(
): Promise<Record<string, string> | undefined> {
if (!(await cli.cliConstraints.supportsWorkspaceReferences())) {
// remove the workspace references from the qlpack
const qlpack = load(readFileSync(qlpackFileWithWorkspaceRefs, "utf8"));
const qlpack = load(
readFileSync(qlpackFileWithWorkspaceRefs, "utf8"),
) as QlPackFile;
const originalDeps = { ...qlpack.dependencies };
removeWorkspaceRefs(qlpack);
writeFileSync(qlpackFileWithWorkspaceRefs, dump(qlpack));
@@ -113,7 +116,9 @@ export async function restoreWorkspaceReferences(
if (!originalDeps) {
return;
}
const qlpack = load(readFileSync(qlpackFileWithWorkspaceRefs, "utf8"));
const qlpack = load(
readFileSync(qlpackFileWithWorkspaceRefs, "utf8"),
) as QlPackFile;
qlpack.dependencies = originalDeps;
writeFileSync(qlpackFileWithWorkspaceRefs, dump(qlpack));
}

View File

@@ -20,6 +20,7 @@ import { pathExists, readFile } from "fs-extra";
import { load as loadYaml } from "js-yaml";
import { CancellationTokenSource } from "vscode-jsonrpc";
import { QueryOutputDir } from "../../../../src/run-queries-shared";
import { ModelExtensionFile } from "../../../../src/model-editor/model-extension-file";
describe("runAutoModelQueries", () => {
let resolveQueriesSpy: jest.SpiedFunction<
@@ -186,7 +187,9 @@ describe("generateCandidateFilterPack", () => {
const filterFile = join(packDir, "filter.yml");
expect(await pathExists(filterFile)).toBe(true);
// Read the contents of filterFile and parse as yaml
const yaml = await loadYaml(await readFile(filterFile, "utf8"));
const yaml = (await loadYaml(
await readFile(filterFile, "utf8"),
)) as ModelExtensionFile;
const extensions = yaml.extensions;
expect(extensions).toBeInstanceOf(Array);
expect(extensions).toHaveLength(1);

View File

@@ -363,6 +363,43 @@ describe("pickExtensionPack", () => {
expect(cliServer.resolveQlpacks).toHaveBeenCalled();
});
it("shows an error when the pack YAML does not contain name", async () => {
const tmpDir = await dir({
unsafeCleanup: true,
});
const cliServer = mockCliServer({
"github/vscode-codeql-java": [tmpDir.path],
});
await outputFile(
join(tmpDir.path, "codeql-pack.yml"),
dumpYaml({
version: "0.0.0",
library: true,
extensionTargets: {
"codeql/java-all": "*",
},
dataExtensions: ["models/**/*.yml"],
}),
);
expect(
await pickExtensionPack(
cliServer,
databaseItem,
logger,
progress,
maxStep,
),
).toEqual(undefined);
expect(logger.showErrorMessage).toHaveBeenCalledTimes(1);
expect(logger.showErrorMessage).toHaveBeenCalledWith(
"Could not read extension pack github/vscode-codeql-java",
);
expect(cliServer.resolveQlpacks).toHaveBeenCalled();
});
it("shows an error when the pack YAML does not contain dataExtensions", async () => {
const tmpDir = await dir({
unsafeCleanup: true,