From c7a1cf7236cee0705eb5176382d78145ed05e616 Mon Sep 17 00:00:00 2001 From: Nora Date: Wed, 10 Jan 2024 09:59:54 +0000 Subject: [PATCH] Add test, config and action --- .github/workflows/e2e-tests.yml | 43 ++++++++++ .gitignore | 3 + extensions/ql-vscode/package-lock.json | 60 ++++++++++++++ extensions/ql-vscode/package.json | 1 + extensions/ql-vscode/playwright.config.ts | 79 +++++++++++++++++++ extensions/ql-vscode/test/e2e/README.md | 9 +++ .../test/e2e/docker/User/settings.json | 3 +- .../ql-vscode/test/e2e/run-query.spec.ts | 73 +++++++++++++++++ 8 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/e2e-tests.yml create mode 100644 extensions/ql-vscode/playwright.config.ts create mode 100644 extensions/ql-vscode/test/e2e/README.md create mode 100644 extensions/ql-vscode/test/e2e/run-query.spec.ts diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml new file mode 100644 index 000000000..95aa756f5 --- /dev/null +++ b/.github/workflows/e2e-tests.yml @@ -0,0 +1,43 @@ +name: Run E2E Playwright tests +on: + push: + +jobs: + e2e-test: + name: E2E Test + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version-file: extensions/ql-vscode/.nvmrc + cache: 'npm' + cache-dependency-path: extensions/ql-vscode/package-lock.json + + - name: Install dependencies + working-directory: extensions/ql-vscode + run: npm ci + + - name: Start containers + working-directory: extensions/ql-vscode/test/e2e + run: docker-compose -f "docker-compose.yml" up -d --build + + - name: Install Playwright Browsers + working-directory: extensions/ql-vscode + run: npx playwright install --with-deps + - name: Run Playwright tests + working-directory: extensions/ql-vscode + run: npx playwright test + - uses: actions/upload-artifact@v4 + if: always() + with: + name: playwright-report + path: extension/ql-vscode/playwright-report/ + retention-days: 30 + - name: Stop containers + working-directory: extensions/ql-vscode/test/e2e + if: always() + run: docker-compose -f "docker-compose.yml" down -v \ No newline at end of file diff --git a/.gitignore b/.gitignore index 94107c69e..28013dc67 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ artifacts/ # CodeQL metadata .cache/ .codeql/ + +# E2E Reports +**/playwright-report/** diff --git a/extensions/ql-vscode/package-lock.json b/extensions/ql-vscode/package-lock.json index fcc53e582..87d192576 100644 --- a/extensions/ql-vscode/package-lock.json +++ b/extensions/ql-vscode/package-lock.json @@ -54,6 +54,7 @@ "@faker-js/faker": "^8.0.2", "@github/markdownlint-github": "^0.6.0", "@octokit/plugin-throttling": "^8.0.0", + "@playwright/test": "^1.40.1", "@storybook/addon-a11y": "^7.4.6", "@storybook/addon-actions": "^7.1.0", "@storybook/addon-essentials": "^7.1.0", @@ -4428,6 +4429,21 @@ "node": ">= 8" } }, + "node_modules/@playwright/test": { + "version": "1.40.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.40.1.tgz", + "integrity": "sha512-EaaawMTOeEItCRvfmkI9v6rBkF1svM8wjl/YPRrg2N2Wmp+4qJYkWtJsbew1szfKKDm6fPLy4YAanBhIlf9dWw==", + "dev": true, + "dependencies": { + "playwright": "1.40.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/@pmmmwh/react-refresh-webpack-plugin": { "version": "0.5.11", "resolved": "https://registry.npmjs.org/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.5.11.tgz", @@ -26775,6 +26791,50 @@ "node": ">=10" } }, + "node_modules/playwright": { + "version": "1.40.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.40.1.tgz", + "integrity": "sha512-2eHI7IioIpQ0bS1Ovg/HszsN/XKNwEG1kbzSDDmADpclKc7CyqkHw7Mg2JCz/bbCxg25QUPcjksoMW7JcIFQmw==", + "dev": true, + "dependencies": { + "playwright-core": "1.40.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.40.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.40.1.tgz", + "integrity": "sha512-+hkOycxPiV534c4HhpfX6yrlawqVUzITRKwHAmYfmsVreltEl6fAZJ3DPfLMOODw0H3s1Itd6MDCWmP1fl/QvQ==", + "dev": true, + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/plugin-error": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/plugin-error/-/plugin-error-2.0.1.tgz", diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index 1fa606758..7790d031b 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -1952,6 +1952,7 @@ "@faker-js/faker": "^8.0.2", "@github/markdownlint-github": "^0.6.0", "@octokit/plugin-throttling": "^8.0.0", + "@playwright/test": "^1.40.1", "@storybook/addon-a11y": "^7.4.6", "@storybook/addon-actions": "^7.1.0", "@storybook/addon-essentials": "^7.1.0", diff --git a/extensions/ql-vscode/playwright.config.ts b/extensions/ql-vscode/playwright.config.ts new file mode 100644 index 000000000..bdd47ce3d --- /dev/null +++ b/extensions/ql-vscode/playwright.config.ts @@ -0,0 +1,79 @@ +import { defineConfig, devices } from "@playwright/test"; + +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +// require('dotenv').config(); + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: "./test/e2e", + + timeout: 5 * 60 * 1000, + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: "html", + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like `await page.goto('/')`. */ + // baseURL: 'http://127.0.0.1:3000', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: "on-first-retry", + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + }, + + // { + // name: "firefox", + // use: { ...devices["Desktop Firefox"] }, + // }, + + // { + // name: "webkit", + // use: { ...devices["Desktop Safari"] }, + // }, + + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { ...devices['Pixel 5'] }, + // }, + // { + // name: 'Mobile Safari', + // use: { ...devices['iPhone 12'] }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { ...devices['Desktop Edge'], channel: 'msedge' }, + // }, + // { + // name: 'Google Chrome', + // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, + // }, + ], + + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'npm run start', + // url: 'http://127.0.0.1:3000', + // reuseExistingServer: !process.env.CI, + // }, +}); diff --git a/extensions/ql-vscode/test/e2e/README.md b/extensions/ql-vscode/test/e2e/README.md new file mode 100644 index 000000000..2effa1159 --- /dev/null +++ b/extensions/ql-vscode/test/e2e/README.md @@ -0,0 +1,9 @@ +## VS Code CodeQL E2E Tests + +### Set up + +- run `docker-compose build` from the e2e test folder `vscode-codeql/extensions/ql-vscode/test/e2e` +- run `docker-compose up` from the e2e test folder `vscode-codeql/extensions/ql-vscode/test/e2e` +- run `npx playwright test --ui` from the vscode-codeql folder `vscode-codeql/extensions/ql-vscode` + +When you've run the test once! remove the volume data, e.g. by running `docker-compose down -v` \ No newline at end of file diff --git a/extensions/ql-vscode/test/e2e/docker/User/settings.json b/extensions/ql-vscode/test/e2e/docker/User/settings.json index bd4090483..8dfd44b28 100644 --- a/extensions/ql-vscode/test/e2e/docker/User/settings.json +++ b/extensions/ql-vscode/test/e2e/docker/User/settings.json @@ -1,5 +1,6 @@ { "workbench.startupEditor": "none", "security.workspace.trust.enabled": false, - "codeQL.cli.executablePath": "/opt/codeql/codeql" + "codeQL.cli.executablePath": "/opt/codeql/codeql", + "codeQL.telemetry.enableTelemetry": false } diff --git a/extensions/ql-vscode/test/e2e/run-query.spec.ts b/extensions/ql-vscode/test/e2e/run-query.spec.ts new file mode 100644 index 000000000..a4c000632 --- /dev/null +++ b/extensions/ql-vscode/test/e2e/run-query.spec.ts @@ -0,0 +1,73 @@ +import { test, expect } from "@playwright/test"; + +test("run query and open it from history", async ({ page }) => { + await page.goto("http://localhost:8080/?folder=/home/coder/project"); + + await page.getByRole("tab", { name: "CodeQL" }).locator("a").click(); + + // decline extension telemetry + await page.getByRole("button", { name: "No", exact: true }).click({ + timeout: 60000, + }); + + await page.keyboard.press("Control+Shift+P"); + await page.keyboard.type("Create Query"); + await page.keyboard.press("Enter"); + + await page.getByLabel("JavaScript, javascript").locator("a").click({ + timeout: 60000, + }); + + // select folder for first query + await page + .getByText( + "Results0 SelectedPress 'Enter' to confirm your input or 'Escape' to cancelOK", + ) + .press("Enter"); + + // Download database + await page + .getByRole("button", { name: "Download database" }) + .click({ timeout: 60000 }); + await page.getByPlaceholder("https://github.com//<").press("Enter"); + await page + .locator("#list_id_3_0") + .getByText("javascript") + .click({ timeout: 60000 }); + + await page.keyboard.press("Control+Shift+P"); + await page.keyboard.type("Run Query on selected"); + await page.keyboard.press("Enter"); + + // select folder for first query run + // await page.getByRole("button", { name: "OK" }).click(); + + // check that query was run successfully and results page opens + await page.getByText("Hello world on d3/d3").click({ timeout: 800000 }); + + // close results page and open query from history + await page + .getByLabel("CodeQL Query Results, Editor Group") + .getByLabel("Close (Ctrl+F4)") + .click(); + await expect( + page + .frameLocator(".webview") + .frameLocator('iframe[title="CodeQL Query Results"]') + .getByText("#selectalerts32 resultsShow"), + ).not.toBeVisible(); + + await page + .locator("#list_id_6_0") + .getByLabel("Hello world on d3/d3 -") + .locator("div") + .first() + .click(); + + await expect( + page + .frameLocator('iframe[name="\\34 884429a-d667-4121-932e-99abaa20b599"]') + .frameLocator('iframe[title="CodeQL Query Results"]') + .getByText("#selectalerts32 resultsShow"), + ).not.toBeVisible(); +});