Add test, config and action

.github/workflows/e2e-tests.yml

@@ -0,0 +1,43 @@
+name: Run E2E Playwright tests
+on:
+  push:
+
+jobs:
+  e2e-test:
+    name: E2E Test
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: extensions/ql-vscode/.nvmrc
+          cache: 'npm'
+          cache-dependency-path: extensions/ql-vscode/package-lock.json
+
+      - name: Install dependencies
+        working-directory: extensions/ql-vscode
+        run: npm ci
+
+      - name: Start containers
+        working-directory: extensions/ql-vscode/test/e2e
+        run: docker-compose -f "docker-compose.yml" up -d --build
+
+      - name: Install Playwright Browsers
+        working-directory: extensions/ql-vscode
+        run: npx playwright install --with-deps
+      - name: Run Playwright tests
+        working-directory: extensions/ql-vscode
+        run: npx playwright test
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: playwright-report
+          path: extension/ql-vscode/playwright-report/
+          retention-days: 30
+      - name: Stop containers
+        working-directory: extensions/ql-vscode/test/e2e
+        if: always()
+        run: docker-compose -f "docker-compose.yml" down -v

.gitignore

@@ -19,3 +19,6 @@ artifacts/
 # CodeQL metadata
 .cache/
 .codeql/
+
+# E2E Reports
+**/playwright-report/**

extensions/ql-vscode/package-lock.json

@@ -54,6 +54,7 @@
         "@faker-js/faker": "^8.0.2",
         "@github/markdownlint-github": "^0.6.0",
         "@octokit/plugin-throttling": "^8.0.0",
+        "@playwright/test": "^1.40.1",
         "@storybook/addon-a11y": "^7.4.6",
         "@storybook/addon-actions": "^7.1.0",
         "@storybook/addon-essentials": "^7.1.0",
@@ -4428,6 +4429,21 @@
         "node": ">= 8"
       }
     },
+    "node_modules/@playwright/test": {
+      "version": "1.40.1",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.40.1.tgz",
+      "integrity": "sha512-EaaawMTOeEItCRvfmkI9v6rBkF1svM8wjl/YPRrg2N2Wmp+4qJYkWtJsbew1szfKKDm6fPLy4YAanBhIlf9dWw==",
+      "dev": true,
+      "dependencies": {
+        "playwright": "1.40.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=16"
+      }
+    },
     "node_modules/@pmmmwh/react-refresh-webpack-plugin": {
       "version": "0.5.11",
       "resolved": "https://registry.npmjs.org/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.5.11.tgz",
@@ -26775,6 +26791,50 @@
         "node": ">=10"
       }
     },
+    "node_modules/playwright": {
+      "version": "1.40.1",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.40.1.tgz",
+      "integrity": "sha512-2eHI7IioIpQ0bS1Ovg/HszsN/XKNwEG1kbzSDDmADpclKc7CyqkHw7Mg2JCz/bbCxg25QUPcjksoMW7JcIFQmw==",
+      "dev": true,
+      "dependencies": {
+        "playwright-core": "1.40.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=16"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.40.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.40.1.tgz",
+      "integrity": "sha512-+hkOycxPiV534c4HhpfX6yrlawqVUzITRKwHAmYfmsVreltEl6fAZJ3DPfLMOODw0H3s1Itd6MDCWmP1fl/QvQ==",
+      "dev": true,
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=16"
+      }
+    },
+    "node_modules/playwright/node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
     "node_modules/plugin-error": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/plugin-error/-/plugin-error-2.0.1.tgz",

extensions/ql-vscode/package.json

@@ -1952,6 +1952,7 @@
     "@faker-js/faker": "^8.0.2",
     "@github/markdownlint-github": "^0.6.0",
     "@octokit/plugin-throttling": "^8.0.0",
+    "@playwright/test": "^1.40.1",
     "@storybook/addon-a11y": "^7.4.6",
     "@storybook/addon-actions": "^7.1.0",
     "@storybook/addon-essentials": "^7.1.0",

extensions/ql-vscode/playwright.config.ts

@@ -0,0 +1,79 @@
+import { defineConfig, devices } from "@playwright/test";
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// require('dotenv').config();
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: "./test/e2e",
+
+  timeout: 5 * 60 * 1000,
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: "html",
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://127.0.0.1:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: "on-first-retry",
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: "chromium",
+      use: { ...devices["Desktop Chrome"] },
+    },
+
+    // {
+    //   name: "firefox",
+    //   use: { ...devices["Desktop Firefox"] },
+    // },
+
+    // {
+    //   name: "webkit",
+    //   use: { ...devices["Desktop Safari"] },
+    // },
+
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: { ...devices['Pixel 5'] },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: { ...devices['iPhone 12'] },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: 'Microsoft Edge',
+    //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
+    // },
+    // {
+    //   name: 'Google Chrome',
+    //   use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+    // },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  // webServer: {
+  //   command: 'npm run start',
+  //   url: 'http://127.0.0.1:3000',
+  //   reuseExistingServer: !process.env.CI,
+  // },
+});

extensions/ql-vscode/test/e2e/README.md

@@ -0,0 +1,9 @@
+## VS Code CodeQL E2E Tests
+
+### Set up
+
+- run `docker-compose build` from the e2e test folder `vscode-codeql/extensions/ql-vscode/test/e2e`
+- run `docker-compose up` from the e2e test folder `vscode-codeql/extensions/ql-vscode/test/e2e`
+- run  `npx playwright test --ui` from the vscode-codeql folder `vscode-codeql/extensions/ql-vscode`
+
+When you've run the test once! remove the volume data, e.g. by running `docker-compose down -v`

extensions/ql-vscode/test/e2e/docker/User/settings.json

@@ -1,5 +1,6 @@
 {
   "workbench.startupEditor": "none",
   "security.workspace.trust.enabled": false,
-  "codeQL.cli.executablePath": "/opt/codeql/codeql"
+  "codeQL.cli.executablePath": "/opt/codeql/codeql",
+  "codeQL.telemetry.enableTelemetry": false
 }

extensions/ql-vscode/test/e2e/run-query.spec.ts

@@ -0,0 +1,73 @@
+import { test, expect } from "@playwright/test";
+
+test("run query and open it from history", async ({ page }) => {
+  await page.goto("http://localhost:8080/?folder=/home/coder/project");
+
+  await page.getByRole("tab", { name: "CodeQL" }).locator("a").click();
+
+  // decline extension telemetry
+  await page.getByRole("button", { name: "No", exact: true }).click({
+    timeout: 60000,
+  });
+
+  await page.keyboard.press("Control+Shift+P");
+  await page.keyboard.type("Create Query");
+  await page.keyboard.press("Enter");
+
+  await page.getByLabel("JavaScript, javascript").locator("a").click({
+    timeout: 60000,
+  });
+
+  // select folder for first query
+  await page
+    .getByText(
+      "Results0 SelectedPress 'Enter' to confirm your input or 'Escape' to cancelOK",
+    )
+    .press("Enter");
+
+  // Download database
+  await page
+    .getByRole("button", { name: "Download database" })
+    .click({ timeout: 60000 });
+  await page.getByPlaceholder("https://github.com/<owner>/<").press("Enter");
+  await page
+    .locator("#list_id_3_0")
+    .getByText("javascript")
+    .click({ timeout: 60000 });
+
+  await page.keyboard.press("Control+Shift+P");
+  await page.keyboard.type("Run Query on selected");
+  await page.keyboard.press("Enter");
+
+  // select folder for first query run
+  // await page.getByRole("button", { name: "OK" }).click();
+
+  // check that query was run successfully and results page opens
+  await page.getByText("Hello world on d3/d3").click({ timeout: 800000 });
+
+  // close results page and open query from history
+  await page
+    .getByLabel("CodeQL Query Results, Editor Group")
+    .getByLabel("Close (Ctrl+F4)")
+    .click();
+  await expect(
+    page
+      .frameLocator(".webview")
+      .frameLocator('iframe[title="CodeQL Query Results"]')
+      .getByText("#selectalerts32 resultsShow"),
+  ).not.toBeVisible();
+
+  await page
+    .locator("#list_id_6_0")
+    .getByLabel("Hello world on d3/d3 -")
+    .locator("div")
+    .first()
+    .click();
+
+  await expect(
+    page
+      .frameLocator('iframe[name="\\34 884429a-d667-4121-932e-99abaa20b599"]')
+      .frameLocator('iframe[title="CodeQL Query Results"]')
+      .getByText("#selectalerts32 resultsShow"),
+  ).not.toBeVisible();
+});