Merge pull request #3215 from github/nora/add-e2e-tests

E2E test prototype
2024-01-24 12:59:50 +01:00
parent 0265befb26 a6f7ee3df8
commit 3daefcdffc
13 changed files with 332 additions and 2 deletions
--- a/.github/workflows/e2e-tests.yml
+++ b/.github/workflows/e2e-tests.yml
@@ -0,0 +1,46 @@
+name: Run E2E Playwright tests
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  e2e-test:
+    name: E2E Test
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: extensions/ql-vscode/.nvmrc
+          cache: 'npm'
+          cache-dependency-path: extensions/ql-vscode/package-lock.json
+
+      - name: Install dependencies
+        working-directory: extensions/ql-vscode
+        run: npm ci
+
+      - name: Start containers
+        working-directory: extensions/ql-vscode/test/e2e
+        run: docker-compose -f "docker-compose.yml" up -d --build
+
+      - name: Install Playwright Browsers
+        working-directory: extensions/ql-vscode
+        run: npx playwright install --with-deps
+      - name: Run Playwright tests
+        working-directory: extensions/ql-vscode/test/e2e
+        run: npx playwright test
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: playwright-report
+          path: extensions/ql-vscode/playwright-report/
+          retention-days: 30
+      - name: Stop containers
+        working-directory: extensions/ql-vscode/test/e2e
+        if: always()
+        run: docker-compose -f "docker-compose.yml" down -v
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,7 @@ artifacts/
 # CodeQL metadata
 .cache/
 .codeql/
+
+# E2E Reports
+**/playwright-report/**
+**/test-results/**
--- a/extensions/ql-vscode/package-lock.json
+++ b/extensions/ql-vscode/package-lock.json
@@ -53,6 +53,7 @@
        "@faker-js/faker": "^8.0.2",
        "@github/markdownlint-github": "^0.6.0",
        "@octokit/plugin-throttling": "^8.0.0",
+        "@playwright/test": "^1.40.1",
        "@storybook/addon-a11y": "^7.6.9",
        "@storybook/addon-actions": "^7.1.0",
        "@storybook/addon-essentials": "^7.1.0",
@@ -4442,6 +4443,21 @@
        "node": ">= 8"
      }
    },
+    "node_modules/@playwright/test": {
+      "version": "1.40.1",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.40.1.tgz",
+      "integrity": "sha512-EaaawMTOeEItCRvfmkI9v6rBkF1svM8wjl/YPRrg2N2Wmp+4qJYkWtJsbew1szfKKDm6fPLy4YAanBhIlf9dWw==",
+      "dev": true,
+      "dependencies": {
+        "playwright": "1.40.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=16"
+      }
+    },
    "node_modules/@pmmmwh/react-refresh-webpack-plugin": {
      "version": "0.5.11",
      "resolved": "https://registry.npmjs.org/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.5.11.tgz",
@@ -26822,6 +26838,50 @@
        "node": ">=10"
      }
    },
+    "node_modules/playwright": {
+      "version": "1.40.1",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.40.1.tgz",
+      "integrity": "sha512-2eHI7IioIpQ0bS1Ovg/HszsN/XKNwEG1kbzSDDmADpclKc7CyqkHw7Mg2JCz/bbCxg25QUPcjksoMW7JcIFQmw==",
+      "dev": true,
+      "dependencies": {
+        "playwright-core": "1.40.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=16"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.40.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.40.1.tgz",
+      "integrity": "sha512-+hkOycxPiV534c4HhpfX6yrlawqVUzITRKwHAmYfmsVreltEl6fAZJ3DPfLMOODw0H3s1Itd6MDCWmP1fl/QvQ==",
+      "dev": true,
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=16"
+      }
+    },
+    "node_modules/playwright/node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
    "node_modules/plugin-error": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/plugin-error/-/plugin-error-2.0.1.tgz",
--- a/extensions/ql-vscode/package.json
+++ b/extensions/ql-vscode/package.json
@@ -1971,6 +1971,7 @@
    "@faker-js/faker": "^8.0.2",
    "@github/markdownlint-github": "^0.6.0",
    "@octokit/plugin-throttling": "^8.0.0",
+    "@playwright/test": "^1.40.1",
    "@storybook/addon-a11y": "^7.6.9",
    "@storybook/addon-actions": "^7.1.0",
    "@storybook/addon-essentials": "^7.1.0",
--- a/extensions/ql-vscode/scripts/find-deadcode.ts
+++ b/extensions/ql-vscode/scripts/find-deadcode.ts
@@ -14,7 +14,8 @@ function ignoreFile(file: string): boolean {
    ) ||
    basename(file) === "jest.config.ts" ||
    basename(file) === "index.tsx" ||
-    basename(file) === "index.ts"
+    basename(file) === "index.ts" ||
+    basename(file) === "playwright.config.ts"
  );
 }

--- a/extensions/ql-vscode/test/e2e/README.md
+++ b/extensions/ql-vscode/test/e2e/README.md
@@ -0,0 +1,20 @@
+## VS Code CodeQL E2E Tests
+
+When running the tests locally on a mac a different processor has to be emulated, which makes everythign VERY slow. Therefore we need to add higher timeouts in the test, so that they pass locally.
+
+### How to use locally
+
+Setup
+
+- install playwright if you haven't yet (`npx playwright install`)
+- go to the e2e test folder on your terminal
+- make sure docker is running
+- run `docker-compose build`
+- run `docker-compose up`
+
+Run tests
+
+- run  `npx playwright test --ui` from the e2e test folder to follow the test while it's running. This UI has a 'locator' tool with which elements on the test screen can be found
+- use `npx playwright test --debug` to follow the test in real time and interact with the interface, e.g. press enter or input into fields, stop and start
+
+During the test elements are created in the docker volume, e.g. the downloaded database or query data. This might interfer with other tests or when running a test twice. If that happens restart your docker volume by using `docker-compose down -v` and `docker-compose up`. Sometimes already existing queries from former runs change the input the extension needs.
--- a/extensions/ql-vscode/test/e2e/docker-compose.yml
+++ b/extensions/ql-vscode/test/e2e/docker-compose.yml
@@ -0,0 +1,55 @@
+version: "3.8"
+
+services:
+  code-server:
+    build:
+      context: docker
+      dockerfile: Dockerfile
+    platform: linux/amd64
+    container_name: code-server
+    user: "1000"
+    volumes:
+      - local-data:/home/coder/.local/share/code-server
+      - local-user-data:/home/coder/.local/share/code-server/User
+      - ./docker/config/config.yaml:/home/coder/.config/code-server/config.yaml
+      - ./docker/User/settings.json:/home/coder/.local/share/code-server/User/settings.json
+      - project-data:/home/coder/project
+    ports:
+      - 8080:8080
+    restart: unless-stopped
+    depends_on:
+      code-server-init:
+        condition: service_completed_successfully
+  code-server-init:
+    build:
+      context: docker
+      dockerfile: Dockerfile
+    platform: linux/amd64
+    user: "1000"
+    volumes:
+      - local-data:/home/coder/.local/share/code-server
+      - local-user-data:/home/coder/.local/share/code-server/User
+      - ./docker/config/config.yaml:/home/coder/.config/code-server/config.yaml
+      - ./docker/User/settings.json:/home/coder/.local/share/code-server/User/settings.json
+      - project-data:/home/coder/project
+    entrypoint: |
+      /usr/bin/entrypoint.sh --install-extension GitHub.vscode-codeql
+    restart: "no"
+    depends_on:
+      - files-init
+  files-init:
+    image: alpine:3.19.0
+    restart: "no"
+    # Since we're not running the code-server container using the same user as our host user,
+    # we need to set the permissions on the mounted volumes to match the user inside the container.
+    entrypoint: |
+      /bin/sh -c "chown 1000:1000 /home/coder/.local/share/code-server /home/coder/.local/share/code-server/User /home/coder/project"
+    volumes:
+      - local-data:/home/coder/.local/share/code-server
+      - local-user-data:/home/coder/.local/share/code-server/User
+      - project-data:/home/coder/project
+
+volumes:
+  local-data:
+  local-user-data:
+  project-data:
--- a/extensions/ql-vscode/test/e2e/docker/Dockerfile
+++ b/extensions/ql-vscode/test/e2e/docker/Dockerfile
@@ -0,0 +1,16 @@
+FROM codercom/code-server:4.20.0
+
+USER root
+
+RUN apt-get update \
+  && apt-get install -y \
+    unzip \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN wget -q -O /tmp/codeql.zip https://github.com/github/codeql-cli-binaries/releases/download/v2.15.5/codeql-linux64.zip \
+  && unzip -q /tmp/codeql.zip -d /opt \
+  && rm -rf /tmp/codeql.zip
+
+ENV PATH="/opt/codeql:${PATH}"
+
+USER 1000
--- a/extensions/ql-vscode/test/e2e/docker/User/settings.json
+++ b/extensions/ql-vscode/test/e2e/docker/User/settings.json
@@ -0,0 +1,6 @@
+{
+  "workbench.startupEditor": "none",
+  "security.workspace.trust.enabled": false,
+  "codeQL.cli.executablePath": "/opt/codeql/codeql",
+  "codeQL.telemetry.enableTelemetry": false
+}
--- a/extensions/ql-vscode/test/e2e/docker/config/config.yaml
+++ b/extensions/ql-vscode/test/e2e/docker/config/config.yaml
@@ -0,0 +1,6 @@
+bind-addr: 127.0.0.1:8080
+auth: none
+cert: false
+disable-workspace-trust: true
+disable-telemetry: true
+disable-update-check: true
--- a/extensions/ql-vscode/test/e2e/playwright.config.ts
+++ b/extensions/ql-vscode/test/e2e/playwright.config.ts
@@ -0,0 +1,36 @@
+import { defineConfig, devices } from "@playwright/test";
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: ".",
+
+  timeout: 5 * 60 * 1000,
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: "html",
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    baseURL: "http://localhost:8080",
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: "on-first-retry",
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: "chromium",
+      use: { ...devices["Desktop Chrome"] },
+    },
+  ],
+});
--- a/extensions/ql-vscode/test/e2e/run-query.spec.ts
+++ b/extensions/ql-vscode/test/e2e/run-query.spec.ts
@@ -0,0 +1,79 @@
+import { test, expect } from "@playwright/test";
+
+test("run query and open it from history", async ({ page }) => {
+  await page.goto("/?folder=/home/coder/project");
+
+  await page.getByRole("tab", { name: "CodeQL" }).locator("a").click();
+
+  // decline extension telemetry
+  await page.getByRole("button", { name: "No", exact: true }).click({
+    timeout: 60000,
+  });
+
+  await page.keyboard.press("Control+Shift+P");
+  await page.keyboard.type("Create Query");
+  await page.keyboard.press("Enter");
+
+  await page.getByLabel("JavaScript, javascript").locator("a").click({
+    timeout: 60000,
+  });
+
+  // select folder for first query
+  await page
+    .getByText(
+      "Results0 SelectedPress 'Enter' to confirm your input or 'Escape' to cancelOK",
+    )
+    .press("Enter");
+
+  // download database
+  await page
+    .getByRole("button", { name: "Download database" })
+    .click({ timeout: 60000 });
+  await page.getByPlaceholder("https://github.com/<owner>/<").press("Enter");
+  await page
+    .locator("#list_id_3_0")
+    .getByText("javascript")
+    .click({ timeout: 60000 });
+
+  await page.keyboard.press("Control+Shift+P");
+  await page.keyboard.type("Run Query on selected");
+  await page.keyboard.press("Enter");
+
+  // check if results page is visible
+  await expect(page.getByText("CodeQL Query Results")).toBeVisible({
+    timeout: 600000,
+  });
+
+  // wait for query history item to be finished
+  await expect(
+    page
+      .locator("#list_id_6_0")
+      .getByLabel("Hello world on d3/d3 -")
+      .locator("div")
+      .first(),
+  ).toBeVisible({ timeout: 60000 });
+
+  // close results page and open query from history
+  await page
+    .getByLabel("CodeQL Query Results, Editor Group")
+    .getByLabel("Close (Ctrl+F4)")
+    .click();
+
+  await expect(
+    page
+      .frameLocator(".webview")
+      .frameLocator('iframe[title="CodeQL Query Results"]')
+      .getByText("#selectalerts32 resultsShow"),
+  ).not.toBeVisible();
+
+  await page
+    .locator("#list_id_6_0")
+    .getByLabel("Hello world on d3/d3 -")
+    .locator("div")
+    .first()
+    .click();
+
+  await expect(
+    page.getByLabel("CodeQL Query Results", { exact: true }).locator("div"),
+  ).toBeVisible({ timeout: 60000 });
+});
--- a/extensions/ql-vscode/tsconfig.json
+++ b/extensions/ql-vscode/tsconfig.json
@@ -24,5 +24,5 @@
    "noEmit": true
  },
  "include": ["src/**/*.ts"],
-  "exclude": ["node_modules", "test", "**/view"]
+  "exclude": ["node_modules", "*.config.ts", "test", "**/view"]
 }