fix: reconciled status names between server/agent/vscode-codeql

2025-03-14 12:45:36 -07:00
parent ffa2f8c2be
commit 55c74ae9a3
8 changed files with 280 additions and 17 deletions
--- a/.env
+++ b/.env
@@ -0,0 +1 @@
+COMPOSE_PROJECT_NAME=mrva
--- a/README.org
+++ b/README.org
@@ -42,10 +42,10 @@

       #* Copy the new binary
       cd ~/work-gh/mrva/mrvaserver
-       docker cp mrvaserver server:/usr/local/bin/mrvaserver
+       docker cp mrvaserver mrva-server:/usr/local/bin/mrvaserver

       #* Restart the binary
-       docker exec server pkill mrvaserver
+       docker exec mrva-server pkill mrvaserver

     #+END_SRC
   - agent
@@ -59,10 +59,10 @@

       #* Copy the new binary
       cd ~/work-gh/mrva/mrvaagent
-       docker cp mrvaagent agent:/usr/local/bin/mrvaagent
+       docker cp mrvaagent mrva-agent:/usr/local/bin/mrvaagent

       #* Restart the binary
-       docker exec agent pkill mrvaagent
+       docker exec mrva-agent pkill mrvaagent

     #+END_SRC

@@ -391,6 +391,15 @@
 *** Submit the analysis job
      : Fprintf.ql > right click > run variant analysis

+*** XX: post-interface reload <2025-03-12 Wed>
+    #+BEGIN_SRC text
+      No orphaned databases found.
+      Reading query history
+      Reading cached query history from '/home/coder/.local/share/code-server/User/workspaceStorage/579340b4/GitHub.vscode-codeql/workspace-query-history.json'.
+      Successfully finished extension initialization.
+      Variant analysis view loaded
+    #+END_SRC
+
 *** XX: bug: <2025-02-19 Wed>
      #+BEGIN_SRC text
        +lle1Oo44XUE87h82ShEVmzTAe02Csf9HKt7kLw2xPU0mt3mYNtbaFEsO3HQNh3tWNS7B+lFoaLw/9t2t9r1//h4P/z1P77+fP35+vP15xb//D9DrYDfAKi6AA==","repositories":["Serial-Studio/Serial-Studio","UEFITool/UEFITool","aircrack-ng/aircrack-ng","bulk-builder/bulk-builder","tesseract/tesseract"]}
--- a/containers/node-build/Dockerfile
+++ b/containers/node-build/Dockerfile
@@ -0,0 +1,34 @@
+# Use a lightweight Debian-based image
+FROM debian:latest
+
+# Set environment variables
+ENV DEBIAN_FRONTEND=noninteractive
+ENV NVM_DIR="/usr/local/nvm"
+
+# Install dependencies
+RUN apt-get update && apt-get install -y \
+    curl \
+    build-essential \
+    git \
+    python3 \
+    unzip \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Create NVM directory
+RUN mkdir -p $NVM_DIR
+
+# Install NVM and Node.js
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash 
+
+# # Manually set Node.js path (since `nvm` is not persistent in non-interactive shells)
+# ENV NODE_PATH="$NVM_DIR/versions/node/$(ls $NVM_DIR/versions/node)/lib/node_modules"
+# ENV PATH="$NVM_DIR/versions/node/$(ls $NVM_DIR/versions/node)/bin:$PATH"
+
+# # Install VSCE (Visual Studio Code Extension CLI)
+# RUN npm install -g @vscode/vsce
+
+# # Set working directory to match extension location
+# WORKDIR /workspace/extensions/ql-vscode
+
+# # Default command: build and package the extension
+# CMD ["sh", "-c", "npm install && npm run build && vsce package"]
--- a/containers/node-build/README.org
+++ b/containers/node-build/README.org
@@ -0,0 +1,47 @@
+* node build container
+
+** Build the container
+  #+BEGIN_SRC sh 
+    cd ~/work-gh/mrva/mrva-docker/containers/node-build
+    docker build -t vscode-extension-builder .
+  #+END_SRC
+
+** Use the container to build a project
+  For a project with structure
+  #+BEGIN_SRC text
+    my-project/
+      vscode-codeql/
+        extensions/
+          ql-vscode/  <-- (Contains package.json and source code)
+  #+END_SRC
+
+  run 
+  #+BEGIN_SRC sh 
+    cd ~/work-gh/mrva/vscode-codeql/
+
+    # start container
+    docker run --rm -ti -v "$(pwd):/workspace" vscode-extension-builder /bin/bash
+
+    # Run the following in the container 
+    # XX: Inside container for now, move to Dockerfile
+
+    # Build the extension
+    cd /workspace/extensions/ql-vscode
+
+    # set up environment
+    source /usr/local/nvm/nvm.sh
+    nvm install
+    npm install -g @vscode/vsce
+    chmod +x /workspace/extensions/ql-vscode/node_modules/.bin/vsce
+    npm install husky
+
+    # build vsix
+    npm install
+    npm run build
+
+    # extension is in /workspace/dist/vscode-codeql-1.13.2-dev.2025.3.12.17.37.12.vsix
+
+    # Back on the host
+    ls ~/work-gh/mrva/vscode-codeql/dist/vscode-codeql*.vsix
+  #+END_SRC
+
--- a/containers/vscode/README.org
+++ b/containers/vscode/README.org
@@ -1,5 +1,5 @@
 * MRVA VS Code server container
-  On the host:
+** On the host

  - mixed install, pre and post-container build
    #+BEGIN_SRC sh 
@@ -13,6 +13,26 @@
             -d -p 9080:9080 code-server-initialized:0.1.24 
    #+END_SRC

+  - build vscode-codeql plugin
+    - [[file:~/work-gh/mrva/mrva-docker/containers/node-build/README.org::*node build container][node build container]] produces
+      : ~/work-gh/mrva/vscode-codeql/dist/vscode-codeql*.vsix
+    - For patching the vscode-codeql extension, edit source there and rebuild; for
+      one-time use, install extension in running code-server container;
+      copy it via
+      #+BEGIN_SRC sh 
+        VSCID=$(docker ps --format "table {{.ID}}\t{{.Image}}\t{{.Names}}" |grep code-server |cut -f1 -d " ")
+
+        cat ~/work-gh/mrva/vscode-codeql/dist/vscode-codeql-1.13.2-dev.2025.3.12.17.37.12.vsix |\
+            docker exec -i $VSCID sh -c 'cat > /home/coder/vscode-codeql-1.13.2-dev.2025.3.12.17.37.12.vsix'
+
+      #+END_SRC
+      install via GUI; the file listing has a right-click entry.
+      Could also use
+      #+BEGIN_SRC sh 
+      /bin/code-server --force --install-extension vscode-codeql-*.vsix
+      #+END_SRC
+
+
  - pre-container build
    #+BEGIN_SRC sh 
      # Collect artifacts
@@ -36,7 +56,132 @@

  - Connect to it at http://localhost:9080/?folder=/home/coder, password is =mrva=.

-  Inside the container:
+** Inside the container
+
+  - [ ] To see changes in container
+    #+BEGIN_SRC sh 
+      docker diff $(docker ps |grep code-server |cut -f1 -d " ")
+
+      # 
+      # determine S_relevant = S_install - S_start, where:
+      # 
+      # S_start: Changes after the container is first run and the VS Code UI is opened.
+      # S_install: Changes after additional steps (like installing software).
+      # S_relevant: The differences that come specifically from the installation steps.
+
+      # Capture S_start before installation
+      docker diff $(docker ps | grep code-server | cut -f1 -d " ") > S_start.txt
+
+      # Perform the installation or modifications
+
+      # Capture S_install after installation
+      docker diff $(docker ps | grep code-server | cut -f1 -d " ") > S_install.txt
+
+      # Compute S_relevant using comm
+      comm -13 <(sort S_start.txt) <(sort S_install.txt) > S_relevant.txt
+
+      less S_relevant.txt
+
+      # As first step, filter wanted files
+      cat S_relevant.txt | \
+          grep -ve /home/coder/mrva-qldemo \
+               -ve /home/coder/.codeql/packages \
+               -ve /home/coder/.local/share/code-server/User/globalStorage/github.vscode-codeql |\
+          less
+
+      # list wanted files
+      C_wanted=$(cat S_relevant.txt | \
+                     grep -e /home/coder/mrva-qldemo \
+                          -e /home/coder/.codeql/packages \
+                          -e /home/coder/.local/share/code-server/User/globalStorage/github.vscode-codeql|\
+                     grep -ve globalStorage/github.vscode-codeql/variant-analyses |\
+                     cut -f2 -d " ")
+      echo "$C_wanted" |less
+
+      # export from container to host
+      VSCID=$(docker ps --format "table {{.ID}}\t{{.Image}}\t{{.Names}}" |grep code-server |cut -f1 -d " ")
+      echo "$C_wanted" | docker exec -i $VSCID tar cf - -T - > mrva-docker-installs.tar
+
+      0:$ tar tvf mrva-docker-installs.tar |sort -rn +4 |head
+      -rwxr-xr-x  0 coder  coder 58954792 Mar 11 20:37 home/coder/.local/share/code-server/User/globalStorage/github.vscode-codeql/distribution11/codeql/cpp/tools/linux64/extractor
+      -rwxr-xr-x  0 coder  coder 58587032 Mar 11 20:37 home/coder/.local/share/code-server/User/globalStorage/github.vscode-codeql/distribution11/codeql/cpp/tools/linux64/cpp-telemetry
+      -rwxr-xr-x  0 coder  coder 57264264 Mar 11 20:37 home/coder/.local/share/code-server/User/globalStorage/github.vscode-codeql/distribution11/codeql/cpp/tools/linux64/extractor-static
+      -rwxr-xr-x  0 coder  coder 57250624 Mar 11 20:37 home/coder/.local/share/code-server/User/globalStorage/github.vscode-codeql/distribution11/codeql/cpp/tools/linux64/trap-cache-reader
+
+      # snapshot again
+      VSCID=$(docker ps --format "table {{.ID}}\t{{.Image}}\t{{.Names}}" |grep code-server |cut -f1 -d " ")
+      docker commit $VSCID mrva-client-vscode:0.3.4
+    #+END_SRC
+    - XX:
+      - plain
+      - add shar
+      - open Fprintf, installs codeql cli
+      - install pack dependencies
+      - S_crth.txt
+      - paste variant analysis repo file
+        + get vscode-selection from hepc
+
+          [[file:~/work-gh/mrva/mrvahepc/db-collection.tmp/metadata.json::{"git_branch": "HEAD", "git_commit_id": "2b41915dac8966e95f9e63638d30769b0d69ad68", "git_repo": "aircrack-ng", "ingestion_datetime_utc": "2024-06-07 16:57:47.683012+00:00", "result_url": "http://hepc/db/db-collection.tmp/aircrack-ng-aircrack-ng-ctsj-41ebbe.zip", "tool_id": "9f2f9642-febb-4435-9204-fb50bbd43de4", "tool_name": "codeql-cpp", "tool_version": "2.17.4", "projname": "aircrack-ng/aircrack-ng"}]]
+
+          #+BEGIN_SRC sh 
+            jq '.projname' < ~/work-gh/mrva/mrvahepc/db-collection.tmp/metadata.json |sort -u
+          #+END_SRC
+
+          Set up 'variant analysis repositories', continuing from the
+          =scratch/vscode-selection.json= file formed via jq:
+        + Select '{}' and open db selection file
+        + paste
+          #+begin_src javascript
+            {
+                "version": 1,
+                "databases": {
+                    "variantAnalysis": {
+                        "repositoryLists": [
+                            {
+                                "name": "mirva-list",
+                                "repositories": [
+                                    "Serial-Studio/Serial-Studio",
+                                    "UEFITool/UEFITool",
+                                    "aircrack-ng/aircrack-ng",
+                                    "bulk-builder/bulk-builder",
+                                    "tesseract/tesseract"
+                                ]
+                            }
+                        ],
+                        "owners": [],
+                        "repositories": []
+                    }
+                },
+                "selected": {
+                    "kind": "variantAnalysisUserDefinedList",
+                    "listName": "mirva-list"
+                }
+            }
+          #+end_src
+      - run variant analysis to see if more downloads happen
+        + open =Fprintf.ql=
+        + right click =>= 'run variant analysis'
+        + XX:
+          #+BEGIN_SRC text
+            Unhandled error: Error saving query history to
+            /home/coder/.local/share/code-server/User/workspaceStorage/579340b4/GitHub.vscode-codeql/workspace-query-history.json:
+            Internal error: exhaustivity checking failure Error: Error saving query history to
+            /home/coder/.local/share/code-server/User/workspaceStorage/579340b4/GitHub.vscode-codeql/workspace-query-history.json:
+            Internal error: exhaustivity checking failure 
+          #+END_SRC
+        + all actual runs successful
+      - snapshot container [[file:~/work-gh/mrva/mrva-docker/README.org::*Update Container Images][Update Container Images]]
+        #+BEGIN_SRC sh 
+          #* Get the container IDs
+
+          docker ps --format "table {{.ID}}\t{{.Image}}\t{{.Names}}"
+          # 0:$ docker ps --format "table {{.ID}}\t{{.Image}}\t{{.Names}}"
+
+          VSCID=$(docker ps --format "table {{.ID}}\t{{.Image}}\t{{.Names}}" |grep code-server |cut -f1 -d " ")
+          docker commit $VSCID mrva-client-vscode:0.3.3
+        #+END_SRC
+
+

  - [ ] Setup inside the container
    XX: Add one for C++
@@ -74,7 +219,7 @@
  - [ ] Set the database as default and run the query =simple.ql=

  - [ ] Add the customized VS Code plugin
-    On the host
+    On the host, build the vscode-codeql plugin
    #+BEGIN_SRC sh 
      cd ~/work-gh/mrva/vscode-codeql
      git checkout mrva-standalone
--- a/containers/vscode/mrva-qldemo/Fprintf.ql
+++ b/containers/vscode/mrva-qldemo/Fprintf.ql
@@ -0,0 +1,14 @@
+/**
+ * @name findPrintf
+ * @description find calls to plain fprintf
+ * @kind problem
+ * @id cpp-fprintf-call
+ * @problem.severity warning
+ */
+
+import cpp
+
+from FunctionCall fc
+where
+  fc.getTarget().getName() = "fprintf"
+select fc, "call of fprintf"
--- a/containers/vscode/mrva-qldemo/qlpack.yml
+++ b/containers/vscode/mrva-qldemo/qlpack.yml
@@ -0,0 +1,5 @@
+library: false
+name: codeql-dataflow-ii-cpp
+version: 0.0.1
+dependencies:
+  codeql/cpp-all: 0.5.3
--- a/docker-compose-demo.yml
+++ b/docker-compose-demo.yml
@@ -7,13 +7,15 @@ services:
        #     - /qldb  # Directory inside the container that contains the data
        volumes:
            - dbsdata:/data
-        container_name: dbssvc
+        container_name: mrva-dbssvc
+        hostname: dbssvc
        networks:
            - backend

    dbstore:
        image: minio/minio:RELEASE.2024-06-11T03-13-30Z
-        container_name: dbstore
+        container_name: mrva-dbstore
+        hostname: dbstore
        ports:
            - "9000:9000"
            - "9001:9001"
@@ -32,6 +34,7 @@ services:
     
    client-ghmrva:
        # ./containers/ghmrva/Dockerfile
+        container_name: mrva-ghmrva
        image: ghcr.io/hohn/client-ghmrva-container:0.1.24 
        network_mode: "service:server" # Share the 'server' network namespace
        environment:
@@ -39,7 +42,8 @@ services:
            
    code-server:
        # ./containers/vscode/Dockerfile
-        image: ghcr.io/hohn/code-server-initialized:0.1.24
+        container_name: mrva-code-server
+        image: code-server-initialized:0.3.3
        ports:
            - "9080:9080"
        # XX: Include codeql binary in code-server (if it's not there already)
@@ -51,7 +55,8 @@ services:
    hepc:
        # ./containers/hepc/Dockerfile
        image: ghcr.io/hohn/mrva-hepc-container:0.1.24
-        container_name: hepc
+        container_name: mrva-hepc
+        hostname: hepc
        command: >-
            mc-hepc-serve --codeql-db-dir db-collection.tmp
            --host "0.0.0.0"
@@ -64,7 +69,7 @@ services:
    rabbitmq:
        image: rabbitmq:3-management
        hostname: rabbitmq
-        container_name: rabbitmq
+        container_name: mrva-rabbitmq
        volumes:
            - ./init/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro
            - ./init/rabbitmq/definitions.json:/etc/rabbitmq/definitions.json:ro
@@ -83,7 +88,8 @@ services:
        # ./containers/server/Dockerfile
        image: ghcr.io/hohn/mrva-server:0.1.24
        command: [ '--mode=container', '--loglevel=debug' ]
-        container_name: server
+        container_name: mrva-server
+        hostname: server
        stop_grace_period: 1s
        depends_on:
            - rabbitmq
@@ -102,7 +108,8 @@ services:
     
    artifactstore:
        image: minio/minio:RELEASE.2024-06-11T03-13-30Z
-        container_name: artifactstore
+        container_name: mrva-artifactstore
+        hostname: artifactstore
        ports:
            - "19000:9000" # host:container
            - "19001:9001"
@@ -121,7 +128,8 @@ services:
        # ./containers/agent/Dockerfile
        image: ghcr.io/hohn/mrva-agent:0.1.24
        command: [ '--loglevel=debug' ]
-        container_name: agent
+        container_name: mrva-agent
+        hostname: agent
        depends_on:
            - rabbitmq
            - dbstore