diff --git a/.env b/.env new file mode 100644 index 0000000..d2bca90 --- /dev/null +++ b/.env @@ -0,0 +1 @@ +COMPOSE_PROJECT_NAME=mrva diff --git a/README.org b/README.org index 2b641ea..07778e3 100644 --- a/README.org +++ b/README.org @@ -42,10 +42,10 @@ #* Copy the new binary cd ~/work-gh/mrva/mrvaserver - docker cp mrvaserver server:/usr/local/bin/mrvaserver + docker cp mrvaserver mrva-server:/usr/local/bin/mrvaserver #* Restart the binary - docker exec server pkill mrvaserver + docker exec mrva-server pkill mrvaserver #+END_SRC - agent @@ -59,10 +59,10 @@ #* Copy the new binary cd ~/work-gh/mrva/mrvaagent - docker cp mrvaagent agent:/usr/local/bin/mrvaagent + docker cp mrvaagent mrva-agent:/usr/local/bin/mrvaagent #* Restart the binary - docker exec agent pkill mrvaagent + docker exec mrva-agent pkill mrvaagent #+END_SRC @@ -391,6 +391,15 @@ *** Submit the analysis job : Fprintf.ql > right click > run variant analysis +*** XX: post-interface reload <2025-03-12 Wed> + #+BEGIN_SRC text + No orphaned databases found. + Reading query history + Reading cached query history from '/home/coder/.local/share/code-server/User/workspaceStorage/579340b4/GitHub.vscode-codeql/workspace-query-history.json'. + Successfully finished extension initialization. + Variant analysis view loaded + #+END_SRC + *** XX: bug: <2025-02-19 Wed> #+BEGIN_SRC text +lle1Oo44XUE87h82ShEVmzTAe02Csf9HKt7kLw2xPU0mt3mYNtbaFEsO3HQNh3tWNS7B+lFoaLw/9t2t9r1//h4P/z1P77+fP35+vP15xb//D9DrYDfAKi6AA==","repositories":["Serial-Studio/Serial-Studio","UEFITool/UEFITool","aircrack-ng/aircrack-ng","bulk-builder/bulk-builder","tesseract/tesseract"]} @@ -552,7 +561,7 @@ grep 'docker tag' containers/*/*.org containers/*/Makefile tagpushimg () { name=$1 version=$2 - + docker tag $name:$version ghcr.io/hohn/$name:$version docker push ghcr.io/hohn/$name:$version } diff --git a/containers/node-build/Dockerfile b/containers/node-build/Dockerfile new file mode 100644 index 0000000..47ca208 --- /dev/null +++ b/containers/node-build/Dockerfile @@ -0,0 +1,34 @@ +# Use a lightweight Debian-based image +FROM debian:latest + +# Set environment variables +ENV DEBIAN_FRONTEND=noninteractive +ENV NVM_DIR="/usr/local/nvm" + +# Install dependencies +RUN apt-get update && apt-get install -y \ + curl \ + build-essential \ + git \ + python3 \ + unzip \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# Create NVM directory +RUN mkdir -p $NVM_DIR + +# Install NVM and Node.js +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash + +# # Manually set Node.js path (since `nvm` is not persistent in non-interactive shells) +# ENV NODE_PATH="$NVM_DIR/versions/node/$(ls $NVM_DIR/versions/node)/lib/node_modules" +# ENV PATH="$NVM_DIR/versions/node/$(ls $NVM_DIR/versions/node)/bin:$PATH" + +# # Install VSCE (Visual Studio Code Extension CLI) +# RUN npm install -g @vscode/vsce + +# # Set working directory to match extension location +# WORKDIR /workspace/extensions/ql-vscode + +# # Default command: build and package the extension +# CMD ["sh", "-c", "npm install && npm run build && vsce package"] diff --git a/containers/node-build/README.org b/containers/node-build/README.org new file mode 100644 index 0000000..160bfbe --- /dev/null +++ b/containers/node-build/README.org @@ -0,0 +1,47 @@ +* node build container + +** Build the container + #+BEGIN_SRC sh + cd ~/work-gh/mrva/mrva-docker/containers/node-build + docker build -t vscode-extension-builder . + #+END_SRC + +** Use the container to build a project + For a project with structure + #+BEGIN_SRC text + my-project/ + vscode-codeql/ + extensions/ + ql-vscode/ <-- (Contains package.json and source code) + #+END_SRC + + run + #+BEGIN_SRC sh + cd ~/work-gh/mrva/vscode-codeql/ + + # start container + docker run --rm -ti -v "$(pwd):/workspace" vscode-extension-builder /bin/bash + + # Run the following in the container + # XX: Inside container for now, move to Dockerfile + + # Build the extension + cd /workspace/extensions/ql-vscode + + # set up environment + source /usr/local/nvm/nvm.sh + nvm install + npm install -g @vscode/vsce + chmod +x /workspace/extensions/ql-vscode/node_modules/.bin/vsce + npm install husky + + # build vsix + npm install + npm run build + + # extension is in /workspace/dist/vscode-codeql-1.13.2-dev.2025.3.12.17.37.12.vsix + + # Back on the host + ls ~/work-gh/mrva/vscode-codeql/dist/vscode-codeql*.vsix + #+END_SRC + diff --git a/containers/vscode/README.org b/containers/vscode/README.org index ac9e0bd..f141d73 100644 --- a/containers/vscode/README.org +++ b/containers/vscode/README.org @@ -1,5 +1,5 @@ * MRVA VS Code server container - On the host: +** On the host - mixed install, pre and post-container build #+BEGIN_SRC sh @@ -13,6 +13,26 @@ -d -p 9080:9080 code-server-initialized:0.1.24 #+END_SRC + - build vscode-codeql plugin + - [[file:~/work-gh/mrva/mrva-docker/containers/node-build/README.org::*node build container][node build container]] produces + : ~/work-gh/mrva/vscode-codeql/dist/vscode-codeql*.vsix + - For patching the vscode-codeql extension, edit source there and rebuild; for + one-time use, install extension in running code-server container; + copy it via + #+BEGIN_SRC sh + VSCID=$(docker ps --format "table {{.ID}}\t{{.Image}}\t{{.Names}}" |grep code-server |cut -f1 -d " ") + + cat ~/work-gh/mrva/vscode-codeql/dist/vscode-codeql-1.13.2-dev.2025.3.12.17.37.12.vsix |\ + docker exec -i $VSCID sh -c 'cat > /home/coder/vscode-codeql-1.13.2-dev.2025.3.12.17.37.12.vsix' + + #+END_SRC + install via GUI; the file listing has a right-click entry. + Could also use + #+BEGIN_SRC sh + /bin/code-server --force --install-extension vscode-codeql-*.vsix + #+END_SRC + + - pre-container build #+BEGIN_SRC sh # Collect artifacts @@ -36,7 +56,132 @@ - Connect to it at http://localhost:9080/?folder=/home/coder, password is =mrva=. - Inside the container: +** Inside the container + + - [ ] To see changes in container + #+BEGIN_SRC sh + docker diff $(docker ps |grep code-server |cut -f1 -d " ") + + # + # determine S_relevant = S_install - S_start, where: + # + # S_start: Changes after the container is first run and the VS Code UI is opened. + # S_install: Changes after additional steps (like installing software). + # S_relevant: The differences that come specifically from the installation steps. + + # Capture S_start before installation + docker diff $(docker ps | grep code-server | cut -f1 -d " ") > S_start.txt + + # Perform the installation or modifications + + # Capture S_install after installation + docker diff $(docker ps | grep code-server | cut -f1 -d " ") > S_install.txt + + # Compute S_relevant using comm + comm -13 <(sort S_start.txt) <(sort S_install.txt) > S_relevant.txt + + less S_relevant.txt + + # As first step, filter wanted files + cat S_relevant.txt | \ + grep -ve /home/coder/mrva-qldemo \ + -ve /home/coder/.codeql/packages \ + -ve /home/coder/.local/share/code-server/User/globalStorage/github.vscode-codeql |\ + less + + # list wanted files + C_wanted=$(cat S_relevant.txt | \ + grep -e /home/coder/mrva-qldemo \ + -e /home/coder/.codeql/packages \ + -e /home/coder/.local/share/code-server/User/globalStorage/github.vscode-codeql|\ + grep -ve globalStorage/github.vscode-codeql/variant-analyses |\ + cut -f2 -d " ") + echo "$C_wanted" |less + + # export from container to host + VSCID=$(docker ps --format "table {{.ID}}\t{{.Image}}\t{{.Names}}" |grep code-server |cut -f1 -d " ") + echo "$C_wanted" | docker exec -i $VSCID tar cf - -T - > mrva-docker-installs.tar + + 0:$ tar tvf mrva-docker-installs.tar |sort -rn +4 |head + -rwxr-xr-x 0 coder coder 58954792 Mar 11 20:37 home/coder/.local/share/code-server/User/globalStorage/github.vscode-codeql/distribution11/codeql/cpp/tools/linux64/extractor + -rwxr-xr-x 0 coder coder 58587032 Mar 11 20:37 home/coder/.local/share/code-server/User/globalStorage/github.vscode-codeql/distribution11/codeql/cpp/tools/linux64/cpp-telemetry + -rwxr-xr-x 0 coder coder 57264264 Mar 11 20:37 home/coder/.local/share/code-server/User/globalStorage/github.vscode-codeql/distribution11/codeql/cpp/tools/linux64/extractor-static + -rwxr-xr-x 0 coder coder 57250624 Mar 11 20:37 home/coder/.local/share/code-server/User/globalStorage/github.vscode-codeql/distribution11/codeql/cpp/tools/linux64/trap-cache-reader + + # snapshot again + VSCID=$(docker ps --format "table {{.ID}}\t{{.Image}}\t{{.Names}}" |grep code-server |cut -f1 -d " ") + docker commit $VSCID mrva-client-vscode:0.3.4 + #+END_SRC + - XX: + - plain + - add shar + - open Fprintf, installs codeql cli + - install pack dependencies + - S_crth.txt + - paste variant analysis repo file + + get vscode-selection from hepc + + [[file:~/work-gh/mrva/mrvahepc/db-collection.tmp/metadata.json::{"git_branch": "HEAD", "git_commit_id": "2b41915dac8966e95f9e63638d30769b0d69ad68", "git_repo": "aircrack-ng", "ingestion_datetime_utc": "2024-06-07 16:57:47.683012+00:00", "result_url": "http://hepc/db/db-collection.tmp/aircrack-ng-aircrack-ng-ctsj-41ebbe.zip", "tool_id": "9f2f9642-febb-4435-9204-fb50bbd43de4", "tool_name": "codeql-cpp", "tool_version": "2.17.4", "projname": "aircrack-ng/aircrack-ng"}]] + + #+BEGIN_SRC sh + jq '.projname' < ~/work-gh/mrva/mrvahepc/db-collection.tmp/metadata.json |sort -u + #+END_SRC + + Set up 'variant analysis repositories', continuing from the + =scratch/vscode-selection.json= file formed via jq: + + Select '{}' and open db selection file + + paste + #+begin_src javascript + { + "version": 1, + "databases": { + "variantAnalysis": { + "repositoryLists": [ + { + "name": "mirva-list", + "repositories": [ + "Serial-Studio/Serial-Studio", + "UEFITool/UEFITool", + "aircrack-ng/aircrack-ng", + "bulk-builder/bulk-builder", + "tesseract/tesseract" + ] + } + ], + "owners": [], + "repositories": [] + } + }, + "selected": { + "kind": "variantAnalysisUserDefinedList", + "listName": "mirva-list" + } + } + #+end_src + - run variant analysis to see if more downloads happen + + open =Fprintf.ql= + + right click =>= 'run variant analysis' + + XX: + #+BEGIN_SRC text + Unhandled error: Error saving query history to + /home/coder/.local/share/code-server/User/workspaceStorage/579340b4/GitHub.vscode-codeql/workspace-query-history.json: + Internal error: exhaustivity checking failure Error: Error saving query history to + /home/coder/.local/share/code-server/User/workspaceStorage/579340b4/GitHub.vscode-codeql/workspace-query-history.json: + Internal error: exhaustivity checking failure + #+END_SRC + + all actual runs successful + - snapshot container [[file:~/work-gh/mrva/mrva-docker/README.org::*Update Container Images][Update Container Images]] + #+BEGIN_SRC sh + #* Get the container IDs + + docker ps --format "table {{.ID}}\t{{.Image}}\t{{.Names}}" + # 0:$ docker ps --format "table {{.ID}}\t{{.Image}}\t{{.Names}}" + + VSCID=$(docker ps --format "table {{.ID}}\t{{.Image}}\t{{.Names}}" |grep code-server |cut -f1 -d " ") + docker commit $VSCID mrva-client-vscode:0.3.3 + #+END_SRC + + - [ ] Setup inside the container XX: Add one for C++ @@ -74,7 +219,7 @@ - [ ] Set the database as default and run the query =simple.ql= - [ ] Add the customized VS Code plugin - On the host + On the host, build the vscode-codeql plugin #+BEGIN_SRC sh cd ~/work-gh/mrva/vscode-codeql git checkout mrva-standalone diff --git a/containers/vscode/mrva-qldemo/Fprintf.ql b/containers/vscode/mrva-qldemo/Fprintf.ql new file mode 100644 index 0000000..57e3094 --- /dev/null +++ b/containers/vscode/mrva-qldemo/Fprintf.ql @@ -0,0 +1,14 @@ +/** + * @name findPrintf + * @description find calls to plain fprintf + * @kind problem + * @id cpp-fprintf-call + * @problem.severity warning + */ + +import cpp + +from FunctionCall fc +where + fc.getTarget().getName() = "fprintf" +select fc, "call of fprintf" diff --git a/containers/vscode/mrva-qldemo/qlpack.yml b/containers/vscode/mrva-qldemo/qlpack.yml new file mode 100644 index 0000000..bda2149 --- /dev/null +++ b/containers/vscode/mrva-qldemo/qlpack.yml @@ -0,0 +1,5 @@ +library: false +name: codeql-dataflow-ii-cpp +version: 0.0.1 +dependencies: + codeql/cpp-all: 0.5.3 diff --git a/docker-compose-demo.yml b/docker-compose-demo.yml index a816f1d..ac9b912 100644 --- a/docker-compose-demo.yml +++ b/docker-compose-demo.yml @@ -7,13 +7,15 @@ services: # - /qldb # Directory inside the container that contains the data volumes: - dbsdata:/data - container_name: dbssvc + container_name: mrva-dbssvc + hostname: dbssvc networks: - backend dbstore: image: minio/minio:RELEASE.2024-06-11T03-13-30Z - container_name: dbstore + container_name: mrva-dbstore + hostname: dbstore ports: - "9000:9000" - "9001:9001" @@ -32,6 +34,7 @@ services: client-ghmrva: # ./containers/ghmrva/Dockerfile + container_name: mrva-ghmrva image: ghcr.io/hohn/client-ghmrva-container:0.1.24 network_mode: "service:server" # Share the 'server' network namespace environment: @@ -39,7 +42,8 @@ services: code-server: # ./containers/vscode/Dockerfile - image: ghcr.io/hohn/code-server-initialized:0.1.24 + container_name: mrva-code-server + image: code-server-initialized:0.3.3 ports: - "9080:9080" # XX: Include codeql binary in code-server (if it's not there already) @@ -51,7 +55,8 @@ services: hepc: # ./containers/hepc/Dockerfile image: ghcr.io/hohn/mrva-hepc-container:0.1.24 - container_name: hepc + container_name: mrva-hepc + hostname: hepc command: >- mc-hepc-serve --codeql-db-dir db-collection.tmp --host "0.0.0.0" @@ -64,7 +69,7 @@ services: rabbitmq: image: rabbitmq:3-management hostname: rabbitmq - container_name: rabbitmq + container_name: mrva-rabbitmq volumes: - ./init/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro - ./init/rabbitmq/definitions.json:/etc/rabbitmq/definitions.json:ro @@ -83,7 +88,8 @@ services: # ./containers/server/Dockerfile image: ghcr.io/hohn/mrva-server:0.1.24 command: [ '--mode=container', '--loglevel=debug' ] - container_name: server + container_name: mrva-server + hostname: server stop_grace_period: 1s depends_on: - rabbitmq @@ -102,7 +108,8 @@ services: artifactstore: image: minio/minio:RELEASE.2024-06-11T03-13-30Z - container_name: artifactstore + container_name: mrva-artifactstore + hostname: artifactstore ports: - "19000:9000" # host:container - "19001:9001" @@ -121,7 +128,8 @@ services: # ./containers/agent/Dockerfile image: ghcr.io/hohn/mrva-agent:0.1.24 command: [ '--loglevel=debug' ] - container_name: agent + container_name: mrva-agent + hostname: agent depends_on: - rabbitmq - dbstore @@ -134,7 +142,7 @@ services: - MRVA_HEPC_CACHE_DURATION=60 networks: - backend - + networks: backend: driver: bridge