Set up and push Docker containers for demonstration purposes

These containers take the place of a desktop install

Makefile

@@ -14,22 +14,34 @@ html: README.html
 dbt: client-qldbtools-container
 client-qldbtools-container:
 	cd client/containers/qldbtools && \
-		docker build -t $@ .
+		docker build -t $@:0.1.24 .
+	touch $@
 
 # Run a shell in the container with the qldbtools
-dbt-run:
+dbt-run: dbt
-	docker run --rm -it client-qldbtools-container /bin/bash
+	docker run --rm -it client-qldbtools-container:0.1.24 /bin/bash
 
-dbt-run:
+# Run one of the scripts in the container as check
-	docker run --rm -it client-qldbtools-container /bin/bash
+dbt-check: dbt
+	docker run --rm -it client-qldbtools-container:0.1.24 mc-db-initial-info
+
+dbt-push: dbt
+	docker tag client-qldbtools-container:0.1.24 ghcr.io/hohn/client-qldbtools-container:0.1.24 
+	docker push ghcr.io/hohn/client-qldbtools-container:0.1.24
+	touch $@
 
-dbt-check:
-	docker run --rm -it client-qldbtools-container mc-db-initial-info
 
 ghm: client-ghmrva-container
 client-ghmrva-container:
 	cd client/containers/ghmrva && \
-		docker build -t $@ .
+		docker build -t $@:0.1.24 .
+	touch $@
+
+ghm-push: ghm
+	docker tag client-ghmrva-container:0.1.24 ghcr.io/hohn/client-ghmrva-container:0.1.24 
+	docker push ghcr.io/hohn/client-ghmrva-container:0.1.24 
+	touch $@
+
 ghm-run:
 	docker run --rm client-ghmrva-container --help
 

client/containers/ghmrva/README.org

@@ -0,0 +1,12 @@
+* MRVA cli tools container
+  Set up via
+  #+BEGIN_SRC sh 
+    # Build
+    cd ~/work-gh/mrva/mrvacommander/
+    make ghm
+    # Run
+    docker run --rm gh-mrva
+    # Push
+    make ghm-push
+  #+END_SRC
+  

client/containers/mrvadata/Dockerfile

@@ -4,14 +4,6 @@ FROM busybox
 # This data container holds data from existing runs.  It cannot be built without
 # ready data.
 
-# Before running docker build, make sure you have set up the directory mirrors
-# dbstore-data and scratch.
-
-# E.g.:
-    # cd ~/work-gh/mrva/mrvacommander/client/containers/mrvadata/
-    # mkdir tmp
-    # cp -r ../../../dbstore-data   tmp
-    # cp -r ../../qldbtools/scratch tmp
 
 # Mirror parts of the host directory structure in the container
 COPY tmp/dbstore-data /data/mrvacommander/dbstore-data

client/containers/mrvadata/Makefile

@@ -1,16 +0,0 @@
-all: image
-
-image:
-	docker build -t mrvadata .
-
-image-shell: image
-	docker run --rm -it mrvadata sh
-
-image-push: image
-	docker tag mrvadata ghcr.io/hohn/mrvadata:0.1.24
-	docker push ghcr.io/hohn/mrvadata:0.1.24
-
-image-test:
-	docker pull ghcr.io/hohn/mrvadata:0.1.24
-	docker run --rm -it --name test-mrvadata ghcr.io/hohn/mrvadata:0.1.24 sh
-

client/containers/mrvadata/REAME.org

@@ -0,0 +1,86 @@
+* Data container for demonstration 
+  Contains [[https://github.com/hohn/mrva-open-source-download.git][mrva-open-source-download]] repo and some downloads.
+
+  This container has several non-trivial build prerequisites which is why it
+  exists in the first place.  The [[*Building the container][Building the container]] instructions are intended
+  for building it /after/ much work has been done, to make it easy to share that
+  work with others.
+
+  If you just want to use the container, see [[*Using the container][Using the container]]
+
+** Using the container
+   To get and test this image, run the container via the following
+   #+BEGIN_SRC sh 
+     docker pull ghcr.io/hohn/mrvadata:0.1.24
+     docker run --rm -it --name test-mrvadata ghcr.io/hohn/mrvadata:0.1.24 sh
+   #+END_SRC
+
+   Inside the container, check the contents via
+   #+BEGIN_SRC sh 
+     ls /data/mrvacommander/qldbtools/*
+     ls /data/mrvacommander/dbstore-data/*
+   #+END_SRC
+
+   This should be something of the form
+   #+BEGIN_SRC text
+     / # ls /data/mrvacommander/qldbtools/*
+
+     cp-dbs-to-upload.txt       db-info-1.csv              gh-mrva-selection.json
+     cp-gh-mrva-selection.json  db-info-2.csv              selection-full-info
+     cp-vscode-selection.json   db-info-3.csv              vscode-selection.json
+
+     / # ls /data/mrvacommander/dbstore-data/*
+
+     BoomingTech$Piccoloctsj6d7177.zip           mawww$kakounectsjc54fab.zip
+     KhronosGroup$OpenXR-SDKctsj984ee6.zip       microsoft$node-native-keymapctsj4cc9a2.zip
+     OpenRCT2$OpenRCT2ctsj975d7c.zip             nem0$LumixEnginectsjfab756.zip
+     StanfordLegion$legionctsj39cbe4.zip         pocoproject$pococtsj26b932.zip
+     USCiLab$cerealctsj264953.zip                quickfix$quickfixctsjebfd13.zip
+     WinMerge$winmergectsj101305.zip             rui314$moldctsjfec16a.zip
+     draios$sysdigctsj12c02d.zip                 swig$swigctsj78bcd3.zip
+     gildor2$UEViewerctsjfefdd8.zip              tdlib$telegram-bot-apictsj8529d9.zip
+     git-for-windows$gitctsjb7c2bd.zip           timescale$timescaledbctsjf617cf.zip
+     google$orbitctsj9bbeaf.zip                  xoreaxeaxeax$movfuscatorctsj8f7e5b.zip
+     libfuse$libfusectsj7a66a4.zip               xrootd$xrootdctsje4b745.zip
+     luigirizzo$netmapctsj6417fa.zip
+
+   #+END_SRC
+
+
+
+** Building the container
+
+   The Dockerfile is used to make an image contain data; it uses busybox as
+   base.
+
+   Steps to build this image from a preloaded minio instance[fn:1].
+
+   - Prepare data.  This requires that a minio instance has already been populated
+     and will be documented in the future.
+     # TODO
+     #+BEGIN_SRC sh 
+       cd ~/work-gh/mrva/mrvacommander/client/containers/mrvadata/ && \
+           rm -fR tmp && \
+           mkdir tmp
+       cp -r ../../../dbstore-data   tmp
+       cp -r ../../qldbtools/scratch tmp
+     #+END_SRC
+
+   - Make image
+     #+BEGIN_SRC sh 
+       cd ~/work-gh/mrva/mrvacommander/client/containers/mrvadata/ 
+       docker build -t mrvadata .
+     #+END_SRC
+
+   - Tag and Push to GHCR
+     #+BEGIN_SRC sh 
+       echo $GHCR_GITHUB_TOKEN | docker login ghcr.io -u hohn --password-stdin
+
+       cd ~/work-gh/mrva/mrvacommander/client/containers/mrvadata
+       docker tag mrvadata ghcr.io/hohn/mrvadata:0.1.24
+       docker push ghcr.io/hohn/mrvadata:0.1.24
+     #+END_SRC
+
+* Footnotes
+
+[fn:1] This is itself effort, hence the creation of this image. 

client/containers/qldbtools/README.org

@@ -0,0 +1,18 @@
+* MRVA python tools container
+  Set upDocker image with python 3.11 and pip and the qldbtools
+
+  #+BEGIN_SRC sh 
+    # Build
+    cd ~/work-gh/mrva/mrvacommander
+    make dbt
+
+    # Check
+    cd ~/work-gh/mrva/mrvacommander
+    make dbt-check
+
+    # Use
+    cd ~/work-gh/mrva/mrvacommander
+    make dbt-run
+
+  #+END_SRC
+  

client/containers/vscode/Dockerfile

@@ -0,0 +1,65 @@
+FROM codercom/code-server:4.92.2-debian
+
+# ======================
+# Pre-install a custom JDK for this platform and redirect CodeQL to it
+
+USER root
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install packages
+RUN apt-get update && apt-get install --no-install-recommends --assume-yes \
+    ca-certificates                                 \
+    curl                                            \
+    default-jdk                                     \
+    git                                             \
+    libcurl4-openssl-dev                            \
+    libssl-dev                                      \
+    python3                                         \
+    python3-dev                                     \
+    unzip
+
+# Build argument for CodeQL version, defaulting to the latest release
+ARG CODEQL_VERSION=latest
+
+# If the version is 'latest', get the latest release version from GitHub, unzip
+# the bundle into /opt, and delete the archive
+RUN if [ "$CODEQL_VERSION" = "latest" ]; then \
+    CODEQL_VERSION=$(curl -s https://api.github.com/repos/github/codeql-cli-binaries/releases/latest | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/'); \
+    fi && \
+    echo "Using CodeQL version $CODEQL_VERSION" && \
+    curl -L "https://github.com/github/codeql-cli-binaries/releases/download/$CODEQL_VERSION/codeql-linux64.zip" -o /tmp/codeql.zip && \
+    unzip /tmp/codeql.zip -d /opt && \
+    rm /tmp/codeql.zip && \
+    chmod -R +x /opt/codeql
+
+# ======================
+# Install code-server
+USER coder
+
+# Set environment variables
+ENV PASSWORD mrva
+
+# Install VS Code extensions as user root -- globally
+RUN code-server --install-extension ms-python.python \
+    && code-server --install-extension esbenp.prettier-vscode \
+    && code-server --install-extension GitHub.vscode-codeql
+
+# Expose the port that Code Server runs on
+EXPOSE 9080
+
+# Point CodeQL to the java binary for this platform
+ENV CODEQL_JAVA_HOME=/usr
+
+# Add
+#   codeQl.cli.executablePath
+# to user settings.
+# This is in addition to the environment variable CODEQL_JAVA_HOME which has no
+# effect on the plugin
+COPY ./settings.json  /home/coder/.local/share/code-server/User/
+
+# Start Code Server
+ENTRYPOINT ["dumb-init", "code-server", "--bind-addr", "0.0.0.0:9080", "."]
+
+# Run as the coder user
+USER coder

client/containers/vscode/Makefile

@@ -0,0 +1,26 @@
+all: code-server-initialized
+
+CSI_TARGET := code-server-initialized:0.1.24
+csi: code-server-initialized
+code-server-initialized:
+	docker build -t ${CSI_TARGET} .
+	touch $@
+
+csi-serve: csi
+	docker run -d -p 9080:9080 ${CSI_TARGET}
+
+clean:
+	-docker rmi -f ${CSI_TARGET}
+	-rm code-server-initialized
+
+# Targets below are used after some manual setup of the container.  See README.org
+# for details
+
+csi-push: csi
+	docker tag ${CSI_TARGET} ghcr.io/hohn/${CSI_TARGET}
+	docker push ghcr.io/hohn/${CSI_TARGET}
+	touch $@
+
+csi-test:
+	docker pull ghcr.io/hohn/${CSI_TARGET}
+	docker run --rm -it --name test-code-server-codeql ghcr.io/hohn/${CSI_TARGET} sh

client/containers/vscode/README.org

@@ -0,0 +1,60 @@
+* MRVA VS Code server container
+  On the host:
+
+  - Build the container via 
+    #+BEGIN_SRC sh 
+      make csi
+    #+END_SRC
+
+  - Run the container via
+    #+BEGIN_SRC sh 
+      make csi-serve
+    #+END_SRC
+
+  - Connect to it at http://localhost:9080/?folder=/home/coder, password is =mrva=.
+
+  Inside the container:
+
+  - Setup inside the container
+    #+BEGIN_SRC shell
+      export PATH=/opt/codeql:$PATH
+      codeql pack init qldemo
+      cd qldemo
+      codeql pack add codeql/python-all@1.0.6
+    #+END_SRC
+
+  - Open a new file =qldemo/simple.ql= and add this this query to it.  The plugin
+    will download the CodeQL binaries (but never use them -- the configuration
+    redirects) 
+    #+BEGIN_SRC java
+      import python
+          select 42
+    #+END_SRC
+
+  - Create database.
+    #+BEGIN_SRC sh 
+      cd ~/qldemo
+
+      cat > short.py <<EOF
+      print('hello world')
+      EOF
+      export PATH=/opt/codeql:$PATH  
+      codeql database create --language=python -s . -v short-db
+    #+END_SRC
+
+  - Set the database as default and run the query.
+
+  - Capture the state of this container and create a new image from it
+    #+BEGIN_SRC sh 
+      docker ps
+      docker commit 0c15aeeaa914 code-server-initialized:0.1.24
+      docker kill 0c15aeeaa914
+      docker run -d -p 9080:9080 code-server-initialized:0.1.24
+    #+END_SRC
+
+  - Push this container
+    [[file:~/work-gh/mrva/mrvacommander/client/containers/vscode/Makefile::image-push: image]]
+    #+BEGIN_SRC sh 
+      docker images |head
+      make csi-push
+    #+END_SRC

client/containers/vscode/settings.json

@@ -0,0 +1,4 @@
+{
+    "codeQL.runningQueries.numberOfThreads": 2,
+    "codeQl.cli.executablePath": "/opt/codeql"
+}

cmd/agent/Dockerfile

@@ -48,4 +48,4 @@ WORKDIR /app
 COPY --from=builder /bin/mrva_agent ./mrva_agent
 
 # Run the agent
-ENTRYPOINT ["./mrva_agent"]
+ENTRYPOINT ["./mrva_agent"]

docker-compose-demo.yml

@@ -0,0 +1,111 @@
+services:
+  code-server:
+    image: codercom/code-server
+    ports:
+      - "9080:8080"
+    volumes:
+      - ./project:/home/coder/project
+      - ~/.config/code-server:/home/coder/.config/code-server
+      - ~/.local/share/code-server:/home/coder/.local/share/code-server
+    environment:
+      - PASSWORD=mrva
+
+  rabbitmq:
+    image: rabbitmq:3-management
+    hostname: rabbitmq
+    container_name: rabbitmq
+    volumes:
+      - ./init/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro
+      - ./init/rabbitmq/definitions.json:/etc/rabbitmq/definitions.json:ro
+    ports:
+      - "5672:5672"
+      - "15672:15672"
+    networks:
+      - backend
+    healthcheck:
+      test: rabbitmq-diagnostics check_port_connectivity
+      interval: 30s
+      timeout: 30s
+      retries: 10
+
+  server:
+    build:
+      context: .
+      dockerfile: ./cmd/server/Dockerfile
+    command: [ '--mode=container', '--loglevel=debug' ]
+    container_name: server
+    stop_grace_period: 1s
+    ports:
+      # - "8081:8080" # host:container for proxy
+      - "8080:8080" # host:container
+    depends_on:
+      - rabbitmq
+      - dbstore
+      - artifactstore
+    networks:
+      - backend
+    env_file:
+      - path: ./.env.container
+        required: true
+
+  dbstore:
+    image: minio/minio:RELEASE.2024-06-11T03-13-30Z
+    container_name: dbstore
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    env_file:
+      - path: .env.container
+        required: true
+
+    command: server /data --console-address ":9001"
+    # Demo: Mount preloaded minio backing store
+    volumes:
+      - type: volume
+        source: dbstore-volume
+        target: /data/dbs
+        volume:
+          nocopy: true
+    networks:
+      - backend
+
+  artifactstore:
+    image: minio/minio:RELEASE.2024-06-11T03-13-30Z
+    container_name: artifactstore
+    ports:
+      - "19000:9000" # host:container
+      - "19001:9001"
+    env_file:
+      - path: ./.env.container
+        required: true
+    command: server /data --console-address ":9001"
+    volumes:
+      - ./qpstore-data:/data
+    networks:
+      - backend
+
+  agent:
+    build:
+      context: .
+      dockerfile: ./cmd/agent/Dockerfile
+    command: [ '--loglevel=debug' ]
+    container_name: agent
+    depends_on:
+      - rabbitmq
+      - dbstore
+      - artifactstore
+    env_file:
+      - path: ./.env.container
+        required: true
+    networks:
+      - backend
+
+networks:
+  backend:
+    driver: bridge
+
+# Demo: Provide preloaded minio backing store
+volumes:
+  dbstore-volume:
+    external: true
+    name: dbstore-container:/data