wip: add k8s manifest as first step in conversion

README.org

@@ -63,167 +63,11 @@
      nerdctl run hello-world
    #+END_SRC
 
-** TODO container build and run
-   #+BEGIN_SRC sh 
-     # Build the containers
-     cd ~/work-gh/mrva/mrva-docker/containers/ghmrva/
-     nerdctl --namespace=default build -t client-ghmrva-container:0.4.0 .
-
-     cd ~/work-gh/mrva/mrva-docker/containers/vscode/
-     nerdctl --namespace=default build -t code-server-initialized:0.4.0 .
-
-     cd ~/work-gh/mrva/mrva-docker/containers/hepc && \
-         rm -fR ./mrvahepc && cp -r ../../../mrvahepc .
-     nerdctl --namespace=default build -t mrva-hepc-container:0.4.0 -f Dockerfile .
-
-     cd ~/work-gh/mrva/mrva-docker/containers/server/
-     nerdctl --namespace=default build -t mrva-server:0.4.0 .
-
-     cd ~/work-gh/mrva/mrva-docker/containers/agent/
-     nerdctl --namespace=default build -t mrva-agent:0.4.0 .
-
-     # List the containers
-     nerdctl images
-
-     # XX:
-     #* Start the containers
-     cd ~/work-gh/mrva/mrva-docker/
-     docker-compose -f docker-compose-demo.yml down
-     docker ps
-
-     docker-compose -f docker-compose-demo.yml up 
-
-     # 
-     #* update the binaries in a running container
-     #** compile server locally
-     cd /Users/hohn/work-gh/mrva/mrvaserver
-     go build
-
-     # Copy the new binary
-     cd /Users/hohn/work-gh/mrva/mrvaserver
-     docker cp mrvaserver mrva-server:/usr/local/bin/mrvaserver
-
-     # Restart the binary
-     docker exec mrva-server pkill mrvaserver
-
-     #** recompile agent
-     # compile locally
-     cd /Users/hohn/work-gh/mrva/mrvaagent
-     go build
-
-     # Copy the new binary
-     cd /Users/hohn/work-gh/mrva/mrvaagent
-     docker cp mrvaagent mrva-agent:/usr/local/bin/mrvaagent
-
-     # Restart the binary
-     docker exec mrva-agent pkill mrvaagent
-
-     #** gh-mrva
-     # compile locally
-     cd /Users/hohn/work-gh/mrva/gh-mrva
-     go mod edit -replace="github.com/GitHubSecurityLab/gh-mrva=/Users/hohn/work-gh/mrva/gh-mrva"
-     go mod tidy 
-     GOOS=linux GOARCH=arm64 go build
-
-     # Copy the new binary
-     cd /Users/hohn/work-gh/mrva/gh-mrva
-     docker cp gh-mrva mrva-ghmrva:/usr/local/bin/gh-mrva
-
-   #+END_SRC
-
+** TODO k8s manifest
    Running the container collection is only needed for local use; docker is fine
    for that.  For k3s etc., we don't need the docker-compose file.
 
-   Abandon conversion attempts for docker-compose to nerdctl.  Go to k3s.
-   
-** TODO set up traffic tracing
-   In a separate terminal:
-   #+BEGIN_SRC sh 
-     limactl shell ubu
-
-     # in vm
-     # capture traffic
-     ip link show | grep br-
-     # form:
-     export BR1=br-442ef935d9f8
-     export BR2=br-e2ebd32bfe66
-     sudo tcpdump -i $BR1 -w trace-$BR1.pcap &
-     sudo tcpdump -i $BR2 -w trace-$BR2.pcap &
-
-     # Note docker0 is DOWN
-
-     # send requests elsewhere
-
-     # view full content:
-     cp trace-$BR1.pcap /tmp/foo
-     tshark -r /tmp/foo -Y 'http.request' -T fields -e http.request.method \
-            -e http.host -e http.request.uri | grep '/repositories/'
-     tshark -r /tmp/foo -Y http -V | less
-
-     cp trace-$BR2.pcap /tmp/foo
-     tshark -r /tmp/foo -Y 'http.request' -T fields -e http.request.method \
-            -e http.host -e http.request.uri | grep '/repositories/'
-     tshark -r /tmp/foo -Y http -V | less
-
-     # should find these
-     tshark -r /tmp/foo \
-            -Y 'http.request.uri contains "/repos/"' \
-            -T fields -e http.request.uri
-     # should not find these
-     tshark -r /tmp/foo \
-            -Y 'http.request.uri contains "/repositories/"' \
-            -T fields -e http.request.uri
-   #+END_SRC
-
-** TODO send requests via cli
-   In [[./bin/ma.send-request]]
-** TODO send requests via vs code plugin
-   In [[*Send request via gui, using vs code][Send request via gui, using vs code]]
-** TODO strace containers and processes from lima
-   #+BEGIN_SRC sh 
-     # In the lima VM
-
-     # #
-     # # install strace in container, if in-container tracing is wanted
-     # docker exec -it mrva-code-server bash
-     # sudo apt update
-     # sudo apt install -y strace
-
-     # 
-     # Find the container's main process PID:
-     docker ps --format "table {{.ID}}\t{{.Image}}\t{{.Names}}"
-     CID=mrva-code-server
-     PID=$(docker inspect --format '{{.State.Pid}}' "$CID")
-     echo $CID $PID
-
-     #
-     # Get the in-container process id.  Example:
-     docker exec mrva-code-server ps -ef | grep -i 'node.*extensionhost'
-     # coder        824      25  2 18:02 ?        00:01:06 /usr/lib/code-server/lib/node --dns-result-order=ipv4first /usr/lib/code-server/lib/vscode/out/bootstrap-fork --type=extensionHost --transformURIs --useHostProxy=false
-     NDID=$(docker exec mrva-code-server ps -ef | grep -i 'node.*extensionhost' | awk '{print($2);}')
-     echo $CID $PID $NDID
-
-     # 
-     # Run strace on the process in the container
-     # using the host's strace
-     sudo nsenter -t "$PID" -n -p \
-          strace -f -tt -s 200 -e trace=connect,sendto,recvfrom -p 824 -o /tmp/strace-extensionhost.log
-     # or
-     sudo nsenter -t "$PID" -n -p -m \
-          strace -f -tt -s 200 -e trace=connect,sendto,recvfrom,write,read \
-          -p $NDID -o /tmp/strace-extensionhost.log
-     # or trace the parent
-     sudo nsenter -t "$PID" -n -p -m \
-          strace -f -tt -s 200 -e trace=connect,sendto,recvfrom,write,read \
-          -p 7 -o /tmp/strace-parent.log
-
-
-     # # using the container's strace
-     # sudo nsenter -t "$PID" -n -p -m \
-     #      strace -f -tt -s 200 -e trace=connect,sendto,recvfrom -p 824 -o /tmp/strace-extensionhost.log
-   #+END_SRC
-
-
+   Initial manifest in k8s/
 
 * Lima VM for running Docker with full control
   Docker Desktop restricts low-level control, especially for tracing inter-container

k8s/configmap.yaml

@@ -0,0 +1,34 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mrva-config
+  namespace: mrva
+data:
+  POSTGRES_USER: "mrva"
+  POSTGRES_DB: "mrvadb"
+  POSTGRES_HOST: "postgres"
+  POSTGRES_PORT: "5432"
+
+  RABBITMQ_HOST: "rabbitmq"
+  RABBITMQ_USER: "user"
+  RABBITMQ_PORT: "5672"
+
+  MINIO_ROOT_USER: "user"
+  MINIO_ENDPOINT: "http://minio:9000"
+
+  MRVA_HEPC_ENDPOINT: "http://hepc:8070"
+  MRVA_HEPC_CACHE_DURATION: "60"
+  MRVA_HEPC_TOOL: "codeql-javascript"
+  MRVA_HEPC_REFROOT: "/refroot/"
+  MRVA_HEPC_OUTDIR: "WedMay7"
+  MRVA_HEPC_DATAVIACLI: "0"
+  MRVA_HEPC_COMMAND: "spigot-cli"
+
+  MRVA_MINIO_VIRTUAL_HOST: "0"
+  SERVER_HOST: "server"
+  SERVER_PORT: "8080"
+
+  ARTIFACT_MINIO_ENDPOINT: "http://minio:9000"
+  QLDB_MINIO_ENDPOINT: "http://minio:9000"
+  QLDB_MINIO_ID: "user"
+  ARTIFACT_MINIO_ID: "user"

k8s/hepc/deployment.yaml

@@ -0,0 +1,33 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hepc
+  namespace: mrva
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: hepc
+  template:
+    metadata:
+      labels:
+        app: hepc
+    spec:
+      containers:
+      - name: hepc
+        image: mrva-hepc-container:0.4.0
+        args:
+          - "mc-hepc-serve"
+          - "--codeql-db-dir"
+          - "db-collection.tmp"
+          - "--host"
+          - "0.0.0.0"
+          - "--port"
+          - "8070"
+        envFrom:
+          - configMapRef:
+              name: mrva-config
+          - secretRef:
+              name: mrva-secrets
+        ports:
+          - containerPort: 8070

k8s/hepc/service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: hepc
+  namespace: mrva
+spec:
+  ports:
+    - port: 8070
+  selector:
+    app: hepc

k8s/minio/pvc.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: minio-data
+  namespace: mrva
+spec:
+  accessModes: ["ReadWriteOnce"]
+  resources:
+    requests:
+      storage: 20Gi

k8s/minio/service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio
+  namespace: mrva
+spec:
+  ports:
+    - port: 9000
+    - port: 9001
+  selector:
+    app: minio

k8s/minio/statefulset.yaml

@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: minio
+  namespace: mrva
+spec:
+  selector:
+    matchLabels:
+      app: minio
+  serviceName: minio
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: minio
+    spec:
+      containers:
+      - name: minio
+        image: minio/minio:RELEASE.2024-06-11T03-13-30Z
+        args: ["server", "/data", "--console-address", ":9001"]
+        envFrom:
+          - configMapRef:
+              name: mrva-config
+          - secretRef:
+              name: mrva-secrets
+        ports:
+          - containerPort: 9000
+          - containerPort: 9001
+        volumeMounts:
+          - name: minio-data
+            mountPath: /data
+  volumeClaimTemplates:
+    - metadata:
+        name: minio-data
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 20Gi

k8s/mrva-agent/deployment.yaml

@@ -0,0 +1,26 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mrva-agent
+  namespace: mrva
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mrva-agent
+  template:
+    metadata:
+      labels:
+        app: mrva-agent
+    spec:
+      containers:
+      - name: mrva-agent
+        image: mrva-agent:0.4.0
+        args: ["--loglevel=debug"]
+        envFrom:
+          - configMapRef:
+              name: mrva-config
+          - secretRef:
+              name: mrva-secrets
+        ports:
+          - containerPort: 8071

k8s/mrva-agent/service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: agent
+  namespace: mrva
+spec:
+  ports:
+    - port: 8071
+  selector:
+    app: mrva-agent

k8s/mrva-server/deployment.yaml

@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mrva-server
+  namespace: mrva
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mrva-server
+  template:
+    metadata:
+      labels:
+        app: mrva-server
+    spec:
+      containers:
+      - name: mrva-server
+        image: mrva-server:0.4.0
+        args: ["--mode=container", "--loglevel=debug"]
+        envFrom:
+          - configMapRef:
+              name: mrva-config
+          - secretRef:
+              name: mrva-secrets
+        ports:
+          - containerPort: 8080
+        readinessProbe:
+          httpGet:
+            path: /health
+            port: 8080
+          initialDelaySeconds: 5
+          periodSeconds: 3

k8s/mrva-server/service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: server
+  namespace: mrva
+spec:
+  ports:
+    - port: 8080
+  selector:
+    app: mrva-server

k8s/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: mrva

k8s/postgres/pvc.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: postgres-data
+  namespace: mrva
+spec:
+  accessModes: ["ReadWriteOnce"]
+  resources:
+    requests:
+      storage: 10Gi

k8s/postgres/service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+  namespace: mrva
+spec:
+  ports:
+    - port: 5432
+  selector:
+    app: postgres

k8s/postgres/statefulset.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: postgres
+  namespace: mrva
+spec:
+  selector:
+    matchLabels:
+      app: postgres
+  serviceName: postgres
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: postgres
+    spec:
+      containers:
+      - name: postgres
+        image: postgres:15
+        envFrom:
+          - configMapRef:
+              name: mrva-config
+          - secretRef:
+              name: mrva-secrets
+        ports:
+          - containerPort: 5432
+        volumeMounts:
+          - name: postgres-data
+            mountPath: /var/lib/postgresql/data
+  volumeClaimTemplates:
+    - metadata:
+        name: postgres-data
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 10Gi

k8s/rabbitmq/deployment.yaml

@@ -0,0 +1,28 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rabbitmq
+  namespace: mrva
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: rabbitmq
+  template:
+    metadata:
+      labels:
+        app: rabbitmq
+    spec:
+      containers:
+      - name: rabbitmq
+        image: rabbitmq:3-management
+        ports:
+          - containerPort: 5672
+          - containerPort: 15672
+        volumeMounts:
+          - name: rabbitmq-data
+            mountPath: /var/lib/rabbitmq
+      volumes:
+        - name: rabbitmq-data
+          persistentVolumeClaim:
+            claimName: rabbitmq-data

k8s/rabbitmq/pvc.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: rabbitmq-data
+  namespace: mrva
+spec:
+  accessModes: ["ReadWriteOnce"]
+  resources:
+    requests:
+      storage: 2Gi

k8s/rabbitmq/service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: rabbitmq
+  namespace: mrva
+spec:
+  ports:
+    - port: 5672
+    - port: 15672
+  selector:
+    app: rabbitmq

k8s/secrets.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: mrva-secrets
+  namespace: mrva
+type: Opaque
+data:
+  POSTGRES_PASSWORD: bXJ2YXBn
+  RABBITMQ_PASSWORD: cGFzc3dvcmQ=
+  MINIO_ROOT_PASSWORD: bW11c3R5ODQzMg==