Add hepc-{init,serve} to provide codeql database access via http

client/qldbtools/bin/hepc-init

@@ -18,7 +18,7 @@ def log(level, message):
     timestamp = date("+%Y-%m-%d %H:%M:%S").strip()
     print(f"{colors[level]}[{timestamp}] [{level}] {message}{colors['RESET']}", file=sys.stderr)
 
-# Generate a CID
+# Generate a CID (cumulative id)
 def generate_cid(cli_version, creation_time, primary_language, sha):
     hash_input = f"{cli_version} {creation_time} {primary_language} {sha}".encode()
     return hashlib.sha256(hash_input).hexdigest()[:6]
@@ -50,7 +50,6 @@ def process_db_file(zip_path, db_collection_dir):
         cli_version = creation_metadata["cliVersion"]
         creation_time = creation_metadata["creationTime"]
         source_location_prefix = local.path(yaml_data["sourceLocationPrefix"])
-
         repo = source_location_prefix.name
         owner = source_location_prefix.parent.name
         cid = generate_cid(cli_version, creation_time, primary_language, sha)

client/qldbtools/bin/hepc-serve

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+import logging
+from pathlib import Path
+from plumbum import cli
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import FileResponse
+import uvicorn
+
+# Logging configuration
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[logging.StreamHandler()]
+)
+logger = logging.getLogger(__name__)
+
+# FastAPI application
+app = FastAPI()
+db_dir = None  # This will be set by the CLI application
+
+@app.get("/db/{file_path:path}")
+def serve_file(file_path: str):
+    """
+    Serve files from the database directory, such as .zip files or metadata.json.
+    """
+    logger.info(f"Requested file: {file_path}")
+    # Resolve symlink
+    resolved_path = Path(file_path).resolve(strict=True)
+    logger.info(f"file resolved to: {resolved_path}")
+    if not resolved_path.exists():
+        logger.error(f"File not found: {resolved_path}")
+        raise HTTPException(status_code=404, detail=f"{resolved_path} not found")
+    return FileResponse(resolved_path)
+
+
+@app.get("/index")
+@app.get("/api/v1/latest_results/codeql-all")
+def serve_metadata_json():
+    """
+    Serve the metadata.json file for multiple routes.
+    """
+    metadata_path = Path(db_dir) / "metadata.json"
+    logger.info(f"Requested metadata.json at: {metadata_path}")
+    if not metadata_path.exists():
+        logger.error("metadata.json not found.")
+        raise HTTPException(status_code=404, detail="metadata.json not found")
+    logger.info(f"Serving metadata.json from: {metadata_path}")
+    return FileResponse(metadata_path)
+
+@app.middleware("http")
+async def log_request(request, call_next):
+    logger.info(f"Incoming request: {request.method} {request.url}")
+    response = await call_next(request)
+    return response
+
+class DBService(cli.Application):
+    """
+    DBService serves:
+    1. CodeQL database .zip files symlinked in the --codeql-db-dir
+    2. Metadata for those zip files, contained in metadata.json in the same
+       directory.
+    The HTTP endpoints are:
+    1. /db/{filename}
+    2. /index
+    3. /api/v1/latest_results/codeql-all
+    """
+
+    codeql_db_dir = cli.SwitchAttr("--codeql-db-dir", str, mandatory=True,
+                                   help="Directory containing CodeQL database files")
+    host = cli.SwitchAttr("--host", str, default="127.0.0.1",
+                          help="Host address for the HTTP server")
+    port = cli.SwitchAttr("--port", int, default=8080, help="Port for the HTTP server")
+
+    def main(self):
+        global db_dir
+        db_dir = Path(self.codeql_db_dir)
+        if not db_dir.is_dir():
+            logger.error(f"Invalid directory: {db_dir}")
+            return 1
+
+        logger.info(f"Starting server at {self.host}:{self.port}")
+        logger.info(f"Serving files from directory: {db_dir}")
+
+        # Run the FastAPI server using Uvicorn
+        uvicorn.run(app, host=self.host, port=self.port)
+
+
+if __name__ == "__main__":
+    DBService.run()

client/qldbtools/requirements.txt

@@ -1,3 +1,4 @@
+annotated-types==0.7.0
 anyio==4.4.0
 appnope==0.1.4
 argon2-cffi==23.1.0
@@ -9,15 +10,19 @@ attrs==24.2.0
 babel==2.16.0
 beautifulsoup4==4.12.3
 bleach==6.1.0
+blinker==1.9.0
 certifi==2024.7.4
 cffi==1.17.0
 charset-normalizer==3.3.2
+click==8.1.7
 comm==0.2.2
 debugpy==1.8.5
 decorator==5.1.1
 defusedxml==0.7.1
 executing==2.0.1
+fastapi==0.115.5
 fastjsonschema==2.20.0
+Flask==3.1.0
 fqdn==1.5.1
 h11==0.14.0
 httpcore==1.0.5
@@ -26,6 +31,7 @@ idna==3.7
 ipykernel==6.29.5
 ipython==8.26.0
 isoduration==20.11.0
+itsdangerous==2.2.0
 jedi==0.19.1
 Jinja2==3.1.4
 json5==0.9.25
@@ -66,6 +72,8 @@ ptyprocess==0.7.0
 pure_eval==0.2.3
 pycparser==2.22
 pycryptodome==3.20.0
+pydantic==2.10.2
+pydantic_core==2.27.1
 Pygments==2.18.0
 python-dateutil==2.9.0.post0
 python-json-logger==2.0.7
@@ -78,10 +86,12 @@ rfc3339-validator==0.1.4
 rfc3986-validator==0.1.1
 rpds-py==0.20.0
 Send2Trash==1.8.3
+setuptools==75.5.0
 six==1.16.0
 sniffio==1.3.1
 soupsieve==2.6
 stack-data==0.6.3
+starlette==0.41.3
 terminado==0.18.1
 tinycss2==1.3.0
 tornado==6.4.1
@@ -91,7 +101,9 @@ typing_extensions==4.12.2
 tzdata==2024.1
 uri-template==1.3.0
 urllib3==2.2.2
+uvicorn==0.32.1
 wcwidth==0.2.13
 webcolors==24.8.0
 webencodings==0.5.1
 websocket-client==1.8.0
+Werkzeug==3.1.3