#!/usr/bin/env python """ Read a table of CodeQL DB information (like those produced by mc-db-refine-info) and push the databases it lists to the mrvacommander minio DB. """ # /// script # dependencies = [ # "pandas", # "numpy", # "minio", # ] # /// import argparse import qldbtools.utils as utils import logging import pandas as pd import numpy as np import sys from minio import Minio from minio.error import S3Error from pathlib import Path # #* Configure logger # logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') # Overwrite log level set by minio root_logger = logging.getLogger() root_logger.setLevel(logging.INFO) # #* Process command line # parser = argparse.ArgumentParser( description=""" Read a table of CodeQL DB information (like those produced by mc-db-refine-info) and push the databases it lists to the mrvacommander minio DB. """) parser.add_argument('-n', '--num-entries', type=int, help='Only use N entries', default=None) parser.add_argument('-s', '--seed', type=int, help='Random number seed', default=4242) args = parser.parse_args() # #* Collect the information and select subset # df = pd.read_csv(sys.stdin) if args.num_entries == None: # Use all entries entries = df else: # Use num_entries, chosen via pseudo-random numbers entries = df.sample(n=args.num_entries, random_state=np.random.RandomState(args.seed)) # #* Push the DBs # # Configuration MINIO_URL = "http://localhost:9000" MINIO_ROOT_USER = "user" MINIO_ROOT_PASSWORD = "mmusty8432" QL_DB_BUCKET_NAME = "qldb" # Initialize MinIO client client = Minio( MINIO_URL.replace("http://", "").replace("https://", ""), access_key=MINIO_ROOT_USER, secret_key=MINIO_ROOT_PASSWORD, secure=False ) # Create the bucket if it doesn't exist try: if not client.bucket_exists(QL_DB_BUCKET_NAME): client.make_bucket(QL_DB_BUCKET_NAME) else: logging.info(f"Bucket '{QL_DB_BUCKET_NAME}' already exists.") except S3Error as err: logging.error(f"Error creating bucket: {err}") # Get info from dataframe and push the files for index, row in entries[['owner', 'name', 'CID', 'path']].iterrows(): owner, name, CID, path = row new_name = utils.form_db_bucket_name(owner, name, CID) try: client.fput_object(QL_DB_BUCKET_NAME, new_name, path) logging.info(f"Uploaded {path} as {new_name} to bucket {QL_DB_BUCKET_NAME}") except S3Error as err: logging.error(f"Error uploading file {local_path}: {err}") # Local Variables: # python-shell-virtualenv-root: "~/work-gh/mrva/mrvacommander/client/qldbtools/venv/" # End: