michael hohn
47de30a56e
- Create pyproject.toml with all dependencies from requirements.txt - Configure for Python >=3.11 - Include standalone scripts via script-files - Set README.org content-type for proper packaging - Enable uv sync and uv run workflows
94 lines
2.7 KiB
Python
Executable File
94 lines
2.7 KiB
Python
Executable File
#!/usr/bin/env python
|
|
""" Read a table of CodeQL DB information (like those produced by
|
|
mc-db-refine-info) and push the databases it lists to the mrvacommander minio
|
|
DB.
|
|
"""
|
|
# /// script
|
|
# dependencies = [
|
|
# "pandas",
|
|
# "numpy",
|
|
# "minio",
|
|
# ]
|
|
# ///
|
|
import argparse
|
|
import qldbtools.utils as utils
|
|
import logging
|
|
import pandas as pd
|
|
import numpy as np
|
|
import sys
|
|
from minio import Minio
|
|
from minio.error import S3Error
|
|
from pathlib import Path
|
|
#
|
|
#* Configure logger
|
|
#
|
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
|
|
# Overwrite log level set by minio
|
|
root_logger = logging.getLogger()
|
|
root_logger.setLevel(logging.INFO)
|
|
|
|
#
|
|
#* Process command line
|
|
#
|
|
parser = argparse.ArgumentParser(
|
|
description=""" Read a table of CodeQL DB information (like those produced by
|
|
mc-db-refine-info) and push the databases it lists to the mrvacommander minio
|
|
DB. """)
|
|
parser.add_argument('-n', '--num-entries', type=int,
|
|
help='Only use N entries',
|
|
default=None)
|
|
parser.add_argument('-s', '--seed', type=int,
|
|
help='Random number seed',
|
|
default=4242)
|
|
args = parser.parse_args()
|
|
#
|
|
#* Collect the information and select subset
|
|
#
|
|
df = pd.read_csv(sys.stdin)
|
|
if args.num_entries == None:
|
|
# Use all entries
|
|
entries = df
|
|
else:
|
|
# Use num_entries, chosen via pseudo-random numbers
|
|
entries = df.sample(n=args.num_entries,
|
|
random_state=np.random.RandomState(args.seed))
|
|
#
|
|
#* Push the DBs
|
|
#
|
|
# Configuration
|
|
MINIO_URL = "http://localhost:9000"
|
|
MINIO_ROOT_USER = "user"
|
|
MINIO_ROOT_PASSWORD = "mmusty8432"
|
|
QL_DB_BUCKET_NAME = "qldb"
|
|
|
|
# Initialize MinIO client
|
|
client = Minio(
|
|
MINIO_URL.replace("http://", "").replace("https://", ""),
|
|
access_key=MINIO_ROOT_USER,
|
|
secret_key=MINIO_ROOT_PASSWORD,
|
|
secure=False
|
|
)
|
|
|
|
# Create the bucket if it doesn't exist
|
|
try:
|
|
if not client.bucket_exists(QL_DB_BUCKET_NAME):
|
|
client.make_bucket(QL_DB_BUCKET_NAME)
|
|
else:
|
|
logging.info(f"Bucket '{QL_DB_BUCKET_NAME}' already exists.")
|
|
except S3Error as err:
|
|
logging.error(f"Error creating bucket: {err}")
|
|
|
|
# Get info from dataframe and push the files
|
|
for index, row in entries[['owner', 'name', 'CID', 'path']].iterrows():
|
|
owner, name, CID, path = row
|
|
new_name = utils.form_db_bucket_name(owner, name, CID)
|
|
try:
|
|
client.fput_object(QL_DB_BUCKET_NAME, new_name, path)
|
|
logging.info(f"Uploaded {path} as {new_name} to bucket {QL_DB_BUCKET_NAME}")
|
|
except S3Error as err:
|
|
logging.error(f"Error uploading file {local_path}: {err}")
|
|
|
|
# Local Variables:
|
|
# python-shell-virtualenv-root: "~/work-gh/mrva/mrvacommander/client/qldbtools/venv/"
|
|
# End:
|