1e1daf9330
The current api (<2024-07-26 Fri>) is set up only for (owner,name). This is
insufficient for distinguishing CodeQL databases.
Other differences must be considered; this patch combines the fields
| cliVersion |
| creationTime |
| language |
| sha |
into one called CID. The CID field is a hash of these others and therefore can be
changed in the future without affecting workflows or the server.
The cid is combined with the owner/name to form one
identifier. This requires no changes to server or client -- the db
selection's interface is separate from VS Code and gh-mrva in any case.
To test this, this version imports multiple versions of the same owner/repo pairs from multiple directories. In this case, from
~/work-gh/mrva/mrva-open-source-download/repos
and
~/work-gh/mrva/mrva-open-source-download/repos-2024-04-29/
The unique database count increases from 3000 to 5360 -- see README.md,
./bin/mc-db-view-info < db-info-3.csv &
Other code modifications:
- Push (owner,repo,cid) names to minio
- Generate databases.json for use in vs code extension
- Generate list-databases.json for use by gh-mrva client
88 lines
2.6 KiB
Python
Executable File
88 lines
2.6 KiB
Python
Executable File
#!/usr/bin/env python
|
|
""" Read a table of CodeQL DB information (like those produced by
|
|
mc-db-refine-info) and push the databases it lists to the mrvacommander minio
|
|
DB.
|
|
"""
|
|
import argparse
|
|
import qldbtools.utils as utils
|
|
import logging
|
|
import pandas as pd
|
|
import numpy as np
|
|
import sys
|
|
from minio import Minio
|
|
from minio.error import S3Error
|
|
from pathlib import Path
|
|
#
|
|
#* Configure logger
|
|
#
|
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
|
|
# Overwrite log level set by minio
|
|
root_logger = logging.getLogger()
|
|
root_logger.setLevel(logging.INFO)
|
|
|
|
#
|
|
#* Process command line
|
|
#
|
|
parser = argparse.ArgumentParser(
|
|
description=""" Read a table of CodeQL DB information (like those produced by
|
|
mc-db-refine-info) and push the databases it lists to the mrvacommander minio
|
|
DB. """)
|
|
parser.add_argument('-n', '--num-entries', type=int,
|
|
help='Only use N entries',
|
|
default=None)
|
|
parser.add_argument('-s', '--seed', type=int,
|
|
help='Random number seed',
|
|
default=4242)
|
|
args = parser.parse_args()
|
|
#
|
|
#* Collect the information and select subset
|
|
#
|
|
df = pd.read_csv(sys.stdin)
|
|
if args.num_entries == None:
|
|
# Use all entries
|
|
entries = df
|
|
else:
|
|
# Use num_entries, chosen via pseudo-random numbers
|
|
entries = df.sample(n=args.num_entries,
|
|
random_state=np.random.RandomState(args.seed))
|
|
#
|
|
#* Push the DBs
|
|
#
|
|
# Configuration
|
|
MINIO_URL = "http://localhost:9000"
|
|
MINIO_ROOT_USER = "user"
|
|
MINIO_ROOT_PASSWORD = "mmusty8432"
|
|
QL_DB_BUCKET_NAME = "qldb"
|
|
|
|
# Initialize MinIO client
|
|
client = Minio(
|
|
MINIO_URL.replace("http://", "").replace("https://", ""),
|
|
access_key=MINIO_ROOT_USER,
|
|
secret_key=MINIO_ROOT_PASSWORD,
|
|
secure=False
|
|
)
|
|
|
|
# Create the bucket if it doesn't exist
|
|
try:
|
|
if not client.bucket_exists(QL_DB_BUCKET_NAME):
|
|
client.make_bucket(QL_DB_BUCKET_NAME)
|
|
else:
|
|
logging.info(f"Bucket '{QL_DB_BUCKET_NAME}' already exists.")
|
|
except S3Error as err:
|
|
logging.error(f"Error creating bucket: {err}")
|
|
|
|
# Get info from dataframe and push the files
|
|
# XX: include CID.
|
|
for index, row in entries[['owner', 'name', 'CID', 'path']].iterrows():
|
|
owner, name, CID, path = row
|
|
new_name = utils.form_db_bucket_name(owner, name, CID)
|
|
try:
|
|
client.fput_object(QL_DB_BUCKET_NAME, new_name, path)
|
|
logging.info(f"Uploaded {path} as {new_name} to bucket {QL_DB_BUCKET_NAME}")
|
|
except S3Error as err:
|
|
logging.error(f"Error uploading file {local_path}: {err}")
|
|
|
|
# Local Variables:
|
|
# python-shell-virtualenv-root: "~/work-gh/mrva/mrvacommander/client/qldbtools/venv/"
|
|
# End:
|