mrva/mrvacommander
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip: Move all references to github.com/hohn/ghes-mirva-server

Browse Source
This commit is contained in:
Michael Hohn
2024-05-22 14:39:12 -07:00
committed by =Michael Hohn
parent 4269bacf2a
commit 2ab596bf1d
7 changed files with 354 additions and 296 deletions
Download Patch File Download Diff File Expand all files Collapse all files

227
pkg/common/extapi.go Normal file
View File

@@ -0,0 +1,227 @@
package common
type JobInfo struct {
QueryLanguage string
CreatedAt string
UpdatedAt string
SkippedRepositories SkippedRepositories
}
type SkippedRepositories struct {
AccessMismatchRepos AccessMismatchRepos `json:"access_mismatch_repos"`
NotFoundRepos NotFoundRepos `json:"not_found_repos"`
NoCodeqlDBRepos NoCodeqlDBRepos `json:"no_codeql_db_repos"`
OverLimitRepos OverLimitRepos `json:"over_limit_repos"`
}
type AccessMismatchRepos struct {
RepositoryCount int `json:"repository_count"`
Repositories []string `json:"repositories"`
}
type NotFoundRepos struct {
RepositoryCount int `json:"repository_count"`
RepositoryFullNames []string `json:"repository_full_names"`
}
type NoCodeqlDBRepos struct {
RepositoryCount int `json:"repository_count"`
Repositories []string `json:"repositories"`
}
type OverLimitRepos struct {
RepositoryCount int `json:"repository_count"`
Repositories []string `json:"repositories"`
}
type StatusResponse struct {
SessionId int `json:"id"`
ControllerRepo ControllerRepo `json:"controller_repo"`
Actor Actor `json:"actor"`
QueryLanguage string `json:"query_language"`
QueryPackURL string `json:"query_pack_url"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
ActionsWorkflowRunID int `json:"actions_workflow_run_id"`
Status string `json:"status"`
ScannedRepositories []ScannedRepo `json:"scanned_repositories"`
SkippedRepositories SkippedRepositories `json:"skipped_repositories"`
}
type ControllerRepo struct {
ID int `json:"id"`
NodeID string `json:"node_id"`
Name string `json:"name"`
FullName string `json:"full_name"`
Private bool `json:"private"`
Owner struct{} `json:"owner"`
HTMLURL string `json:"html_url"`
Description string `json:"description"`
Fork bool `json:"fork"`
ForksURL string `json:"forks_url"`
KeysURL string `json:"keys_url"`
CollaboratorsURL string `json:"collaborators_url"`
TeamsURL string `json:"teams_url"`
HooksURL string `json:"hooks_url"`
IssueEventsURL string `json:"issue_events_url"`
EventsURL string `json:"events_url"`
AssigneesURL string `json:"assignees_url"`
BranchesURL string `json:"branches_url"`
TagsURL string `json:"tags_url"`
BlobsURL string `json:"blobs_url"`
GitTagsURL string `json:"git_tags_url"`
GitRefsURL string `json:"git_refs_url"`
TreesURL string `json:"trees_url"`
StatusesURL string `json:"statuses_url"`
LanguagesURL string `json:"languages_url"`
StargazersURL string `json:"stargazers_url"`
ContributorsURL string `json:"contributors_url"`
SubscribersURL string `json:"subscribers_url"`
SubscriptionURL string `json:"subscription_url"`
CommitsURL string `json:"commits_url"`
GitCommitsURL string `json:"git_commits_url"`
CommentsURL string `json:"comments_url"`
IssueCommentURL string `json:"issue_comment_url"`
ContentsURL string `json:"contents_url"`
CompareURL string `json:"compare_url"`
MergesURL string `json:"merges_url"`
ArchiveURL string `json:"archive_url"`
DownloadsURL string `json:"downloads_url"`
IssuesURL string `json:"issues_url"`
PullsURL string `json:"pulls_url"`
MilestonesURL string `json:"milestones_url"`
NotificationsURL string `json:"notifications_url"`
LabelsURL string `json:"labels_url"`
ReleasesURL string `json:"releases_url"`
DeploymentsURL string `json:"deployments_url"`
}
type ScannedRepo struct {
Repository Repository `json:"repository"`
AnalysisStatus string `json:"analysis_status"`
ResultCount int `json:"result_count"`
ArtifactSizeBytes int `json:"artifact_size_in_bytes"`
}
type Repository struct {
ID int `json:"id"`
Name string `json:"name"`
FullName string `json:"full_name"`
Private bool `json:"private"`
StargazersCount int `json:"stargazers_count"`
UpdatedAt string `json:"updated_at"`
}
type DownloadResponse struct {
Repository DownloadRepo `json:"repository"`
AnalysisStatus string `json:"analysis_status"`
ResultCount int `json:"result_count"`
ArtifactSizeBytes int `json:"artifact_size_in_bytes"`
DatabaseCommitSha string `json:"database_commit_sha"`
SourceLocationPrefix string `json:"source_location_prefix"`
ArtifactURL string `json:"artifact_url"`
}
type DownloadRepo struct {
ID int `json:"id"`
NodeID string `json:"node_id"`
Name string `json:"name"`
FullName string `json:"full_name"`
Private bool `json:"private"`
Owner Actor `json:"owner"`
HTMLURL string `json:"html_url"`
Description string `json:"description"`
Fork bool `json:"fork"`
ForksURL string `json:"forks_url"`
KeysURL string `json:"keys_url"`
CollaboratorsURL string `json:"collaborators_url"`
TeamsURL string `json:"teams_url"`
HooksURL string `json:"hooks_url"`
IssueEventsURL string `json:"issue_events_url"`
EventsURL string `json:"events_url"`
AssigneesURL string `json:"assignees_url"`
BranchesURL string `json:"branches_url"`
TagsURL string `json:"tags_url"`
BlobsURL string `json:"blobs_url"`
GitTagsURL string `json:"git_tags_url"`
GitRefsURL string `json:"git_refs_url"`
TreesURL string `json:"trees_url"`
StatusesURL string `json:"statuses_url"`
LanguagesURL string `json:"languages_url"`
StargazersURL string `json:"stargazers_url"`
ContributorsURL string `json:"contributors_url"`
SubscribersURL string `json:"subscribers_url"`
SubscriptionURL string `json:"subscription_url"`
CommitsURL string `json:"commits_url"`
GitCommitsURL string `json:"git_commits_url"`
CommentsURL string `json:"comments_url"`
IssueCommentURL string `json:"issue_comment_url"`
ContentsURL string `json:"contents_url"`
CompareURL string `json:"compare_url"`
MergesURL string `json:"merges_url"`
ArchiveURL string `json:"archive_url"`
DownloadsURL string `json:"downloads_url"`
IssuesURL string `json:"issues_url"`
PullsURL string `json:"pulls_url"`
MilestonesURL string `json:"milestones_url"`
NotificationsURL string `json:"notifications_url"`
LabelsURL string `json:"labels_url"`
ReleasesURL string `json:"releases_url"`
DeploymentsURL string `json:"deployments_url"`
}
type Actor struct {
Login string `json:"login"`
ID int `json:"id"`
NodeID string `json:"node_id"`
AvatarURL string `json:"avatar_url"`
GravatarID string `json:"gravatar_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
FollowersURL string `json:"followers_url"`
FollowingURL string `json:"following_url"`
GistsURL string `json:"gists_url"`
StarredURL string `json:"starred_url"`
SubscriptionsURL string `json:"subscriptions_url"`
OrganizationsURL string `json:"organizations_url"`
ReposURL string `json:"repos_url"`
EventsURL string `json:"events_url"`
ReceivedEventsURL string `json:"received_events_url"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
}
type ignored_repos struct {
RepositoryCount int `json:"repository_count"`
Repositories []string `json:"repositories"`
}
type SubmitResponse struct {
ID int `json:"id"`
ControllerRepo ControllerRepo `json:"controller_repo"`
Actor Actor `json:"actor"`
QueryLanguage string `json:"query_language"`
QueryPackURL string `json:"query_pack_url"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
Status string `json:"status"`
SkippedRepositories SkippedRepositories `json:"skipped_repositories"`
}
type SubmitMsg struct {
ActionRepoRef string `json:"action_repo_ref"`
Language string `json:"language"`
QueryPack string `json:"query_pack"`
Repositories []string `json:"repositories"`
}

52
pkg/common/types.go Normal file
View File

@@ -0,0 +1,52 @@
package common
type AnalyzeJob struct {
MirvaRequestID int
QueryPackId int
QueryLanguage string
ORL OwnerRepo
}
type OwnerRepo struct {
Owner string
Repo string
}
type AnalyzeResult struct {
RunAnalysisSARIF string
RunAnalysisBQRS string
}
type Status int
const (
StatusInProgress = iota
StatusQueued
StatusError
StatusSuccess
StatusFailed
)
func (s Status) ToExternalString() string {
switch s {
case StatusInProgress:
return "in_progress"
case StatusQueued:
return "queued"
case StatusError:
return "error"
case StatusSuccess:
return "succeeded"
case StatusFailed:
return "failed"
default:
return "unknown"
}
}
type JobSpec struct {
ID int
OwnerRepo
}

13
pkg/queue/queue.go
View File

@@ -2,30 +2,29 @@ package queue
import (
"log/slog"
"mrvacommander/pkg/common"
"mrvacommander/pkg/storage"
co "github.com/hohn/ghes-mirva-server/common"
)
var (
NumWorkers int
Jobs chan co.AnalyzeJob
Results chan co.AnalyzeResult
Jobs chan common.AnalyzeJob
Results chan common.AnalyzeResult
)
func StartAnalyses(analysis_repos *map[co.OwnerRepo]storage.DBLocation, session_id int,
func StartAnalyses(analysis_repos *map[common.OwnerRepo]storage.DBLocation, session_id int,
session_language string) {
slog.Debug("Queueing codeql database analyze jobs")
for orl := range *analysis_repos {
info := co.AnalyzeJob{
info := common.AnalyzeJob{
QueryPackId: session_id,
QueryLanguage: session_language,
ORL: orl,
}
Jobs <- info
storage.SetStatus(session_id, orl, co.StatusQueued)
storage.SetStatus(session_id, orl, common.StatusQueued)
storage.AddJob(session_id, info)
}
}

79
pkg/server/server.go
View File

@@ -15,12 +15,11 @@ import (
"strings"
"time"
"mrvacommander/pkg/common"
"mrvacommander/pkg/queue"
"mrvacommander/pkg/storage"
"github.com/gorilla/mux"
"github.com/hohn/ghes-mirva-server/api"
co "github.com/hohn/ghes-mirva-server/common"
)
func (c *CommanderSingle) Run() {
@@ -62,16 +61,16 @@ func (c *CommanderSingle) Setup(st *State) {
log.Fatal(http.ListenAndServe(":8080", r))
}
func (c *CommanderSingle) StatusResponse(w http.ResponseWriter, js co.JobSpec, ji co.JobInfo, vaid int) {
func (c *CommanderSingle) StatusResponse(w http.ResponseWriter, js common.JobSpec, ji common.JobInfo, vaid int) {
slog.Debug("Submitting status response", "session", vaid)
all_scanned := []api.ScannedRepo{}
all_scanned := []common.ScannedRepo{}
jobs := storage.GetJobList(js.ID)
for _, job := range jobs {
astat := storage.GetStatus(js.ID, job.ORL).ToExternalString()
all_scanned = append(all_scanned,
api.ScannedRepo{
Repository: api.Repository{
common.ScannedRepo{
Repository: common.Repository{
ID: 0,
Name: job.ORL.Repo,
FullName: fmt.Sprintf("%s/%s", job.ORL.Owner, job.ORL.Repo),
@@ -88,10 +87,10 @@ func (c *CommanderSingle) StatusResponse(w http.ResponseWriter, js co.JobSpec, j
astat := storage.GetStatus(js.ID, js.OwnerRepo).ToExternalString()
status := api.StatusResponse{
status := common.StatusResponse{
SessionId: js.ID,
ControllerRepo: api.ControllerRepo{},
Actor: api.Actor{},
ControllerRepo: common.ControllerRepo{},
Actor: common.Actor{},
QueryLanguage: ji.QueryLanguage,
QueryPackURL: "", // FIXME
CreatedAt: ji.CreatedAt,
@@ -145,7 +144,7 @@ func (c *CommanderSingle) MirvaStatus(w http.ResponseWriter, r *http.Request) {
job := spec[0]
js := co.JobSpec{
js := common.JobSpec{
ID: job.QueryPackId,
OwnerRepo: job.ORL,
}
@@ -172,9 +171,9 @@ func (c *CommanderSingle) MirvaDownloadArtifact(w http.ResponseWriter, r *http.R
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
js := co.JobSpec{
js := common.JobSpec{
ID: vaid,
OwnerRepo: co.OwnerRepo{
OwnerRepo: common.OwnerRepo{
Owner: vars["repo_owner"],
Repo: vars["repo_name"],
},
@@ -182,13 +181,13 @@ func (c *CommanderSingle) MirvaDownloadArtifact(w http.ResponseWriter, r *http.R
c.DownloadResponse(w, js, vaid)
}
func (c *CommanderSingle) DownloadResponse(w http.ResponseWriter, js co.JobSpec, vaid int) {
func (c *CommanderSingle) DownloadResponse(w http.ResponseWriter, js common.JobSpec, vaid int) {
slog.Debug("Forming download response", "session", vaid, "job", js)
astat := storage.GetStatus(vaid, js.OwnerRepo)
var dlr api.DownloadResponse
if astat == co.StatusSuccess {
var dlr common.DownloadResponse
if astat == common.StatusSuccess {
au, err := storage.ArtifactURL(js, vaid)
if err != nil {
@@ -196,8 +195,8 @@ func (c *CommanderSingle) DownloadResponse(w http.ResponseWriter, js co.JobSpec,
return
}
dlr = api.DownloadResponse{
Repository: api.DownloadRepo{
dlr = common.DownloadResponse{
Repository: common.DownloadRepo{
Name: js.Repo,
FullName: fmt.Sprintf("%s/%s", js.Owner, js.Repo),
},
@@ -209,8 +208,8 @@ func (c *CommanderSingle) DownloadResponse(w http.ResponseWriter, js co.JobSpec,
ArtifactURL: au,
}
} else {
dlr = api.DownloadResponse{
Repository: api.DownloadRepo{
dlr = common.DownloadResponse{
Repository: common.DownloadRepo{
Name: js.Repo,
FullName: fmt.Sprintf("%s/%s", js.Owner, js.Repo),
},
@@ -320,7 +319,7 @@ func (c *CommanderSingle) MirvaRequest(w http.ResponseWriter, r *http.Request) {
w.Write(submit_response)
}
func ORToArr(aor []co.OwnerRepo) ([]string, int) {
func ORToArr(aor []common.OwnerRepo) ([]string, int) {
repos := []string{}
count := len(aor)
for _, repo := range aor {
@@ -331,29 +330,29 @@ func ORToArr(aor []co.OwnerRepo) ([]string, int) {
func submit_response(sn SessionInfo) ([]byte, error) {
// Construct the response bottom-up
var m_cr api.ControllerRepo
var m_ac api.Actor
var m_cr common.ControllerRepo
var m_ac common.Actor
repos, count := ORToArr(sn.NotFoundRepos)
r_nfr := api.NotFoundRepos{RepositoryCount: count, RepositoryFullNames: repos}
r_nfr := common.NotFoundRepos{RepositoryCount: count, RepositoryFullNames: repos}
repos, count = ORToArr(sn.AccessMismatchRepos)
r_amr := api.AccessMismatchRepos{RepositoryCount: count, Repositories: repos}
r_amr := common.AccessMismatchRepos{RepositoryCount: count, Repositories: repos}
repos, count = ORToArr(sn.NoCodeqlDBRepos)
r_ncd := api.NoCodeqlDBRepos{RepositoryCount: count, Repositories: repos}
r_ncd := common.NoCodeqlDBRepos{RepositoryCount: count, Repositories: repos}
// TODO fill these with real values?
repos, count = ORToArr(sn.NoCodeqlDBRepos)
r_olr := api.OverLimitRepos{RepositoryCount: count, Repositories: repos}
r_olr := common.OverLimitRepos{RepositoryCount: count, Repositories: repos}
m_skip := api.SkippedRepositories{
m_skip := common.SkippedRepositories{
AccessMismatchRepos: r_amr,
NotFoundRepos: r_nfr,
NoCodeqlDBRepos: r_ncd,
OverLimitRepos: r_olr}
m_sr := api.SubmitResponse{
m_sr := common.SubmitResponse{
Actor: m_ac,
ControllerRepo: m_cr,
ID: sn.ID,
@@ -369,10 +368,10 @@ func submit_response(sn SessionInfo) ([]byte, error) {
joblist := storage.GetJobList(sn.ID)
for _, job := range joblist {
storage.SetJobInfo(co.JobSpec{
storage.SetJobInfo(common.JobSpec{
ID: sn.ID,
OwnerRepo: job.ORL,
}, co.JobInfo{
}, common.JobInfo{
QueryLanguage: sn.Language,
CreatedAt: m_sr.CreatedAt,
UpdatedAt: m_sr.UpdatedAt,
@@ -391,28 +390,28 @@ func submit_response(sn SessionInfo) ([]byte, error) {
}
func (c *CommanderSingle) collectRequestInfo(w http.ResponseWriter, r *http.Request, sessionId int) (string, []co.OwnerRepo, string, error) {
func (c *CommanderSingle) collectRequestInfo(w http.ResponseWriter, r *http.Request, sessionId int) (string, []common.OwnerRepo, string, error) {
slog.Debug("Collecting session info")
if r.Body == nil {
err := errors.New("missing request body")
log.Println(err)
http.Error(w, err.Error(), http.StatusNoContent)
return "", []co.OwnerRepo{}, "", err
return "", []common.OwnerRepo{}, "", err
}
buf, err := io.ReadAll(r.Body)
if err != nil {
var w http.ResponseWriter
slog.Error("Error reading MRVA submission body", "error", err.Error())
http.Error(w, err.Error(), http.StatusBadRequest)
return "", []co.OwnerRepo{}, "", err
return "", []common.OwnerRepo{}, "", err
}
msg, err := TrySubmitMsg(buf)
if err != nil {
// Unknown message
slog.Error("Unknown MRVA submission body format")
http.Error(w, err.Error(), http.StatusBadRequest)
return "", []co.OwnerRepo{}, "", err
return "", []common.OwnerRepo{}, "", err
}
// Decompose the SubmitMsg and keep information
@@ -421,19 +420,19 @@ func (c *CommanderSingle) collectRequestInfo(w http.ResponseWriter, r *http.Requ
slog.Error("MRVA submission body querypack has invalid format")
err := errors.New("MRVA submission body querypack has invalid format")
http.Error(w, err.Error(), http.StatusBadRequest)
return "", []co.OwnerRepo{}, "", err
return "", []common.OwnerRepo{}, "", err
}
session_tgz_ref, err := c.extract_tgz(msg.QueryPack, sessionId)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return "", []co.OwnerRepo{}, "", err
return "", []common.OwnerRepo{}, "", err
}
// 2. Save the language
session_language := msg.Language
// 3. Save the repositories
var session_repositories []co.OwnerRepo
var session_repositories []common.OwnerRepo
for _, v := range msg.Repositories {
t := strings.Split(v, "/")
@@ -443,18 +442,18 @@ func (c *CommanderSingle) collectRequestInfo(w http.ResponseWriter, r *http.Requ
http.Error(w, err, http.StatusBadRequest)
}
session_repositories = append(session_repositories,
co.OwnerRepo{Owner: t[0], Repo: t[1]})
common.OwnerRepo{Owner: t[0], Repo: t[1]})
}
return session_language, session_repositories, session_tgz_ref, nil
}
// Try to extract a SubmitMsg from a json-encoded buffer
func TrySubmitMsg(buf []byte) (SubmitMsg, error) {
func TrySubmitMsg(buf []byte) (common.SubmitMsg, error) {
buf1 := make([]byte, len(buf))
copy(buf1, buf)
dec := json.NewDecoder(bytes.NewReader(buf1))
dec.DisallowUnknownFields()
var m SubmitMsg
var m common.SubmitMsg
err := dec.Decode(&m)
return m, err
}

233
pkg/server/types.go
View File

@@ -2,231 +2,12 @@ package server
import (
"mrvacommander/pkg/agent"
"mrvacommander/pkg/common"
"mrvacommander/pkg/logger"
"mrvacommander/pkg/queue"
"mrvacommander/pkg/storage"
co "github.com/hohn/ghes-mirva-server/common"
)
type DownloadResponse struct {
Repository DownloadRepo `json:"repository"`
AnalysisStatus string `json:"analysis_status"`
ResultCount int `json:"result_count"`
ArtifactSizeBytes int `json:"artifact_size_in_bytes"`
DatabaseCommitSha string `json:"database_commit_sha"`
SourceLocationPrefix string `json:"source_location_prefix"`
ArtifactURL string `json:"artifact_url"`
}
type DownloadRepo struct {
ID int `json:"id"`
NodeID string `json:"node_id"`
Name string `json:"name"`
FullName string `json:"full_name"`
Private bool `json:"private"`
Owner Actor `json:"owner"`
HTMLURL string `json:"html_url"`
Description string `json:"description"`
Fork bool `json:"fork"`
ForksURL string `json:"forks_url"`
KeysURL string `json:"keys_url"`
CollaboratorsURL string `json:"collaborators_url"`
TeamsURL string `json:"teams_url"`
HooksURL string `json:"hooks_url"`
IssueEventsURL string `json:"issue_events_url"`
EventsURL string `json:"events_url"`
AssigneesURL string `json:"assignees_url"`
BranchesURL string `json:"branches_url"`
TagsURL string `json:"tags_url"`
BlobsURL string `json:"blobs_url"`
GitTagsURL string `json:"git_tags_url"`
GitRefsURL string `json:"git_refs_url"`
TreesURL string `json:"trees_url"`
StatusesURL string `json:"statuses_url"`
LanguagesURL string `json:"languages_url"`
StargazersURL string `json:"stargazers_url"`
ContributorsURL string `json:"contributors_url"`
SubscribersURL string `json:"subscribers_url"`
SubscriptionURL string `json:"subscription_url"`
CommitsURL string `json:"commits_url"`
GitCommitsURL string `json:"git_commits_url"`
CommentsURL string `json:"comments_url"`
IssueCommentURL string `json:"issue_comment_url"`
ContentsURL string `json:"contents_url"`
CompareURL string `json:"compare_url"`
MergesURL string `json:"merges_url"`
ArchiveURL string `json:"archive_url"`
DownloadsURL string `json:"downloads_url"`
IssuesURL string `json:"issues_url"`
PullsURL string `json:"pulls_url"`
MilestonesURL string `json:"milestones_url"`
NotificationsURL string `json:"notifications_url"`
LabelsURL string `json:"labels_url"`
ReleasesURL string `json:"releases_url"`
DeploymentsURL string `json:"deployments_url"`
}
type ControllerRepo struct {
ID int `json:"id"`
NodeID string `json:"node_id"`
Name string `json:"name"`
FullName string `json:"full_name"`
Private bool `json:"private"`
Owner struct{} `json:"owner"`
HTMLURL string `json:"html_url"`
Description string `json:"description"`
Fork bool `json:"fork"`
ForksURL string `json:"forks_url"`
KeysURL string `json:"keys_url"`
CollaboratorsURL string `json:"collaborators_url"`
TeamsURL string `json:"teams_url"`
HooksURL string `json:"hooks_url"`
IssueEventsURL string `json:"issue_events_url"`
EventsURL string `json:"events_url"`
AssigneesURL string `json:"assignees_url"`
BranchesURL string `json:"branches_url"`
TagsURL string `json:"tags_url"`
BlobsURL string `json:"blobs_url"`
GitTagsURL string `json:"git_tags_url"`
GitRefsURL string `json:"git_refs_url"`
TreesURL string `json:"trees_url"`
StatusesURL string `json:"statuses_url"`
LanguagesURL string `json:"languages_url"`
StargazersURL string `json:"stargazers_url"`
ContributorsURL string `json:"contributors_url"`
SubscribersURL string `json:"subscribers_url"`
SubscriptionURL string `json:"subscription_url"`
CommitsURL string `json:"commits_url"`
GitCommitsURL string `json:"git_commits_url"`
CommentsURL string `json:"comments_url"`
IssueCommentURL string `json:"issue_comment_url"`
ContentsURL string `json:"contents_url"`
CompareURL string `json:"compare_url"`
MergesURL string `json:"merges_url"`
ArchiveURL string `json:"archive_url"`
DownloadsURL string `json:"downloads_url"`
IssuesURL string `json:"issues_url"`
PullsURL string `json:"pulls_url"`
MilestonesURL string `json:"milestones_url"`
NotificationsURL string `json:"notifications_url"`
LabelsURL string `json:"labels_url"`
ReleasesURL string `json:"releases_url"`
DeploymentsURL string `json:"deployments_url"`
}
type Actor struct {
Login string `json:"login"`
ID int `json:"id"`
NodeID string `json:"node_id"`
AvatarURL string `json:"avatar_url"`
GravatarID string `json:"gravatar_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
FollowersURL string `json:"followers_url"`
FollowingURL string `json:"following_url"`
GistsURL string `json:"gists_url"`
StarredURL string `json:"starred_url"`
SubscriptionsURL string `json:"subscriptions_url"`
OrganizationsURL string `json:"organizations_url"`
ReposURL string `json:"repos_url"`
EventsURL string `json:"events_url"`
ReceivedEventsURL string `json:"received_events_url"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
}
type SkippedRepositories struct {
AccessMismatchRepos AccessMismatchRepos `json:"access_mismatch_repos"`
NotFoundRepos NotFoundRepos `json:"not_found_repos"`
NoCodeqlDBRepos NoCodeqlDBRepos `json:"no_codeql_db_repos"`
OverLimitRepos OverLimitRepos `json:"over_limit_repos"`
}
type ignored_repos struct {
RepositoryCount int `json:"repository_count"`
Repositories []string `json:"repositories"`
}
type AccessMismatchRepos struct {
RepositoryCount int `json:"repository_count"`
Repositories []string `json:"repositories"`
}
type NotFoundRepos struct {
RepositoryCount int `json:"repository_count"`
RepositoryFullNames []string `json:"repository_full_names"`
}
type NoCodeqlDBRepos struct {
RepositoryCount int `json:"repository_count"`
Repositories []string `json:"repositories"`
}
type OverLimitRepos struct {
RepositoryCount int `json:"repository_count"`
Repositories []string `json:"repositories"`
}
type SubmitResponse struct {
ID int `json:"id"`
ControllerRepo ControllerRepo `json:"controller_repo"`
Actor Actor `json:"actor"`
QueryLanguage string `json:"query_language"`
QueryPackURL string `json:"query_pack_url"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
Status string `json:"status"`
SkippedRepositories SkippedRepositories `json:"skipped_repositories"`
}
type Repository struct {
ID int `json:"id"`
Name string `json:"name"`
FullName string `json:"full_name"`
Private bool `json:"private"`
StargazersCount int `json:"stargazers_count"`
UpdatedAt string `json:"updated_at"`
}
type ScannedRepo struct {
Repository Repository `json:"repository"`
AnalysisStatus string `json:"analysis_status"`
ResultCount int `json:"result_count"`
ArtifactSizeBytes int `json:"artifact_size_in_bytes"`
}
type StatusResponse struct {
SessionId int `json:"id"`
ControllerRepo ControllerRepo `json:"controller_repo"`
Actor Actor `json:"actor"`
QueryLanguage string `json:"query_language"`
QueryPackURL string `json:"query_pack_url"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
ActionsWorkflowRunID int `json:"actions_workflow_run_id"`
Status string `json:"status"`
ScannedRepositories []ScannedRepo `json:"scanned_repositories"`
SkippedRepositories SkippedRepositories `json:"skipped_repositories"`
}
type SubmitMsg struct {
ActionRepoRef string `json:"action_repo_ref"`
Language string `json:"language"`
QueryPack string `json:"query_pack"`
Repositories []string `json:"repositories"`
}
type SessionInfo struct {
ID int
Owner string
@@ -234,14 +15,14 @@ type SessionInfo struct {
QueryPack string
Language string
Repositories []co.OwnerRepo
Repositories []common.OwnerRepo
AccessMismatchRepos []co.OwnerRepo
NotFoundRepos []co.OwnerRepo
NoCodeqlDBRepos []co.OwnerRepo
OverLimitRepos []co.OwnerRepo
AccessMismatchRepos []common.OwnerRepo
NotFoundRepos []common.OwnerRepo
NoCodeqlDBRepos []common.OwnerRepo
OverLimitRepos []common.OwnerRepo
AnalysisRepos *map[co.OwnerRepo]storage.DBLocation
AnalysisRepos *map[common.OwnerRepo]storage.DBLocation
}
type CommanderSingle struct {

40
pkg/storage/storage.go
View File

@@ -12,14 +12,14 @@ import (
"path/filepath"
"sync"
co "github.com/hohn/ghes-mirva-server/common"
"mrvacommander/pkg/common"
)
var (
jobs map[int][]co.AnalyzeJob = make(map[int][]co.AnalyzeJob)
info map[co.JobSpec]co.JobInfo = make(map[co.JobSpec]co.JobInfo)
status map[co.JobSpec]co.Status = make(map[co.JobSpec]co.Status)
result map[co.JobSpec]co.AnalyzeResult = make(map[co.JobSpec]co.AnalyzeResult)
jobs map[int][]common.AnalyzeJob = make(map[int][]common.AnalyzeJob)
info map[common.JobSpec]common.JobInfo = make(map[common.JobSpec]common.JobInfo)
status map[common.JobSpec]common.Status = make(map[common.JobSpec]common.Status)
result map[common.JobSpec]common.AnalyzeResult = make(map[common.JobSpec]common.AnalyzeResult)
mutex sync.Mutex
)
@@ -62,8 +62,8 @@ func (s *StorageSingle) SaveQueryPack(tgz []byte, sessionId int) (string, error)
// Determine for which repositories codeql databases are available.
//
// Those will be the analysis_repos. The rest will be skipped.
func (s *StorageSingle) FindAvailableDBs(analysisReposRequested []co.OwnerRepo) (not_found_repos []co.OwnerRepo,
analysisRepos *map[co.OwnerRepo]DBLocation) {
func (s *StorageSingle) FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (not_found_repos []common.OwnerRepo,
analysisRepos *map[common.OwnerRepo]DBLocation) {
slog.Debug("Looking for available CodeQL databases")
cwd, err := os.Getwd()
@@ -72,9 +72,9 @@ func (s *StorageSingle) FindAvailableDBs(analysisReposRequested []co.OwnerRepo)
return
}
analysisRepos = &map[co.OwnerRepo]DBLocation{}
analysisRepos = &map[common.OwnerRepo]DBLocation{}
not_found_repos = []co.OwnerRepo{}
not_found_repos = []common.OwnerRepo{}
for _, rep := range analysisReposRequested {
dbPrefix := filepath.Join(cwd, "codeql", "dbs", rep.Owner, rep.Repo)
@@ -93,7 +93,7 @@ func (s *StorageSingle) FindAvailableDBs(analysisReposRequested []co.OwnerRepo)
return not_found_repos, analysisRepos
}
func ArtifactURL(js co.JobSpec, vaid int) (string, error) {
func ArtifactURL(js common.JobSpec, vaid int) (string, error) {
// We're looking for paths like
// codeql/sarif/google/flatbuffers/google_flatbuffers.sarif
@@ -114,14 +114,14 @@ func ArtifactURL(js co.JobSpec, vaid int) (string, error) {
return au, nil
}
func GetResult(js co.JobSpec) co.AnalyzeResult {
func GetResult(js common.JobSpec) common.AnalyzeResult {
mutex.Lock()
defer mutex.Unlock()
ar := result[js]
return ar
}
func PackageResults(ar co.AnalyzeResult, owre co.OwnerRepo, vaid int) (zipPath string, e error) {
func PackageResults(ar common.AnalyzeResult, owre common.OwnerRepo, vaid int) (zipPath string, e error) {
slog.Debug("Readying zip file with .sarif/.bqrs", "analyze-result", ar)
cwd, err := os.Getwd()
@@ -178,28 +178,28 @@ func PackageResults(ar co.AnalyzeResult, owre co.OwnerRepo, vaid int) (zipPath s
return zpath, nil
}
func GetJobList(sessionid int) []co.AnalyzeJob {
func GetJobList(sessionid int) []common.AnalyzeJob {
mutex.Lock()
defer mutex.Unlock()
return jobs[sessionid]
}
func GetJobInfo(js co.JobSpec) co.JobInfo {
func GetJobInfo(js common.JobSpec) common.JobInfo {
mutex.Lock()
defer mutex.Unlock()
return info[js]
}
func SetJobInfo(js co.JobSpec, ji co.JobInfo) {
func SetJobInfo(js common.JobSpec, ji common.JobInfo) {
mutex.Lock()
defer mutex.Unlock()
info[js] = ji
}
func GetStatus(sessionid int, orl co.OwnerRepo) co.Status {
func GetStatus(sessionid int, orl common.OwnerRepo) common.Status {
mutex.Lock()
defer mutex.Unlock()
return status[co.JobSpec{ID: sessionid, OwnerRepo: orl}]
return status[common.JobSpec{ID: sessionid, OwnerRepo: orl}]
}
func ResultAsFile(path string) (string, []byte, error) {
@@ -217,13 +217,13 @@ func ResultAsFile(path string) (string, []byte, error) {
return fpath, file, nil
}
func SetStatus(sessionid int, orl co.OwnerRepo, s co.Status) {
func SetStatus(sessionid int, orl common.OwnerRepo, s common.Status) {
mutex.Lock()
defer mutex.Unlock()
status[co.JobSpec{ID: sessionid, OwnerRepo: orl}] = s
status[common.JobSpec{ID: sessionid, OwnerRepo: orl}] = s
}
func AddJob(sessionid int, job co.AnalyzeJob) {
func AddJob(sessionid int, job common.AnalyzeJob) {
mutex.Lock()
defer mutex.Unlock()
jobs[sessionid] = append(jobs[sessionid], job)

6
pkg/storage/types.go
View File

@@ -1,14 +1,14 @@
package storage
import (
co "github.com/hohn/ghes-mirva-server/common"
"mrvacommander/pkg/common"
)
type Storage interface {
NextID() int
SaveQueryPack(tgz []byte, sessionID int) (storagePath string, error error)
FindAvailableDBs(analysisReposRequested []co.OwnerRepo) (not_found_repos []co.OwnerRepo,
analysisRepos *map[co.OwnerRepo]DBLocation)
FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (not_found_repos []common.OwnerRepo,
analysisRepos *map[common.OwnerRepo]DBLocation)
}
type DBLocation struct {