diff --git a/pkg/common/extapi.go b/pkg/common/extapi.go new file mode 100644 index 0000000..0190484 --- /dev/null +++ b/pkg/common/extapi.go @@ -0,0 +1,227 @@ +package common + +type JobInfo struct { + QueryLanguage string + CreatedAt string + UpdatedAt string + + SkippedRepositories SkippedRepositories +} + +type SkippedRepositories struct { + AccessMismatchRepos AccessMismatchRepos `json:"access_mismatch_repos"` + NotFoundRepos NotFoundRepos `json:"not_found_repos"` + NoCodeqlDBRepos NoCodeqlDBRepos `json:"no_codeql_db_repos"` + OverLimitRepos OverLimitRepos `json:"over_limit_repos"` +} + +type AccessMismatchRepos struct { + RepositoryCount int `json:"repository_count"` + Repositories []string `json:"repositories"` +} + +type NotFoundRepos struct { + RepositoryCount int `json:"repository_count"` + RepositoryFullNames []string `json:"repository_full_names"` +} + +type NoCodeqlDBRepos struct { + RepositoryCount int `json:"repository_count"` + Repositories []string `json:"repositories"` +} + +type OverLimitRepos struct { + RepositoryCount int `json:"repository_count"` + Repositories []string `json:"repositories"` +} + +type StatusResponse struct { + SessionId int `json:"id"` + ControllerRepo ControllerRepo `json:"controller_repo"` + Actor Actor `json:"actor"` + QueryLanguage string `json:"query_language"` + QueryPackURL string `json:"query_pack_url"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + ActionsWorkflowRunID int `json:"actions_workflow_run_id"` + Status string `json:"status"` + ScannedRepositories []ScannedRepo `json:"scanned_repositories"` + SkippedRepositories SkippedRepositories `json:"skipped_repositories"` +} + +type ControllerRepo struct { + ID int `json:"id"` + NodeID string `json:"node_id"` + Name string `json:"name"` + FullName string `json:"full_name"` + Private bool `json:"private"` + Owner struct{} `json:"owner"` + HTMLURL string `json:"html_url"` + Description string `json:"description"` + Fork bool `json:"fork"` + ForksURL string `json:"forks_url"` + KeysURL string `json:"keys_url"` + CollaboratorsURL string `json:"collaborators_url"` + TeamsURL string `json:"teams_url"` + HooksURL string `json:"hooks_url"` + IssueEventsURL string `json:"issue_events_url"` + EventsURL string `json:"events_url"` + + AssigneesURL string `json:"assignees_url"` + BranchesURL string `json:"branches_url"` + TagsURL string `json:"tags_url"` + BlobsURL string `json:"blobs_url"` + GitTagsURL string `json:"git_tags_url"` + GitRefsURL string `json:"git_refs_url"` + TreesURL string `json:"trees_url"` + StatusesURL string `json:"statuses_url"` + LanguagesURL string `json:"languages_url"` + + StargazersURL string `json:"stargazers_url"` + ContributorsURL string `json:"contributors_url"` + SubscribersURL string `json:"subscribers_url"` + SubscriptionURL string `json:"subscription_url"` + + CommitsURL string `json:"commits_url"` + GitCommitsURL string `json:"git_commits_url"` + CommentsURL string `json:"comments_url"` + IssueCommentURL string `json:"issue_comment_url"` + ContentsURL string `json:"contents_url"` + CompareURL string `json:"compare_url"` + MergesURL string `json:"merges_url"` + ArchiveURL string `json:"archive_url"` + DownloadsURL string `json:"downloads_url"` + IssuesURL string `json:"issues_url"` + PullsURL string `json:"pulls_url"` + MilestonesURL string `json:"milestones_url"` + NotificationsURL string `json:"notifications_url"` + LabelsURL string `json:"labels_url"` + ReleasesURL string `json:"releases_url"` + DeploymentsURL string `json:"deployments_url"` +} + +type ScannedRepo struct { + Repository Repository `json:"repository"` + AnalysisStatus string `json:"analysis_status"` + ResultCount int `json:"result_count"` + ArtifactSizeBytes int `json:"artifact_size_in_bytes"` +} + +type Repository struct { + ID int `json:"id"` + Name string `json:"name"` + FullName string `json:"full_name"` + Private bool `json:"private"` + StargazersCount int `json:"stargazers_count"` + UpdatedAt string `json:"updated_at"` +} + +type DownloadResponse struct { + Repository DownloadRepo `json:"repository"` + AnalysisStatus string `json:"analysis_status"` + ResultCount int `json:"result_count"` + ArtifactSizeBytes int `json:"artifact_size_in_bytes"` + DatabaseCommitSha string `json:"database_commit_sha"` + SourceLocationPrefix string `json:"source_location_prefix"` + ArtifactURL string `json:"artifact_url"` +} + +type DownloadRepo struct { + ID int `json:"id"` + NodeID string `json:"node_id"` + Name string `json:"name"` + FullName string `json:"full_name"` + Private bool `json:"private"` + Owner Actor `json:"owner"` + + HTMLURL string `json:"html_url"` + Description string `json:"description"` + Fork bool `json:"fork"` + ForksURL string `json:"forks_url"` + KeysURL string `json:"keys_url"` + CollaboratorsURL string `json:"collaborators_url"` + TeamsURL string `json:"teams_url"` + HooksURL string `json:"hooks_url"` + IssueEventsURL string `json:"issue_events_url"` + EventsURL string `json:"events_url"` + + AssigneesURL string `json:"assignees_url"` + BranchesURL string `json:"branches_url"` + TagsURL string `json:"tags_url"` + BlobsURL string `json:"blobs_url"` + GitTagsURL string `json:"git_tags_url"` + GitRefsURL string `json:"git_refs_url"` + TreesURL string `json:"trees_url"` + StatusesURL string `json:"statuses_url"` + LanguagesURL string `json:"languages_url"` + + StargazersURL string `json:"stargazers_url"` + ContributorsURL string `json:"contributors_url"` + SubscribersURL string `json:"subscribers_url"` + SubscriptionURL string `json:"subscription_url"` + + CommitsURL string `json:"commits_url"` + GitCommitsURL string `json:"git_commits_url"` + CommentsURL string `json:"comments_url"` + IssueCommentURL string `json:"issue_comment_url"` + ContentsURL string `json:"contents_url"` + CompareURL string `json:"compare_url"` + MergesURL string `json:"merges_url"` + ArchiveURL string `json:"archive_url"` + DownloadsURL string `json:"downloads_url"` + IssuesURL string `json:"issues_url"` + PullsURL string `json:"pulls_url"` + MilestonesURL string `json:"milestones_url"` + NotificationsURL string `json:"notifications_url"` + LabelsURL string `json:"labels_url"` + ReleasesURL string `json:"releases_url"` + DeploymentsURL string `json:"deployments_url"` +} + +type Actor struct { + Login string `json:"login"` + ID int `json:"id"` + NodeID string `json:"node_id"` + AvatarURL string `json:"avatar_url"` + GravatarID string `json:"gravatar_id"` + + URL string `json:"url"` + HTMLURL string `json:"html_url"` + FollowersURL string `json:"followers_url"` + FollowingURL string `json:"following_url"` + GistsURL string `json:"gists_url"` + + StarredURL string `json:"starred_url"` + SubscriptionsURL string `json:"subscriptions_url"` + OrganizationsURL string `json:"organizations_url"` + ReposURL string `json:"repos_url"` + EventsURL string `json:"events_url"` + + ReceivedEventsURL string `json:"received_events_url"` + Type string `json:"type"` + SiteAdmin bool `json:"site_admin"` +} + +type ignored_repos struct { + RepositoryCount int `json:"repository_count"` + Repositories []string `json:"repositories"` +} + +type SubmitResponse struct { + ID int `json:"id"` + ControllerRepo ControllerRepo `json:"controller_repo"` + Actor Actor `json:"actor"` + QueryLanguage string `json:"query_language"` + QueryPackURL string `json:"query_pack_url"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + Status string `json:"status"` + SkippedRepositories SkippedRepositories `json:"skipped_repositories"` +} + +type SubmitMsg struct { + ActionRepoRef string `json:"action_repo_ref"` + Language string `json:"language"` + QueryPack string `json:"query_pack"` + Repositories []string `json:"repositories"` +} diff --git a/pkg/common/types.go b/pkg/common/types.go new file mode 100644 index 0000000..670df5c --- /dev/null +++ b/pkg/common/types.go @@ -0,0 +1,52 @@ +package common + +type AnalyzeJob struct { + MirvaRequestID int + + QueryPackId int + QueryLanguage string + + ORL OwnerRepo +} + +type OwnerRepo struct { + Owner string + Repo string +} + +type AnalyzeResult struct { + RunAnalysisSARIF string + RunAnalysisBQRS string +} + +type Status int + +const ( + StatusInProgress = iota + StatusQueued + StatusError + StatusSuccess + StatusFailed +) + +func (s Status) ToExternalString() string { + switch s { + case StatusInProgress: + return "in_progress" + case StatusQueued: + return "queued" + case StatusError: + return "error" + case StatusSuccess: + return "succeeded" + case StatusFailed: + return "failed" + default: + return "unknown" + } +} + +type JobSpec struct { + ID int + OwnerRepo +} diff --git a/pkg/queue/queue.go b/pkg/queue/queue.go index 8b8faff..4de83b8 100644 --- a/pkg/queue/queue.go +++ b/pkg/queue/queue.go @@ -2,30 +2,29 @@ package queue import ( "log/slog" + "mrvacommander/pkg/common" "mrvacommander/pkg/storage" - - co "github.com/hohn/ghes-mirva-server/common" ) var ( NumWorkers int - Jobs chan co.AnalyzeJob - Results chan co.AnalyzeResult + Jobs chan common.AnalyzeJob + Results chan common.AnalyzeResult ) -func StartAnalyses(analysis_repos *map[co.OwnerRepo]storage.DBLocation, session_id int, +func StartAnalyses(analysis_repos *map[common.OwnerRepo]storage.DBLocation, session_id int, session_language string) { slog.Debug("Queueing codeql database analyze jobs") for orl := range *analysis_repos { - info := co.AnalyzeJob{ + info := common.AnalyzeJob{ QueryPackId: session_id, QueryLanguage: session_language, ORL: orl, } Jobs <- info - storage.SetStatus(session_id, orl, co.StatusQueued) + storage.SetStatus(session_id, orl, common.StatusQueued) storage.AddJob(session_id, info) } } diff --git a/pkg/server/server.go b/pkg/server/server.go index c2f247c..e47c721 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -15,12 +15,11 @@ import ( "strings" "time" + "mrvacommander/pkg/common" "mrvacommander/pkg/queue" "mrvacommander/pkg/storage" "github.com/gorilla/mux" - "github.com/hohn/ghes-mirva-server/api" - co "github.com/hohn/ghes-mirva-server/common" ) func (c *CommanderSingle) Run() { @@ -62,16 +61,16 @@ func (c *CommanderSingle) Setup(st *State) { log.Fatal(http.ListenAndServe(":8080", r)) } -func (c *CommanderSingle) StatusResponse(w http.ResponseWriter, js co.JobSpec, ji co.JobInfo, vaid int) { +func (c *CommanderSingle) StatusResponse(w http.ResponseWriter, js common.JobSpec, ji common.JobInfo, vaid int) { slog.Debug("Submitting status response", "session", vaid) - all_scanned := []api.ScannedRepo{} + all_scanned := []common.ScannedRepo{} jobs := storage.GetJobList(js.ID) for _, job := range jobs { astat := storage.GetStatus(js.ID, job.ORL).ToExternalString() all_scanned = append(all_scanned, - api.ScannedRepo{ - Repository: api.Repository{ + common.ScannedRepo{ + Repository: common.Repository{ ID: 0, Name: job.ORL.Repo, FullName: fmt.Sprintf("%s/%s", job.ORL.Owner, job.ORL.Repo), @@ -88,10 +87,10 @@ func (c *CommanderSingle) StatusResponse(w http.ResponseWriter, js co.JobSpec, j astat := storage.GetStatus(js.ID, js.OwnerRepo).ToExternalString() - status := api.StatusResponse{ + status := common.StatusResponse{ SessionId: js.ID, - ControllerRepo: api.ControllerRepo{}, - Actor: api.Actor{}, + ControllerRepo: common.ControllerRepo{}, + Actor: common.Actor{}, QueryLanguage: ji.QueryLanguage, QueryPackURL: "", // FIXME CreatedAt: ji.CreatedAt, @@ -145,7 +144,7 @@ func (c *CommanderSingle) MirvaStatus(w http.ResponseWriter, r *http.Request) { job := spec[0] - js := co.JobSpec{ + js := common.JobSpec{ ID: job.QueryPackId, OwnerRepo: job.ORL, } @@ -172,9 +171,9 @@ func (c *CommanderSingle) MirvaDownloadArtifact(w http.ResponseWriter, r *http.R http.Error(w, err.Error(), http.StatusInternalServerError) return } - js := co.JobSpec{ + js := common.JobSpec{ ID: vaid, - OwnerRepo: co.OwnerRepo{ + OwnerRepo: common.OwnerRepo{ Owner: vars["repo_owner"], Repo: vars["repo_name"], }, @@ -182,13 +181,13 @@ func (c *CommanderSingle) MirvaDownloadArtifact(w http.ResponseWriter, r *http.R c.DownloadResponse(w, js, vaid) } -func (c *CommanderSingle) DownloadResponse(w http.ResponseWriter, js co.JobSpec, vaid int) { +func (c *CommanderSingle) DownloadResponse(w http.ResponseWriter, js common.JobSpec, vaid int) { slog.Debug("Forming download response", "session", vaid, "job", js) astat := storage.GetStatus(vaid, js.OwnerRepo) - var dlr api.DownloadResponse - if astat == co.StatusSuccess { + var dlr common.DownloadResponse + if astat == common.StatusSuccess { au, err := storage.ArtifactURL(js, vaid) if err != nil { @@ -196,8 +195,8 @@ func (c *CommanderSingle) DownloadResponse(w http.ResponseWriter, js co.JobSpec, return } - dlr = api.DownloadResponse{ - Repository: api.DownloadRepo{ + dlr = common.DownloadResponse{ + Repository: common.DownloadRepo{ Name: js.Repo, FullName: fmt.Sprintf("%s/%s", js.Owner, js.Repo), }, @@ -209,8 +208,8 @@ func (c *CommanderSingle) DownloadResponse(w http.ResponseWriter, js co.JobSpec, ArtifactURL: au, } } else { - dlr = api.DownloadResponse{ - Repository: api.DownloadRepo{ + dlr = common.DownloadResponse{ + Repository: common.DownloadRepo{ Name: js.Repo, FullName: fmt.Sprintf("%s/%s", js.Owner, js.Repo), }, @@ -320,7 +319,7 @@ func (c *CommanderSingle) MirvaRequest(w http.ResponseWriter, r *http.Request) { w.Write(submit_response) } -func ORToArr(aor []co.OwnerRepo) ([]string, int) { +func ORToArr(aor []common.OwnerRepo) ([]string, int) { repos := []string{} count := len(aor) for _, repo := range aor { @@ -331,29 +330,29 @@ func ORToArr(aor []co.OwnerRepo) ([]string, int) { func submit_response(sn SessionInfo) ([]byte, error) { // Construct the response bottom-up - var m_cr api.ControllerRepo - var m_ac api.Actor + var m_cr common.ControllerRepo + var m_ac common.Actor repos, count := ORToArr(sn.NotFoundRepos) - r_nfr := api.NotFoundRepos{RepositoryCount: count, RepositoryFullNames: repos} + r_nfr := common.NotFoundRepos{RepositoryCount: count, RepositoryFullNames: repos} repos, count = ORToArr(sn.AccessMismatchRepos) - r_amr := api.AccessMismatchRepos{RepositoryCount: count, Repositories: repos} + r_amr := common.AccessMismatchRepos{RepositoryCount: count, Repositories: repos} repos, count = ORToArr(sn.NoCodeqlDBRepos) - r_ncd := api.NoCodeqlDBRepos{RepositoryCount: count, Repositories: repos} + r_ncd := common.NoCodeqlDBRepos{RepositoryCount: count, Repositories: repos} // TODO fill these with real values? repos, count = ORToArr(sn.NoCodeqlDBRepos) - r_olr := api.OverLimitRepos{RepositoryCount: count, Repositories: repos} + r_olr := common.OverLimitRepos{RepositoryCount: count, Repositories: repos} - m_skip := api.SkippedRepositories{ + m_skip := common.SkippedRepositories{ AccessMismatchRepos: r_amr, NotFoundRepos: r_nfr, NoCodeqlDBRepos: r_ncd, OverLimitRepos: r_olr} - m_sr := api.SubmitResponse{ + m_sr := common.SubmitResponse{ Actor: m_ac, ControllerRepo: m_cr, ID: sn.ID, @@ -369,10 +368,10 @@ func submit_response(sn SessionInfo) ([]byte, error) { joblist := storage.GetJobList(sn.ID) for _, job := range joblist { - storage.SetJobInfo(co.JobSpec{ + storage.SetJobInfo(common.JobSpec{ ID: sn.ID, OwnerRepo: job.ORL, - }, co.JobInfo{ + }, common.JobInfo{ QueryLanguage: sn.Language, CreatedAt: m_sr.CreatedAt, UpdatedAt: m_sr.UpdatedAt, @@ -391,28 +390,28 @@ func submit_response(sn SessionInfo) ([]byte, error) { } -func (c *CommanderSingle) collectRequestInfo(w http.ResponseWriter, r *http.Request, sessionId int) (string, []co.OwnerRepo, string, error) { +func (c *CommanderSingle) collectRequestInfo(w http.ResponseWriter, r *http.Request, sessionId int) (string, []common.OwnerRepo, string, error) { slog.Debug("Collecting session info") if r.Body == nil { err := errors.New("missing request body") log.Println(err) http.Error(w, err.Error(), http.StatusNoContent) - return "", []co.OwnerRepo{}, "", err + return "", []common.OwnerRepo{}, "", err } buf, err := io.ReadAll(r.Body) if err != nil { var w http.ResponseWriter slog.Error("Error reading MRVA submission body", "error", err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) - return "", []co.OwnerRepo{}, "", err + return "", []common.OwnerRepo{}, "", err } msg, err := TrySubmitMsg(buf) if err != nil { // Unknown message slog.Error("Unknown MRVA submission body format") http.Error(w, err.Error(), http.StatusBadRequest) - return "", []co.OwnerRepo{}, "", err + return "", []common.OwnerRepo{}, "", err } // Decompose the SubmitMsg and keep information @@ -421,19 +420,19 @@ func (c *CommanderSingle) collectRequestInfo(w http.ResponseWriter, r *http.Requ slog.Error("MRVA submission body querypack has invalid format") err := errors.New("MRVA submission body querypack has invalid format") http.Error(w, err.Error(), http.StatusBadRequest) - return "", []co.OwnerRepo{}, "", err + return "", []common.OwnerRepo{}, "", err } session_tgz_ref, err := c.extract_tgz(msg.QueryPack, sessionId) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) - return "", []co.OwnerRepo{}, "", err + return "", []common.OwnerRepo{}, "", err } // 2. Save the language session_language := msg.Language // 3. Save the repositories - var session_repositories []co.OwnerRepo + var session_repositories []common.OwnerRepo for _, v := range msg.Repositories { t := strings.Split(v, "/") @@ -443,18 +442,18 @@ func (c *CommanderSingle) collectRequestInfo(w http.ResponseWriter, r *http.Requ http.Error(w, err, http.StatusBadRequest) } session_repositories = append(session_repositories, - co.OwnerRepo{Owner: t[0], Repo: t[1]}) + common.OwnerRepo{Owner: t[0], Repo: t[1]}) } return session_language, session_repositories, session_tgz_ref, nil } // Try to extract a SubmitMsg from a json-encoded buffer -func TrySubmitMsg(buf []byte) (SubmitMsg, error) { +func TrySubmitMsg(buf []byte) (common.SubmitMsg, error) { buf1 := make([]byte, len(buf)) copy(buf1, buf) dec := json.NewDecoder(bytes.NewReader(buf1)) dec.DisallowUnknownFields() - var m SubmitMsg + var m common.SubmitMsg err := dec.Decode(&m) return m, err } diff --git a/pkg/server/types.go b/pkg/server/types.go index 0fac456..d7b3ac8 100644 --- a/pkg/server/types.go +++ b/pkg/server/types.go @@ -2,231 +2,12 @@ package server import ( "mrvacommander/pkg/agent" + "mrvacommander/pkg/common" "mrvacommander/pkg/logger" "mrvacommander/pkg/queue" "mrvacommander/pkg/storage" - - co "github.com/hohn/ghes-mirva-server/common" ) -type DownloadResponse struct { - Repository DownloadRepo `json:"repository"` - AnalysisStatus string `json:"analysis_status"` - ResultCount int `json:"result_count"` - ArtifactSizeBytes int `json:"artifact_size_in_bytes"` - DatabaseCommitSha string `json:"database_commit_sha"` - SourceLocationPrefix string `json:"source_location_prefix"` - ArtifactURL string `json:"artifact_url"` -} - -type DownloadRepo struct { - ID int `json:"id"` - NodeID string `json:"node_id"` - Name string `json:"name"` - FullName string `json:"full_name"` - Private bool `json:"private"` - Owner Actor `json:"owner"` - - HTMLURL string `json:"html_url"` - Description string `json:"description"` - Fork bool `json:"fork"` - ForksURL string `json:"forks_url"` - KeysURL string `json:"keys_url"` - CollaboratorsURL string `json:"collaborators_url"` - TeamsURL string `json:"teams_url"` - HooksURL string `json:"hooks_url"` - IssueEventsURL string `json:"issue_events_url"` - EventsURL string `json:"events_url"` - - AssigneesURL string `json:"assignees_url"` - BranchesURL string `json:"branches_url"` - TagsURL string `json:"tags_url"` - BlobsURL string `json:"blobs_url"` - GitTagsURL string `json:"git_tags_url"` - GitRefsURL string `json:"git_refs_url"` - TreesURL string `json:"trees_url"` - StatusesURL string `json:"statuses_url"` - LanguagesURL string `json:"languages_url"` - - StargazersURL string `json:"stargazers_url"` - ContributorsURL string `json:"contributors_url"` - SubscribersURL string `json:"subscribers_url"` - SubscriptionURL string `json:"subscription_url"` - - CommitsURL string `json:"commits_url"` - GitCommitsURL string `json:"git_commits_url"` - CommentsURL string `json:"comments_url"` - IssueCommentURL string `json:"issue_comment_url"` - ContentsURL string `json:"contents_url"` - CompareURL string `json:"compare_url"` - MergesURL string `json:"merges_url"` - ArchiveURL string `json:"archive_url"` - DownloadsURL string `json:"downloads_url"` - IssuesURL string `json:"issues_url"` - PullsURL string `json:"pulls_url"` - MilestonesURL string `json:"milestones_url"` - NotificationsURL string `json:"notifications_url"` - LabelsURL string `json:"labels_url"` - ReleasesURL string `json:"releases_url"` - DeploymentsURL string `json:"deployments_url"` -} - -type ControllerRepo struct { - ID int `json:"id"` - NodeID string `json:"node_id"` - Name string `json:"name"` - FullName string `json:"full_name"` - Private bool `json:"private"` - Owner struct{} `json:"owner"` - HTMLURL string `json:"html_url"` - Description string `json:"description"` - Fork bool `json:"fork"` - ForksURL string `json:"forks_url"` - KeysURL string `json:"keys_url"` - CollaboratorsURL string `json:"collaborators_url"` - TeamsURL string `json:"teams_url"` - HooksURL string `json:"hooks_url"` - IssueEventsURL string `json:"issue_events_url"` - EventsURL string `json:"events_url"` - - AssigneesURL string `json:"assignees_url"` - BranchesURL string `json:"branches_url"` - TagsURL string `json:"tags_url"` - BlobsURL string `json:"blobs_url"` - GitTagsURL string `json:"git_tags_url"` - GitRefsURL string `json:"git_refs_url"` - TreesURL string `json:"trees_url"` - StatusesURL string `json:"statuses_url"` - LanguagesURL string `json:"languages_url"` - - StargazersURL string `json:"stargazers_url"` - ContributorsURL string `json:"contributors_url"` - SubscribersURL string `json:"subscribers_url"` - SubscriptionURL string `json:"subscription_url"` - - CommitsURL string `json:"commits_url"` - GitCommitsURL string `json:"git_commits_url"` - CommentsURL string `json:"comments_url"` - IssueCommentURL string `json:"issue_comment_url"` - ContentsURL string `json:"contents_url"` - CompareURL string `json:"compare_url"` - MergesURL string `json:"merges_url"` - ArchiveURL string `json:"archive_url"` - DownloadsURL string `json:"downloads_url"` - IssuesURL string `json:"issues_url"` - PullsURL string `json:"pulls_url"` - MilestonesURL string `json:"milestones_url"` - NotificationsURL string `json:"notifications_url"` - LabelsURL string `json:"labels_url"` - ReleasesURL string `json:"releases_url"` - DeploymentsURL string `json:"deployments_url"` -} - -type Actor struct { - Login string `json:"login"` - ID int `json:"id"` - NodeID string `json:"node_id"` - AvatarURL string `json:"avatar_url"` - GravatarID string `json:"gravatar_id"` - - URL string `json:"url"` - HTMLURL string `json:"html_url"` - FollowersURL string `json:"followers_url"` - FollowingURL string `json:"following_url"` - GistsURL string `json:"gists_url"` - - StarredURL string `json:"starred_url"` - SubscriptionsURL string `json:"subscriptions_url"` - OrganizationsURL string `json:"organizations_url"` - ReposURL string `json:"repos_url"` - EventsURL string `json:"events_url"` - - ReceivedEventsURL string `json:"received_events_url"` - Type string `json:"type"` - SiteAdmin bool `json:"site_admin"` -} - -type SkippedRepositories struct { - AccessMismatchRepos AccessMismatchRepos `json:"access_mismatch_repos"` - NotFoundRepos NotFoundRepos `json:"not_found_repos"` - NoCodeqlDBRepos NoCodeqlDBRepos `json:"no_codeql_db_repos"` - OverLimitRepos OverLimitRepos `json:"over_limit_repos"` -} - -type ignored_repos struct { - RepositoryCount int `json:"repository_count"` - Repositories []string `json:"repositories"` -} - -type AccessMismatchRepos struct { - RepositoryCount int `json:"repository_count"` - Repositories []string `json:"repositories"` -} - -type NotFoundRepos struct { - RepositoryCount int `json:"repository_count"` - RepositoryFullNames []string `json:"repository_full_names"` -} - -type NoCodeqlDBRepos struct { - RepositoryCount int `json:"repository_count"` - Repositories []string `json:"repositories"` -} - -type OverLimitRepos struct { - RepositoryCount int `json:"repository_count"` - Repositories []string `json:"repositories"` -} - -type SubmitResponse struct { - ID int `json:"id"` - ControllerRepo ControllerRepo `json:"controller_repo"` - Actor Actor `json:"actor"` - QueryLanguage string `json:"query_language"` - QueryPackURL string `json:"query_pack_url"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` - Status string `json:"status"` - SkippedRepositories SkippedRepositories `json:"skipped_repositories"` -} - -type Repository struct { - ID int `json:"id"` - Name string `json:"name"` - FullName string `json:"full_name"` - Private bool `json:"private"` - StargazersCount int `json:"stargazers_count"` - UpdatedAt string `json:"updated_at"` -} - -type ScannedRepo struct { - Repository Repository `json:"repository"` - AnalysisStatus string `json:"analysis_status"` - ResultCount int `json:"result_count"` - ArtifactSizeBytes int `json:"artifact_size_in_bytes"` -} - -type StatusResponse struct { - SessionId int `json:"id"` - ControllerRepo ControllerRepo `json:"controller_repo"` - Actor Actor `json:"actor"` - QueryLanguage string `json:"query_language"` - QueryPackURL string `json:"query_pack_url"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` - ActionsWorkflowRunID int `json:"actions_workflow_run_id"` - Status string `json:"status"` - ScannedRepositories []ScannedRepo `json:"scanned_repositories"` - SkippedRepositories SkippedRepositories `json:"skipped_repositories"` -} - -type SubmitMsg struct { - ActionRepoRef string `json:"action_repo_ref"` - Language string `json:"language"` - QueryPack string `json:"query_pack"` - Repositories []string `json:"repositories"` -} - type SessionInfo struct { ID int Owner string @@ -234,14 +15,14 @@ type SessionInfo struct { QueryPack string Language string - Repositories []co.OwnerRepo + Repositories []common.OwnerRepo - AccessMismatchRepos []co.OwnerRepo - NotFoundRepos []co.OwnerRepo - NoCodeqlDBRepos []co.OwnerRepo - OverLimitRepos []co.OwnerRepo + AccessMismatchRepos []common.OwnerRepo + NotFoundRepos []common.OwnerRepo + NoCodeqlDBRepos []common.OwnerRepo + OverLimitRepos []common.OwnerRepo - AnalysisRepos *map[co.OwnerRepo]storage.DBLocation + AnalysisRepos *map[common.OwnerRepo]storage.DBLocation } type CommanderSingle struct { diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 952c710..436170c 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -12,14 +12,14 @@ import ( "path/filepath" "sync" - co "github.com/hohn/ghes-mirva-server/common" + "mrvacommander/pkg/common" ) var ( - jobs map[int][]co.AnalyzeJob = make(map[int][]co.AnalyzeJob) - info map[co.JobSpec]co.JobInfo = make(map[co.JobSpec]co.JobInfo) - status map[co.JobSpec]co.Status = make(map[co.JobSpec]co.Status) - result map[co.JobSpec]co.AnalyzeResult = make(map[co.JobSpec]co.AnalyzeResult) + jobs map[int][]common.AnalyzeJob = make(map[int][]common.AnalyzeJob) + info map[common.JobSpec]common.JobInfo = make(map[common.JobSpec]common.JobInfo) + status map[common.JobSpec]common.Status = make(map[common.JobSpec]common.Status) + result map[common.JobSpec]common.AnalyzeResult = make(map[common.JobSpec]common.AnalyzeResult) mutex sync.Mutex ) @@ -62,8 +62,8 @@ func (s *StorageSingle) SaveQueryPack(tgz []byte, sessionId int) (string, error) // Determine for which repositories codeql databases are available. // // Those will be the analysis_repos. The rest will be skipped. -func (s *StorageSingle) FindAvailableDBs(analysisReposRequested []co.OwnerRepo) (not_found_repos []co.OwnerRepo, - analysisRepos *map[co.OwnerRepo]DBLocation) { +func (s *StorageSingle) FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (not_found_repos []common.OwnerRepo, + analysisRepos *map[common.OwnerRepo]DBLocation) { slog.Debug("Looking for available CodeQL databases") cwd, err := os.Getwd() @@ -72,9 +72,9 @@ func (s *StorageSingle) FindAvailableDBs(analysisReposRequested []co.OwnerRepo) return } - analysisRepos = &map[co.OwnerRepo]DBLocation{} + analysisRepos = &map[common.OwnerRepo]DBLocation{} - not_found_repos = []co.OwnerRepo{} + not_found_repos = []common.OwnerRepo{} for _, rep := range analysisReposRequested { dbPrefix := filepath.Join(cwd, "codeql", "dbs", rep.Owner, rep.Repo) @@ -93,7 +93,7 @@ func (s *StorageSingle) FindAvailableDBs(analysisReposRequested []co.OwnerRepo) return not_found_repos, analysisRepos } -func ArtifactURL(js co.JobSpec, vaid int) (string, error) { +func ArtifactURL(js common.JobSpec, vaid int) (string, error) { // We're looking for paths like // codeql/sarif/google/flatbuffers/google_flatbuffers.sarif @@ -114,14 +114,14 @@ func ArtifactURL(js co.JobSpec, vaid int) (string, error) { return au, nil } -func GetResult(js co.JobSpec) co.AnalyzeResult { +func GetResult(js common.JobSpec) common.AnalyzeResult { mutex.Lock() defer mutex.Unlock() ar := result[js] return ar } -func PackageResults(ar co.AnalyzeResult, owre co.OwnerRepo, vaid int) (zipPath string, e error) { +func PackageResults(ar common.AnalyzeResult, owre common.OwnerRepo, vaid int) (zipPath string, e error) { slog.Debug("Readying zip file with .sarif/.bqrs", "analyze-result", ar) cwd, err := os.Getwd() @@ -178,28 +178,28 @@ func PackageResults(ar co.AnalyzeResult, owre co.OwnerRepo, vaid int) (zipPath s return zpath, nil } -func GetJobList(sessionid int) []co.AnalyzeJob { +func GetJobList(sessionid int) []common.AnalyzeJob { mutex.Lock() defer mutex.Unlock() return jobs[sessionid] } -func GetJobInfo(js co.JobSpec) co.JobInfo { +func GetJobInfo(js common.JobSpec) common.JobInfo { mutex.Lock() defer mutex.Unlock() return info[js] } -func SetJobInfo(js co.JobSpec, ji co.JobInfo) { +func SetJobInfo(js common.JobSpec, ji common.JobInfo) { mutex.Lock() defer mutex.Unlock() info[js] = ji } -func GetStatus(sessionid int, orl co.OwnerRepo) co.Status { +func GetStatus(sessionid int, orl common.OwnerRepo) common.Status { mutex.Lock() defer mutex.Unlock() - return status[co.JobSpec{ID: sessionid, OwnerRepo: orl}] + return status[common.JobSpec{ID: sessionid, OwnerRepo: orl}] } func ResultAsFile(path string) (string, []byte, error) { @@ -217,13 +217,13 @@ func ResultAsFile(path string) (string, []byte, error) { return fpath, file, nil } -func SetStatus(sessionid int, orl co.OwnerRepo, s co.Status) { +func SetStatus(sessionid int, orl common.OwnerRepo, s common.Status) { mutex.Lock() defer mutex.Unlock() - status[co.JobSpec{ID: sessionid, OwnerRepo: orl}] = s + status[common.JobSpec{ID: sessionid, OwnerRepo: orl}] = s } -func AddJob(sessionid int, job co.AnalyzeJob) { +func AddJob(sessionid int, job common.AnalyzeJob) { mutex.Lock() defer mutex.Unlock() jobs[sessionid] = append(jobs[sessionid], job) diff --git a/pkg/storage/types.go b/pkg/storage/types.go index 766e729..1305ce1 100644 --- a/pkg/storage/types.go +++ b/pkg/storage/types.go @@ -1,14 +1,14 @@ package storage import ( - co "github.com/hohn/ghes-mirva-server/common" + "mrvacommander/pkg/common" ) type Storage interface { NextID() int SaveQueryPack(tgz []byte, sessionID int) (storagePath string, error error) - FindAvailableDBs(analysisReposRequested []co.OwnerRepo) (not_found_repos []co.OwnerRepo, - analysisRepos *map[co.OwnerRepo]DBLocation) + FindAvailableDBs(analysisReposRequested []common.OwnerRepo) (not_found_repos []common.OwnerRepo, + analysisRepos *map[common.OwnerRepo]DBLocation) } type DBLocation struct {