This commit is contained in:
Michael Hohn
2024-05-08 10:42:15 -07:00
committed by =Michael Hohn
parent 901999df5c
commit 16bedf89d8
13 changed files with 615 additions and 0 deletions

View File

@@ -0,0 +1,194 @@
/*
Copyright © 2024 github
*/
package cmd
import (
"log"
"log/slog"
"net/http"
"strconv"
cim "github.com/advanced-security/mrvacommander/lib/commander/inmemory"
"github.com/gorilla/mux"
"github.com/hohn/ghes-mirva-server/analyze"
co "github.com/hohn/ghes-mirva-server/common"
"github.com/hohn/ghes-mirva-server/store"
"github.com/spf13/cobra"
)
// startCmd represents the start command
var startCmd = &cobra.Command{
Use: "start",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
LogAbove(LogWarning, "Starting server")
serve()
},
}
func serve() {
r := mux.NewRouter()
//
// First are the API endpoints that mirror those used in the github API
//
r.HandleFunc("/repos/{owner}/{repo}/code-scanning/codeql/variant-analyses", MirvaRequest)
// /repos/hohn /mirva-controller/code-scanning/codeql/variant-analyses
// Or via
r.HandleFunc("/{repository_id}/code-scanning/codeql/variant-analyses", MirvaRequestID)
r.HandleFunc("/", RootHandler)
// This is the standalone status request.
// It's also the first request made when downloading; the difference is on the
// client side's handling.
r.HandleFunc("/repos/{owner}/{repo}/code-scanning/codeql/variant-analyses/{codeql_variant_analysis_id}", MirvaStatus)
r.HandleFunc("/repos/{controller_owner}/{controller_repo}/code-scanning/codeql/variant-analyses/{codeql_variant_analysis_id}/repos/{repo_owner}/{repo_name}", MirvaDownloadArtifact)
r.HandleFunc("/codeql-query-console/codeql-variant-analysis-repo-tasks/{codeql_variant_analysis_id}/{repo_id}/{owner_id}/{controller_repo_id}", MirvaDownLoad3)
r.HandleFunc("/github-codeql-query-console-prod/codeql-variant-analysis-repo-tasks/{codeql_variant_analysis_id}/{repo_id}", MirvaDownLoad4)
//
// Now some support API endpoints
//
r.HandleFunc("/download-server/{local_path:.*}", MirvaDownloadServe)
//
// Bind to a port and pass our router in
//
log.Fatal(http.ListenAndServe(":8080", r))
}
func RootHandler(w http.ResponseWriter, r *http.Request) {
LogAbove(LogWarning, "Request on /")
}
func MirvaStatus(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
slog.Info("mrva status request for ",
"owner", vars["owner"],
"repo", vars["repo"],
"codeql_variant_analysis_id", vars["codeql_variant_analysis_id"])
id, err := strconv.Atoi(vars["codeql_variant_analysis_id"])
if err != nil {
slog.Error("Variant analysis is is not integer", "id",
vars["codeql_variant_analysis_id"])
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// The status reports one status for all jobs belonging to an id.
// So we simply report the status of a job as the status of all.
spec := store.GetJobList(id)
if spec == nil {
slog.Error("No jobs found for given job id",
"id", vars["codeql_variant_analysis_id"])
http.Error(w, err.Error(), http.StatusUnprocessableEntity)
return
}
job := spec[0]
js := co.JobSpec{
ID: job.QueryPackId,
OwnerRepo: job.ORL,
}
ji := store.GetJobInfo(js)
analyze.StatusResponse(w, js, ji, id)
cim.StatusResponse(w, js, ji, id)
}
// Download artifacts
func MirvaDownloadArtifact(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
slog.Info("MRVA artifact download",
"controller_owner", vars["controller_owner"],
"controller_repo", vars["controller_repo"],
"codeql_variant_analysis_id", vars["codeql_variant_analysis_id"],
"repo_owner", vars["repo_owner"],
"repo_name", vars["repo_name"],
)
vaid, err := strconv.Atoi(vars["codeql_variant_analysis_id"])
if err != nil {
slog.Error("Variant analysis is is not integer", "id",
vars["codeql_variant_analysis_id"])
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
js := co.JobSpec{
ID: vaid,
OwnerRepo: co.OwnerRepo{
Owner: vars["repo_owner"],
Repo: vars["repo_name"],
},
}
analyze.DownloadResponse(w, js, vaid)
}
func MirvaDownLoad3(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
LogAbove(LogWarning, "mrva download step 3 for (%s,%s,%s,%s)\n",
vars["codeql_variant_analysis_id"],
vars["repo_id"],
vars["owner_id"],
vars["controller_repo_id"])
}
func MirvaDownLoad4(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
LogAbove(LogWarning, "mrva download step 4 for (%s,%s)\n",
vars["codeql_variant_analysis_id"],
vars["repo_id"])
}
func MirvaDownloadServe(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
slog.Info("File download request", "local_path", vars["local_path"])
analyze.FileDownload(w, vars["local_path"])
}
func MirvaRequestID(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
LogAbove(LogWarning, "New mrva using repository_id=%v\n", vars["repository_id"])
}
func MirvaRequest(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
slog.Info("New mrva run ", "owner", vars["owner"], "repo", vars["repo"])
// TODO Change this to functional style?
session := new(MirvaSession)
session.id = next_id()
session.owner = vars["owner"]
session.controller_repo = vars["repo"]
session.collect_info(w, r)
session.find_available_DBs()
session.start_analyses()
session.submit_response(w)
session.save()
}
func init() {
rootCmd.AddCommand(startCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// startCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// startCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}

93
cmd/root.go Normal file
View File

@@ -0,0 +1,93 @@
/*
Copyright © 2024 github
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"log"
"log/slog"
"os"
"github.com/spf13/cobra"
)
// rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{
Use: "ghes-mirva-server",
Short: "A brief description of your application",
Long: `A longer description that spans multiple lines and likely contains
examples and usage of using your application. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
// Uncomment the following line if your bare application
// has an action associated with it:
// Run: func(cmd *cobra.Command, args []string) {
PersistentPreRun: func(cmd *cobra.Command, args []string) {
switch logging_verbosity {
case "debug":
slog.SetLogLoggerLevel(slog.LevelDebug)
case "info":
slog.SetLogLoggerLevel(slog.LevelInfo)
case "warn":
slog.SetLogLoggerLevel(slog.LevelWarn)
case "error":
slog.SetLogLoggerLevel(slog.LevelError)
default:
log.Printf("Invalid logging verbosity level: %s", logging_verbosity)
}
},
}
func init() {
// Here you will define your flags and configuration settings.
// Cobra supports persistent flags, which, if defined here,
// will be global for your application.
// rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ghes-mirva-server.yaml)")
// Cobra also supports local flags, which will only run
// when this action is called directly.
rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
rootCmd.Flags().StringVar(&Backend, "backend", "local", `Backend to use.
Currently available:
- local
`)
rootCmd.PersistentFlags().StringVar(&logging_verbosity, "verbosity", "info", `Logging verbosity, from least to most verbose:
- error
- warn
- info
- debug
`)
}
// Execute adds all child commands to the root command and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
rootCmd.AddCommand(startCmd)
err := rootCmd.Execute()
if err != nil {
os.Exit(1)
}
}
var logging_verbosity string
var Backend string = "local"

14
go.mod
View File

@@ -0,0 +1,14 @@
module github.com/advanced-security/mrvacommander
go 1.22.0
require (
github.com/gorilla/mux v1.8.1
github.com/hohn/ghes-mirva-server v0.0.0-20240313191620-9917867ea540
github.com/spf13/cobra v1.8.0
)
require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
)

14
go.sum Normal file
View File

@@ -0,0 +1,14 @@
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/hohn/ghes-mirva-server v0.0.0-20240313191620-9917867ea540 h1:ohnDVLM/VvVCVfjvSYKAPZIQhOPRKk1ZcZcMzf4yT8k=
github.com/hohn/ghes-mirva-server v0.0.0-20240313191620-9917867ea540/go.mod h1:ircD+yE4AxWL/DufgcLDi191c+JM9ge/C3yiT/0zL+U=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

0
interfaces/common.go Normal file
View File

View File

@@ -0,0 +1,66 @@
package analyze
import (
"encoding/json"
"fmt"
"log/slog"
"net/http"
"github.com/hohn/ghes-mirva-server/api"
co "github.com/hohn/ghes-mirva-server/common"
"github.com/hohn/ghes-mirva-server/store"
)
func StatusResponse(w http.ResponseWriter, js co.JobSpec, ji co.JobInfo, vaid int) {
slog.Debug("Submitting status response", "session", vaid)
all_scanned := []api.ScannedRepo{}
jobs := store.GetJobList(js.ID)
for _, job := range jobs {
astat := store.GetStatus(js.ID, job.ORL).ToExternalString()
all_scanned = append(all_scanned,
api.ScannedRepo{
Repository: api.Repository{
ID: 0,
Name: job.ORL.Repo,
FullName: fmt.Sprintf("%s/%s", job.ORL.Owner, job.ORL.Repo),
Private: false,
StargazersCount: 0,
UpdatedAt: ji.UpdatedAt,
},
AnalysisStatus: astat,
ResultCount: 123, // FIXME 123 is a lie so the client downloads
ArtifactSizeBytes: 123, // FIXME
},
)
}
astat := store.GetStatus(js.ID, js.OwnerRepo).ToExternalString()
status := api.StatusResponse{
SessionId: js.ID,
ControllerRepo: api.ControllerRepo{},
Actor: api.Actor{},
QueryLanguage: ji.QueryLanguage,
QueryPackURL: "", // FIXME
CreatedAt: ji.CreatedAt,
UpdatedAt: ji.UpdatedAt,
ActionsWorkflowRunID: 0, // FIXME
Status: astat,
ScannedRepositories: all_scanned,
SkippedRepositories: ji.SkippedRepositories,
}
// Encode the response as JSON
submitStatus, err := json.Marshal(status)
if err != nil {
slog.Error("Error encoding response as JSON:",
"error", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Send analysisReposJSON via ResponseWriter
w.Header().Set("Content-Type", "application/json")
w.Write(submitStatus)
}

22
main.go Normal file
View File

@@ -0,0 +1,22 @@
/*
Copyright © 2024 NAME HERE <EMAIL ADDRESS>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import "github.com/advanced-security/mrvacommander/cmd"
func main() {
cmd.Execute()
}

212
types/commander.go Normal file
View File

@@ -0,0 +1,212 @@
package types
type DownloadResponse struct {
Repository DownloadRepo `json:"repository"`
AnalysisStatus string `json:"analysis_status"`
ResultCount int `json:"result_count"`
ArtifactSizeBytes int `json:"artifact_size_in_bytes"`
DatabaseCommitSha string `json:"database_commit_sha"`
SourceLocationPrefix string `json:"source_location_prefix"`
ArtifactURL string `json:"artifact_url"`
}
type DownloadRepo struct {
ID int `json:"id"`
NodeID string `json:"node_id"`
Name string `json:"name"`
FullName string `json:"full_name"`
Private bool `json:"private"`
Owner Actor `json:"owner"`
HTMLURL string `json:"html_url"`
Description string `json:"description"`
Fork bool `json:"fork"`
ForksURL string `json:"forks_url"`
KeysURL string `json:"keys_url"`
CollaboratorsURL string `json:"collaborators_url"`
TeamsURL string `json:"teams_url"`
HooksURL string `json:"hooks_url"`
IssueEventsURL string `json:"issue_events_url"`
EventsURL string `json:"events_url"`
AssigneesURL string `json:"assignees_url"`
BranchesURL string `json:"branches_url"`
TagsURL string `json:"tags_url"`
BlobsURL string `json:"blobs_url"`
GitTagsURL string `json:"git_tags_url"`
GitRefsURL string `json:"git_refs_url"`
TreesURL string `json:"trees_url"`
StatusesURL string `json:"statuses_url"`
LanguagesURL string `json:"languages_url"`
StargazersURL string `json:"stargazers_url"`
ContributorsURL string `json:"contributors_url"`
SubscribersURL string `json:"subscribers_url"`
SubscriptionURL string `json:"subscription_url"`
CommitsURL string `json:"commits_url"`
GitCommitsURL string `json:"git_commits_url"`
CommentsURL string `json:"comments_url"`
IssueCommentURL string `json:"issue_comment_url"`
ContentsURL string `json:"contents_url"`
CompareURL string `json:"compare_url"`
MergesURL string `json:"merges_url"`
ArchiveURL string `json:"archive_url"`
DownloadsURL string `json:"downloads_url"`
IssuesURL string `json:"issues_url"`
PullsURL string `json:"pulls_url"`
MilestonesURL string `json:"milestones_url"`
NotificationsURL string `json:"notifications_url"`
LabelsURL string `json:"labels_url"`
ReleasesURL string `json:"releases_url"`
DeploymentsURL string `json:"deployments_url"`
}
type ControllerRepo struct {
ID int `json:"id"`
NodeID string `json:"node_id"`
Name string `json:"name"`
FullName string `json:"full_name"`
Private bool `json:"private"`
Owner struct{} `json:"owner"`
HTMLURL string `json:"html_url"`
Description string `json:"description"`
Fork bool `json:"fork"`
ForksURL string `json:"forks_url"`
KeysURL string `json:"keys_url"`
CollaboratorsURL string `json:"collaborators_url"`
TeamsURL string `json:"teams_url"`
HooksURL string `json:"hooks_url"`
IssueEventsURL string `json:"issue_events_url"`
EventsURL string `json:"events_url"`
AssigneesURL string `json:"assignees_url"`
BranchesURL string `json:"branches_url"`
TagsURL string `json:"tags_url"`
BlobsURL string `json:"blobs_url"`
GitTagsURL string `json:"git_tags_url"`
GitRefsURL string `json:"git_refs_url"`
TreesURL string `json:"trees_url"`
StatusesURL string `json:"statuses_url"`
LanguagesURL string `json:"languages_url"`
StargazersURL string `json:"stargazers_url"`
ContributorsURL string `json:"contributors_url"`
SubscribersURL string `json:"subscribers_url"`
SubscriptionURL string `json:"subscription_url"`
CommitsURL string `json:"commits_url"`
GitCommitsURL string `json:"git_commits_url"`
CommentsURL string `json:"comments_url"`
IssueCommentURL string `json:"issue_comment_url"`
ContentsURL string `json:"contents_url"`
CompareURL string `json:"compare_url"`
MergesURL string `json:"merges_url"`
ArchiveURL string `json:"archive_url"`
DownloadsURL string `json:"downloads_url"`
IssuesURL string `json:"issues_url"`
PullsURL string `json:"pulls_url"`
MilestonesURL string `json:"milestones_url"`
NotificationsURL string `json:"notifications_url"`
LabelsURL string `json:"labels_url"`
ReleasesURL string `json:"releases_url"`
DeploymentsURL string `json:"deployments_url"`
}
type Actor struct {
Login string `json:"login"`
ID int `json:"id"`
NodeID string `json:"node_id"`
AvatarURL string `json:"avatar_url"`
GravatarID string `json:"gravatar_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
FollowersURL string `json:"followers_url"`
FollowingURL string `json:"following_url"`
GistsURL string `json:"gists_url"`
StarredURL string `json:"starred_url"`
SubscriptionsURL string `json:"subscriptions_url"`
OrganizationsURL string `json:"organizations_url"`
ReposURL string `json:"repos_url"`
EventsURL string `json:"events_url"`
ReceivedEventsURL string `json:"received_events_url"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
}
type SkippedRepositories struct {
AccessMismatchRepos AccessMismatchRepos `json:"access_mismatch_repos"`
NotFoundRepos NotFoundRepos `json:"not_found_repos"`
NoCodeqlDBRepos NoCodeqlDBRepos `json:"no_codeql_db_repos"`
OverLimitRepos OverLimitRepos `json:"over_limit_repos"`
}
type ignored_repos struct {
RepositoryCount int `json:"repository_count"`
Repositories []string `json:"repositories"`
}
type AccessMismatchRepos struct {
RepositoryCount int `json:"repository_count"`
Repositories []string `json:"repositories"`
}
type NotFoundRepos struct {
RepositoryCount int `json:"repository_count"`
RepositoryFullNames []string `json:"repository_full_names"`
}
type NoCodeqlDBRepos struct {
RepositoryCount int `json:"repository_count"`
Repositories []string `json:"repositories"`
}
type OverLimitRepos struct {
RepositoryCount int `json:"repository_count"`
Repositories []string `json:"repositories"`
}
type SubmitResponse struct {
ID int `json:"id"`
ControllerRepo ControllerRepo `json:"controller_repo"`
Actor Actor `json:"actor"`
QueryLanguage string `json:"query_language"`
QueryPackURL string `json:"query_pack_url"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
Status string `json:"status"`
SkippedRepositories SkippedRepositories `json:"skipped_repositories"`
}
type Repository struct {
ID int `json:"id"`
Name string `json:"name"`
FullName string `json:"full_name"`
Private bool `json:"private"`
StargazersCount int `json:"stargazers_count"`
UpdatedAt string `json:"updated_at"`
}
type ScannedRepo struct {
Repository Repository `json:"repository"`
AnalysisStatus string `json:"analysis_status"`
ResultCount int `json:"result_count"`
ArtifactSizeBytes int `json:"artifact_size_in_bytes"`
}
type StatusResponse struct {
SessionId int `json:"id"`
ControllerRepo ControllerRepo `json:"controller_repo"`
Actor Actor `json:"actor"`
QueryLanguage string `json:"query_language"`
QueryPackURL string `json:"query_pack_url"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
ActionsWorkflowRunID int `json:"actions_workflow_run_id"`
Status string `json:"status"`
ScannedRepositories []ScannedRepo `json:"scanned_repositories"`
SkippedRepositories SkippedRepositories `json:"skipped_repositories"`
}

0
types/common.go Normal file
View File

0
types/logger.go Normal file
View File

0
types/queue.go Normal file
View File

0
types/runner.go Normal file
View File

0
types/storage.go Normal file
View File