wip: make server compile post-merge
This commit is contained in:
committed by
=Michael Hohn
parent
02acf3eeaf
commit
1633245444
@@ -8,6 +8,7 @@ import (
|
|||||||
"log"
|
"log"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
"mrvacommander/config/mcc"
|
"mrvacommander/config/mcc"
|
||||||
|
|
||||||
@@ -90,7 +91,19 @@ func main() {
|
|||||||
|
|
||||||
case "container":
|
case "container":
|
||||||
// TODO: take value from configuration
|
// TODO: take value from configuration
|
||||||
sq, err := queue.NewRabbitMQQueue(2)
|
|
||||||
|
rmqHost := os.Getenv("MRVA_RABBITMQ_HOST")
|
||||||
|
rmqPort := os.Getenv("MRVA_RABBITMQ_PORT")
|
||||||
|
rmqUser := os.Getenv("MRVA_RABBITMQ_USER")
|
||||||
|
rmqPass := os.Getenv("MRVA_RABBITMQ_PASSWORD")
|
||||||
|
|
||||||
|
rmqPortAsInt, err := strconv.ParseInt(rmqPort, 10, 16)
|
||||||
|
if err != nil {
|
||||||
|
slog.Error("Failed to parse RabbitMQ port", slog.Any("error", err))
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
sq, err := queue.NewRabbitMQQueue(rmqHost, int16(rmqPortAsInt), rmqUser, rmqPass, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Error("Unable to initialize RabbitMQ queue")
|
slog.Error("Unable to initialize RabbitMQ queue")
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
|
|||||||
@@ -102,7 +102,7 @@ func RunAnalysisJob(job common.AnalyzeJob) (common.AnalyzeResult, error) {
|
|||||||
result = common.AnalyzeResult{
|
result = common.AnalyzeResult{
|
||||||
RequestId: job.RequestId,
|
RequestId: job.RequestId,
|
||||||
ResultCount: runResult.ResultCount,
|
ResultCount: runResult.ResultCount,
|
||||||
ResultLocation: "REPLACE_THIS_WITH_STORED_RESULTS_ARCHIVE", // TODO
|
ResultLocation: artifactstore.ArtifactLocation{}, // TODO "REPLACE_THIS_WITH_STORED_RESULTS_ARCHIVE"
|
||||||
Status: common.StatusSuccess,
|
Status: common.StatusSuccess,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import (
|
|||||||
"mrvacommander/pkg/artifactstore"
|
"mrvacommander/pkg/artifactstore"
|
||||||
"mrvacommander/pkg/common"
|
"mrvacommander/pkg/common"
|
||||||
"mrvacommander/pkg/qldbstore"
|
"mrvacommander/pkg/qldbstore"
|
||||||
|
"mrvacommander/pkg/state"
|
||||||
|
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
)
|
)
|
||||||
@@ -265,8 +266,7 @@ func (c *CommanderSingle) MRVADownloadServe(w http.ResponseWriter, r *http.Reque
|
|||||||
func FileDownload(w http.ResponseWriter, path string) {
|
func FileDownload(w http.ResponseWriter, path string) {
|
||||||
slog.Debug("Sending zip file with .sarif/.bqrs", "path", path)
|
slog.Debug("Sending zip file with .sarif/.bqrs", "path", path)
|
||||||
|
|
||||||
// TODO: @hohn
|
fpath, res, err := state.ResultAsFile(path)
|
||||||
fpath, res, err := ResultAsFile(path)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, "Failed to read results", http.StatusInternalServerError)
|
http.Error(w, "Failed to read results", http.StatusInternalServerError)
|
||||||
return
|
return
|
||||||
@@ -327,7 +327,7 @@ func (c *CommanderSingle) MRVARequest(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
slog.Debug("Forming and sending response for submitted analysis job", "id", si.ID)
|
slog.Debug("Forming and sending response for submitted analysis job", "id", si.ID)
|
||||||
submit_response, err := submitResponse(si)
|
submit_response, err := c.submitResponse(si)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
return
|
return
|
||||||
@@ -346,7 +346,24 @@ func nwoToNwoStringArray(nwo []common.NameWithOwner) ([]string, int) {
|
|||||||
return repos, count
|
return repos, count
|
||||||
}
|
}
|
||||||
|
|
||||||
func submitResponse(si SessionInfo) ([]byte, error) {
|
func nwoToDummyRepositoryArray(nwo []common.NameWithOwner) ([]common.Repository, int) {
|
||||||
|
repos := []common.Repository{}
|
||||||
|
for _, repo := range nwo {
|
||||||
|
repos = append(repos, common.Repository{
|
||||||
|
ID: 0,
|
||||||
|
Name: repo.Repo,
|
||||||
|
FullName: fmt.Sprintf("%s/%s", repo.Owner, repo.Repo),
|
||||||
|
Private: false,
|
||||||
|
StargazersCount: 0,
|
||||||
|
UpdatedAt: time.Now().Format(time.RFC3339),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
count := len(nwo)
|
||||||
|
|
||||||
|
return repos, count
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *CommanderSingle) submitResponse(si SessionInfo) ([]byte, error) {
|
||||||
// Construct the response bottom-up
|
// Construct the response bottom-up
|
||||||
var m_cr common.ControllerRepo
|
var m_cr common.ControllerRepo
|
||||||
var m_ac common.Actor
|
var m_ac common.Actor
|
||||||
@@ -354,15 +371,15 @@ func submitResponse(si SessionInfo) ([]byte, error) {
|
|||||||
repos, count := nwoToNwoStringArray(si.NotFoundRepos)
|
repos, count := nwoToNwoStringArray(si.NotFoundRepos)
|
||||||
r_nfr := common.NotFoundRepos{RepositoryCount: count, RepositoryFullNames: repos}
|
r_nfr := common.NotFoundRepos{RepositoryCount: count, RepositoryFullNames: repos}
|
||||||
|
|
||||||
repos, count = nwoToNwoStringArray(si.AccessMismatchRepos)
|
ra, rac := nwoToDummyRepositoryArray(si.AccessMismatchRepos)
|
||||||
r_amr := common.AccessMismatchRepos{RepositoryCount: count, Repositories: repos}
|
r_amr := common.AccessMismatchRepos{RepositoryCount: rac, Repositories: ra}
|
||||||
|
|
||||||
repos, count = nwoToNwoStringArray(si.NoCodeqlDBRepos)
|
ra, rac = nwoToDummyRepositoryArray(si.NoCodeqlDBRepos)
|
||||||
r_ncd := common.NoCodeqlDBRepos{RepositoryCount: count, Repositories: repos}
|
r_ncd := common.NoCodeqlDBRepos{RepositoryCount: rac, Repositories: ra}
|
||||||
|
|
||||||
// TODO fill these with real values?
|
// TODO fill these with real values?
|
||||||
repos, count = nwoToNwoStringArray(si.NoCodeqlDBRepos)
|
ra, rac = nwoToDummyRepositoryArray(si.NoCodeqlDBRepos)
|
||||||
r_olr := common.OverLimitRepos{RepositoryCount: count, Repositories: repos}
|
r_olr := common.OverLimitRepos{RepositoryCount: rac, Repositories: ra}
|
||||||
|
|
||||||
m_skip := common.SkippedRepositories{
|
m_skip := common.SkippedRepositories{
|
||||||
AccessMismatchRepos: r_amr,
|
AccessMismatchRepos: r_amr,
|
||||||
@@ -375,7 +392,8 @@ func submitResponse(si SessionInfo) ([]byte, error) {
|
|||||||
ControllerRepo: m_cr,
|
ControllerRepo: m_cr,
|
||||||
ID: si.ID,
|
ID: si.ID,
|
||||||
QueryLanguage: si.Language,
|
QueryLanguage: si.Language,
|
||||||
QueryPackURL: si.QueryPack,
|
// TODO: broken, need proper URL using si.data
|
||||||
|
QueryPackURL: "broken-for-now",
|
||||||
CreatedAt: time.Now().Format(time.RFC3339),
|
CreatedAt: time.Now().Format(time.RFC3339),
|
||||||
UpdatedAt: time.Now().Format(time.RFC3339),
|
UpdatedAt: time.Now().Format(time.RFC3339),
|
||||||
Status: "in_progress",
|
Status: "in_progress",
|
||||||
@@ -383,10 +401,12 @@ func submitResponse(si SessionInfo) ([]byte, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Store data needed later
|
// Store data needed later
|
||||||
joblist := storage.GetJobList(si.ID)
|
// joblist := state.GetJobList(si.ID)
|
||||||
|
// (si.JobID)?
|
||||||
|
joblist := c.v.State.GetJobList(si.ID)
|
||||||
|
|
||||||
for _, job := range joblist {
|
for _, job := range joblist {
|
||||||
storage.SetJobInfo(common.JobSpec{
|
c.v.State.SetJobInfo(common.JobSpec{
|
||||||
JobID: si.ID,
|
JobID: si.ID,
|
||||||
NameWithOwner: job.NWO,
|
NameWithOwner: job.NWO,
|
||||||
}, common.JobInfo{
|
}, common.JobInfo{
|
||||||
@@ -401,35 +421,35 @@ func submitResponse(si SessionInfo) ([]byte, error) {
|
|||||||
// Encode the response as JSON
|
// Encode the response as JSON
|
||||||
submit_response, err := json.Marshal(m_sr)
|
submit_response, err := json.Marshal(m_sr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Warn("Error encoding response as JSON:", err)
|
slog.Warn("Error encoding response as JSON:", err.Error())
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return submit_response, nil
|
return submit_response, nil
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *CommanderSingle) collectRequestInfo(w http.ResponseWriter, r *http.Request, sessionId int) (string, []common.NameWithOwner, string, error) {
|
func (c *CommanderSingle) collectRequestInfo(w http.ResponseWriter, r *http.Request, sessionId int) (string, []common.NameWithOwner, artifactstore.ArtifactLocation, error) {
|
||||||
slog.Debug("Collecting session info")
|
slog.Debug("Collecting session info")
|
||||||
|
|
||||||
if r.Body == nil {
|
if r.Body == nil {
|
||||||
err := errors.New("missing request body")
|
err := errors.New("missing request body")
|
||||||
log.Println(err)
|
log.Println(err)
|
||||||
http.Error(w, err.Error(), http.StatusNoContent)
|
http.Error(w, err.Error(), http.StatusNoContent)
|
||||||
return "", []common.NameWithOwner{}, "", err
|
return "", []common.NameWithOwner{}, artifactstore.ArtifactLocation{}, err
|
||||||
}
|
}
|
||||||
buf, err := io.ReadAll(r.Body)
|
buf, err := io.ReadAll(r.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
var w http.ResponseWriter
|
var w http.ResponseWriter
|
||||||
slog.Error("Error reading MRVA submission body", "error", err.Error())
|
slog.Error("Error reading MRVA submission body", "error", err.Error())
|
||||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
return "", []common.NameWithOwner{}, "", err
|
return "", []common.NameWithOwner{}, artifactstore.ArtifactLocation{}, err
|
||||||
}
|
}
|
||||||
msg, err := TrySubmitMsg(buf)
|
msg, err := TrySubmitMsg(buf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Unknown message
|
// Unknown message
|
||||||
slog.Error("Unknown MRVA submission body format")
|
slog.Error("Unknown MRVA submission body format")
|
||||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
return "", []common.NameWithOwner{}, "", err
|
return "", []common.NameWithOwner{}, artifactstore.ArtifactLocation{}, err
|
||||||
}
|
}
|
||||||
// Decompose the SubmitMsg and keep information
|
// Decompose the SubmitMsg and keep information
|
||||||
|
|
||||||
@@ -438,12 +458,12 @@ func (c *CommanderSingle) collectRequestInfo(w http.ResponseWriter, r *http.Requ
|
|||||||
slog.Error("MRVA submission body querypack has invalid format")
|
slog.Error("MRVA submission body querypack has invalid format")
|
||||||
err := errors.New("MRVA submission body querypack has invalid format")
|
err := errors.New("MRVA submission body querypack has invalid format")
|
||||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
return "", []common.NameWithOwner{}, "", err
|
return "", []common.NameWithOwner{}, artifactstore.ArtifactLocation{}, err
|
||||||
}
|
}
|
||||||
session_tgz_ref, err := c.processQueryPackArchive(msg.QueryPack, sessionId)
|
session_tgz_ref, err := c.processQueryPackArchive(msg.QueryPack, sessionId)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
return "", []common.NameWithOwner{}, "", err
|
return "", []common.NameWithOwner{}, artifactstore.ArtifactLocation{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Save the language
|
// 2. Save the language
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ type SessionInfo struct {
|
|||||||
ID int
|
ID int
|
||||||
Owner string
|
Owner string
|
||||||
ControllerRepo string
|
ControllerRepo string
|
||||||
QueryPack string
|
QueryPack artifactstore.ArtifactLocation
|
||||||
Language string
|
Language string
|
||||||
Repositories []common.NameWithOwner
|
Repositories []common.NameWithOwner
|
||||||
AccessMismatchRepos []common.NameWithOwner
|
AccessMismatchRepos []common.NameWithOwner
|
||||||
@@ -34,7 +34,7 @@ func NewCommanderSingle(st *Visibles) *CommanderSingle {
|
|||||||
|
|
||||||
type Visibles struct {
|
type Visibles struct {
|
||||||
Queue queue.Queue
|
Queue queue.Queue
|
||||||
State state.ServerState
|
State *state.LocalState
|
||||||
Artifacts artifactstore.ArtifactStore
|
Artifacts artifactstore.ArtifactStore
|
||||||
CodeQLDBStore qldbstore.CodeQLDatabaseStore
|
CodeQLDBStore qldbstore.CodeQLDatabaseStore
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
package state
|
package state
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"mrvacommander/pkg/common"
|
"mrvacommander/pkg/common"
|
||||||
@@ -84,3 +87,19 @@ func (s *LocalState) AddJob(jobID int, job common.AnalyzeJob) {
|
|||||||
s.jobs[jobID] = append(s.jobs[jobID], job)
|
s.jobs[jobID] = append(s.jobs[jobID], job)
|
||||||
s.mutex.Unlock()
|
s.mutex.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO: @hohn
|
||||||
|
func ResultAsFile(path string) (string, []byte, error) {
|
||||||
|
fpath := path
|
||||||
|
if !filepath.IsAbs(path) {
|
||||||
|
fpath = "/" + path
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := os.ReadFile(fpath)
|
||||||
|
if err != nil {
|
||||||
|
slog.Warn("Failed to read results file", fpath, err)
|
||||||
|
return "", nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return fpath, file, nil
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user