From da846da334c81a6a1adbde3fbbd0aacedf8810c5 Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Mon, 11 Oct 2021 13:09:23 +0300 Subject: [PATCH] api server support sync workspace (#340) --- agent/pkg/controllers/entries_controller.go | 36 +++++++++++++++++---- agent/pkg/models/models.go | 6 ++-- agent/pkg/up9/main.go | 26 +++++++++++---- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/agent/pkg/controllers/entries_controller.go b/agent/pkg/controllers/entries_controller.go index a7fe7a6dc..3d2f0438d 100644 --- a/agent/pkg/controllers/entries_controller.go +++ b/agent/pkg/controllers/entries_controller.go @@ -10,6 +10,7 @@ import ( "mizuserver/pkg/utils" "mizuserver/pkg/validation" "net/http" + "regexp" "time" "github.com/google/martian/har" @@ -72,23 +73,46 @@ func SyncEntries(c *gin.Context) { c.JSON(http.StatusBadRequest, err) return } + if err := validation.Validate(syncParams); err != nil { c.JSON(http.StatusBadRequest, err) return } + if up9.GetAnalyzeInfo().IsAnalyzing { c.String(http.StatusBadRequest, "Cannot analyze, mizu is already analyzing") return } - rlog.Infof("Sync entries - creating token. env %s\n", syncParams.Env) - token, err := up9.CreateAnonymousToken(syncParams.Env) - if err != nil { - c.String(http.StatusServiceUnavailable, "Cannot analyze, mizu is already analyzing") + var ( + token, model string + guestMode bool + ) + if syncParams.Token == "" { + rlog.Infof("Sync entries - creating token. env %s\n", syncParams.Env) + guestToken, err := up9.CreateAnonymousToken(syncParams.Env) + if err != nil { + c.String(http.StatusServiceUnavailable, "Failed creating anonymous token") + return + } + + token = guestToken.Token + model = guestToken.Model + guestMode = true + } else { + token = fmt.Sprintf("bearer %s", syncParams.Token) + model = syncParams.Workspace + guestMode = false + } + + modelRegex, _ := regexp.Compile("[A-Za-z0-9][-A-Za-z0-9_.]*[A-Za-z0-9]+$") + if len(model) > 63 || !modelRegex.MatchString(model) { + c.String(http.StatusBadRequest, "Invalid model name") return } - rlog.Infof("Sync entries - syncing. token: %s model: %s\n", token.Token, token.Model) - go up9.SyncEntriesImpl(token.Token, token.Model, syncParams.Env, syncParams.SleepIntervalSec) + + rlog.Infof("Sync entries - syncing. token: %s, model: %s, guest mode: %v\n", token, model, guestMode) + go up9.SyncEntriesImpl(token, model, syncParams.Env, syncParams.UploadIntervalSec, guestMode) c.String(http.StatusOK, "OK") } diff --git a/agent/pkg/models/models.go b/agent/pkg/models/models.go index 66416f6f8..884ba7da1 100644 --- a/agent/pkg/models/models.go +++ b/agent/pkg/models/models.go @@ -23,8 +23,10 @@ type EntriesFilter struct { } type SyncEntriesRequestQuery struct { - Env string `form:"env"` - SleepIntervalSec int `form:"interval"` + Token string `form:"token"` + Env string `form:"env"` + Workspace string `form:"workspace"` + UploadIntervalSec int `form:"interval"` } type HarFetchRequestQuery struct { diff --git a/agent/pkg/up9/main.go b/agent/pkg/up9/main.go index 3633c4217..9632928e8 100644 --- a/agent/pkg/up9/main.go +++ b/agent/pkg/up9/main.go @@ -59,14 +59,16 @@ func GetRemoteUrl(analyzeDestination string, analyzeToken string) string { return fmt.Sprintf("https://%s/share/%s", analyzeDestination, analyzeToken) } -func CheckIfModelReady(analyzeDestination string, analyzeModel string, analyzeToken string) bool { +func CheckIfModelReady(analyzeDestination string, analyzeModel string, analyzeToken string, guestMode bool) bool { statusUrl, _ := url.Parse(fmt.Sprintf("https://trcc.%s/models/%s/status", analyzeDestination, analyzeModel)) + + authHeader := getAuthHeader(guestMode) req := &http.Request{ Method: http.MethodGet, URL: statusUrl, Header: map[string][]string{ "Content-Type": {"application/json"}, - "Guest-Auth": {analyzeToken}, + authHeader: {analyzeToken}, }, } statusResp, err := http.DefaultClient.Do(req) @@ -81,6 +83,14 @@ func CheckIfModelReady(analyzeDestination string, analyzeModel string, analyzeTo return target.LastMajorGeneration > 0 } +func getAuthHeader(guestMode bool) string { + if guestMode { + return "Guest-Auth" + } + + return "Authorization" +} + func GetTrafficDumpUrl(analyzeDestination string, analyzeModel string) *url.URL { strUrl := fmt.Sprintf("https://traffic.%s/dumpTrafficBulk/%s", analyzeDestination, analyzeModel) if strings.HasPrefix(analyzeDestination, "http") { @@ -92,6 +102,7 @@ func GetTrafficDumpUrl(analyzeDestination string, analyzeModel string) *url.URL type AnalyzeInformation struct { IsAnalyzing bool + GuestMode bool SentCount int AnalyzedModel string AnalyzeToken string @@ -100,6 +111,7 @@ type AnalyzeInformation struct { func (info *AnalyzeInformation) Reset() { info.IsAnalyzing = false + info.GuestMode = true info.AnalyzedModel = "" info.AnalyzeToken = "" info.AnalyzeDestination = "" @@ -112,19 +124,20 @@ func GetAnalyzeInfo() *shared.AnalyzeStatus { return &shared.AnalyzeStatus{ IsAnalyzing: analyzeInformation.IsAnalyzing, RemoteUrl: GetRemoteUrl(analyzeInformation.AnalyzeDestination, analyzeInformation.AnalyzeToken), - IsRemoteReady: CheckIfModelReady(analyzeInformation.AnalyzeDestination, analyzeInformation.AnalyzedModel, analyzeInformation.AnalyzeToken), + IsRemoteReady: CheckIfModelReady(analyzeInformation.AnalyzeDestination, analyzeInformation.AnalyzedModel, analyzeInformation.AnalyzeToken, analyzeInformation.GuestMode), SentCount: analyzeInformation.SentCount, } } -func SyncEntriesImpl(token string, model string, envPrefix string, sleepIntervalSec int) { +func SyncEntriesImpl(token string, model string, envPrefix string, uploadIntervalSec int, guestMode bool) { analyzeInformation.IsAnalyzing = true + analyzeInformation.GuestMode = guestMode analyzeInformation.AnalyzedModel = model analyzeInformation.AnalyzeToken = token analyzeInformation.AnalyzeDestination = envPrefix analyzeInformation.SentCount = 0 - sleepTime := time.Second * time.Duration(sleepIntervalSec) + sleepTime := time.Second * time.Duration(uploadIntervalSec) var timestampFrom int64 = 0 @@ -170,13 +183,14 @@ func SyncEntriesImpl(token string, model string, envPrefix string, sleepInterval _ = w.Close() reqBody := ioutil.NopCloser(bytes.NewReader(in.Bytes())) + authHeader := getAuthHeader(guestMode) req := &http.Request{ Method: http.MethodPost, URL: GetTrafficDumpUrl(envPrefix, model), Header: map[string][]string{ "Content-Encoding": {"deflate"}, "Content-Type": {"application/octet-stream"}, - "Guest-Auth": {token}, + authHeader: {token}, }, Body: reqBody, }