package controllers import ( "encoding/json" "fmt" "github.com/gofiber/fiber/v2" "github.com/google/martian/har" "github.com/romana/rlog" "mizuserver/pkg/database" "mizuserver/pkg/models" "mizuserver/pkg/up9" "mizuserver/pkg/utils" "mizuserver/pkg/validation" "strings" "time" ) func GetEntries(c *fiber.Ctx) error { entriesFilter := &models.EntriesFilter{} if err := c.QueryParser(entriesFilter); err != nil { return c.Status(fiber.StatusBadRequest).JSON(err) } err := validation.Validate(entriesFilter) if err != nil { return c.Status(fiber.StatusBadRequest).JSON(err) } order := database.OperatorToOrderMapping[entriesFilter.Operator] operatorSymbol := database.OperatorToSymbolMapping[entriesFilter.Operator] var entries []models.MizuEntry database.GetEntriesTable(). Order(fmt.Sprintf("timestamp %s", order)). Where(fmt.Sprintf("timestamp %s %v", operatorSymbol, entriesFilter.Timestamp)). Omit("entry"). // remove the "big" entry field Limit(entriesFilter.Limit). Find(&entries) if len(entries) > 0 && order == database.OrderDesc { // the entries always order from oldest to newest so we should revers utils.ReverseSlice(entries) } baseEntries := make([]models.BaseEntryDetails, 0) for _, data := range entries { harEntry := models.BaseEntryDetails{} if err := models.GetEntry(&data, &harEntry); err != nil { continue } baseEntries = append(baseEntries, harEntry) } return c.Status(fiber.StatusOK).JSON(baseEntries) } func GetHARs(c *fiber.Ctx) error { entriesFilter := &models.HarFetchRequestBody{} order := database.OrderDesc if err := c.QueryParser(entriesFilter); err != nil { return c.Status(fiber.StatusBadRequest).JSON(err) } err := validation.Validate(entriesFilter) if err != nil { return c.Status(fiber.StatusBadRequest).JSON(err) } var timestampFrom, timestampTo int64 if entriesFilter.From < 0 { timestampFrom = 0 } else { timestampFrom = entriesFilter.From } if entriesFilter.To <= 0 { timestampTo = time.Now().UnixNano() / int64(time.Millisecond) } else { timestampTo = entriesFilter.To } var entries []models.MizuEntry database.GetEntriesTable(). Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)). Order(fmt.Sprintf("timestamp %s", order)). Find(&entries) if len(entries) > 0 { // the entries always order from oldest to newest so we should revers utils.ReverseSlice(entries) } harsObject := map[string]*models.ExtendedHAR{} for _, entryData := range entries { var harEntry har.Entry _ = json.Unmarshal([]byte(entryData.Entry), &harEntry) if entryData.ResolvedDestination != "" { harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entryData.ResolvedDestination) } var fileName string sourceOfEntry := entryData.ResolvedSource if sourceOfEntry != "" { // naively assumes the proper service source is http sourceOfEntry = fmt.Sprintf("http://%s", sourceOfEntry) //replace / from the file name cause they end up creating a corrupted folder fileName = fmt.Sprintf("%s.har", strings.ReplaceAll(sourceOfEntry, "/", "_")) } else { fileName = "unknown_source.har" } if harOfSource, ok := harsObject[fileName]; ok { harOfSource.Log.Entries = append(harOfSource.Log.Entries, &harEntry) } else { var entriesHar []*har.Entry entriesHar = append(entriesHar, &harEntry) harsObject[fileName] = &models.ExtendedHAR{ Log: &models.ExtendedLog{ Version: "1.2", Creator: &models.ExtendedCreator{ Creator: &har.Creator{ Name: "mizu", Version: "0.0.2", }, }, Entries: entriesHar, }, } // leave undefined when no source is present, otherwise modeler assumes source is empty string "" if sourceOfEntry != "" { harsObject[fileName].Log.Creator.Source = &sourceOfEntry } } } retObj := map[string][]byte{} for k, v := range harsObject { bytesData, _ := json.Marshal(v) retObj[k] = bytesData } buffer := utils.ZipData(retObj) return c.Status(fiber.StatusOK).SendStream(buffer) } func UploadEntries(c *fiber.Ctx) error { rlog.Infof("Upload entries - started\n") uploadRequestBody := &models.UploadEntriesRequestBody{} if err := c.QueryParser(uploadRequestBody); err != nil { return c.Status(fiber.StatusBadRequest).JSON(err) } if err := validation.Validate(uploadRequestBody); err != nil { return c.Status(fiber.StatusBadRequest).JSON(err) } if up9.GetAnalyzeInfo().IsAnalyzing { return c.Status(fiber.StatusBadRequest).SendString("Cannot analyze, mizu is already analyzing") } rlog.Infof("Upload entries - creating token. dest %s\n", uploadRequestBody.Dest) token, err := up9.CreateAnonymousToken(uploadRequestBody.Dest) if err != nil { return c.Status(fiber.StatusServiceUnavailable).SendString("Can't get token") } rlog.Infof("Upload entries - uploading. token: %s model: %s\n", token.Token, token.Model) go up9.UploadEntriesImpl(token.Token, token.Model, uploadRequestBody.Dest, uploadRequestBody.SleepIntervalSec) return c.Status(fiber.StatusOK).SendString("OK") } func GetFullEntries(c *fiber.Ctx) error { entriesFilter := &models.HarFetchRequestBody{} if err := c.QueryParser(entriesFilter); err != nil { return c.Status(fiber.StatusBadRequest).JSON(err) } err := validation.Validate(entriesFilter) if err != nil { return c.Status(fiber.StatusBadRequest).JSON(err) } var timestampFrom, timestampTo int64 if entriesFilter.From < 0 { timestampFrom = 0 } else { timestampFrom = entriesFilter.From } if entriesFilter.To <= 0 { timestampTo = time.Now().UnixNano() / int64(time.Millisecond) } else { timestampTo = entriesFilter.To } entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo) result := make([]models.FullEntryDetails, 0) for _, data := range entriesArray { harEntry := models.FullEntryDetails{} if err := models.GetEntry(&data, &harEntry); err != nil { continue } result = append(result, harEntry) } return c.Status(fiber.StatusOK).JSON(result) } func GetEntry(c *fiber.Ctx) error { var entryData models.MizuEntry database.GetEntriesTable(). Where(map[string]string{"entryId": c.Params("entryId")}). First(&entryData) fullEntry := models.FullEntryDetails{} if err := models.GetEntry(&entryData, &fullEntry); err != nil { return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{ "error": true, "msg": "Can't get entry details", }) } return c.Status(fiber.StatusOK).JSON(fullEntry) } func DeleteAllEntries(c *fiber.Ctx) error { database.GetEntriesTable(). Where("1 = 1"). Delete(&models.MizuEntry{}) return c.Status(fiber.StatusOK).JSON(fiber.Map{ "msg": "Success", }) } func GetGeneralStats(c *fiber.Ctx) error { sqlQuery := "SELECT count(*) as count, min(timestamp) as min, max(timestamp) as max from mizu_entries" var result struct { Count int Min int Max int } database.GetEntriesTable().Raw(sqlQuery).Scan(&result) return c.Status(fiber.StatusOK).JSON(&result) }