From 485bc7fd2b22fbe364ab6758d9e047cfa0ec2aac Mon Sep 17 00:00:00 2001 From: gadotroee <55343099+gadotroee@users.noreply.github.com> Date: Thu, 17 Jun 2021 16:48:05 +0300 Subject: [PATCH] Add export entries endpoint for better up9 connect funcionality (#72) * no message * no message * no message --- api/pkg/controllers/entries_controller.go | 61 ++++++++++++++++++++++- api/pkg/models/models.go | 27 +++++----- api/pkg/routes/public_routes.go | 3 ++ cli/cmd/fetch.go | 10 ++-- cli/cmd/fetchRunner.go | 10 ++-- 5 files changed, 86 insertions(+), 25 deletions(-) diff --git a/api/pkg/controllers/entries_controller.go b/api/pkg/controllers/entries_controller.go index 06b96b5b8..282271b65 100644 --- a/api/pkg/controllers/entries_controller.go +++ b/api/pkg/controllers/entries_controller.go @@ -9,6 +9,7 @@ import ( "mizuserver/pkg/models" "mizuserver/pkg/utils" "mizuserver/pkg/validation" + "time" ) const ( @@ -75,11 +76,23 @@ func GetHARs(c *fiber.Ctx) error { return c.Status(fiber.StatusBadRequest).JSON(err) } + var timestampFrom, timestampTo int64 + + if entriesFilter.From < 0 { + timestampFrom = 0 + } else { + timestampFrom = entriesFilter.From + } + if entriesFilter.To <= 0 { + timestampTo = time.Now().UnixNano() / int64(time.Millisecond) + } else { + timestampTo = entriesFilter.To + } + var entries []models.MizuEntry database.GetEntriesTable(). + Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)). Order(fmt.Sprintf("timestamp %s", order)). - // Where(fmt.Sprintf("timestamp %s %v", operatorSymbol, entriesFilter.Timestamp)). - Limit(1000). Find(&entries) if len(entries) > 0 { @@ -125,6 +138,50 @@ func GetHARs(c *fiber.Ctx) error { return c.Status(fiber.StatusOK).SendStream(buffer) } +func GetFullEntries(c *fiber.Ctx) error { + entriesFilter := &models.HarFetchRequestBody{} + order := OrderDesc + if err := c.QueryParser(entriesFilter); err != nil { + return c.Status(fiber.StatusBadRequest).JSON(err) + } + err := validation.Validate(entriesFilter) + if err != nil { + return c.Status(fiber.StatusBadRequest).JSON(err) + } + + var timestampFrom, timestampTo int64 + + if entriesFilter.From < 0 { + timestampFrom = 0 + } else { + timestampFrom = entriesFilter.From + } + if entriesFilter.To <= 0 { + timestampTo = time.Now().UnixNano() / int64(time.Millisecond) + } else { + timestampTo = entriesFilter.To + } + + var entries []models.MizuEntry + database.GetEntriesTable(). + Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)). + Order(fmt.Sprintf("timestamp %s", order)). + Find(&entries) + + if len(entries) > 0 { + // the entries always order from oldest to newest so we should revers + utils.ReverseSlice(entries) + } + + entriesArray := make([]har.Entry, 0) + for _, entryData := range entries { + var harEntry har.Entry + _ = json.Unmarshal([]byte(entryData.Entry), &harEntry) + entriesArray = append(entriesArray, harEntry) + } + return c.Status(fiber.StatusOK).JSON(entriesArray) +} + func GetEntry(c *fiber.Ctx) error { var entryData models.EntryData database.GetEntriesTable(). diff --git a/api/pkg/models/models.go b/api/pkg/models/models.go index d8dc90184..5cde7622c 100644 --- a/api/pkg/models/models.go +++ b/api/pkg/models/models.go @@ -12,15 +12,15 @@ type MizuEntry struct { ID uint `gorm:"primarykey"` CreatedAt time.Time UpdatedAt time.Time - Entry string `json:"entry,omitempty" gorm:"column:entry"` - EntryId string `json:"entryId" gorm:"column:entryId"` - Url string `json:"url" gorm:"column:url"` - Method string `json:"method" gorm:"column:method"` - Status int `json:"status" gorm:"column:status"` - RequestSenderIp string `json:"requestSenderIp" gorm:"column:requestSenderIp"` - Service string `json:"service" gorm:"column:service"` - Timestamp int64 `json:"timestamp" gorm:"column:timestamp"` - Path string `json:"path" gorm:"column:path"` + Entry string `json:"entry,omitempty" gorm:"column:entry"` + EntryId string `json:"entryId" gorm:"column:entryId"` + Url string `json:"url" gorm:"column:url"` + Method string `json:"method" gorm:"column:method"` + Status int `json:"status" gorm:"column:status"` + RequestSenderIp string `json:"requestSenderIp" gorm:"column:requestSenderIp"` + Service string `json:"service" gorm:"column:service"` + Timestamp int64 `json:"timestamp" gorm:"column:timestamp"` + Path string `json:"path" gorm:"column:path"` ResolvedSource string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"` ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` } @@ -37,7 +37,7 @@ type BaseEntryDetails struct { } type EntryData struct { - Entry string `json:"entry,omitempty"` + Entry string `json:"entry,omitempty"` ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` } @@ -48,7 +48,8 @@ type EntriesFilter struct { } type HarFetchRequestBody struct { - Limit int `query:"limit"` + From int64 `query:"from"` + To int64 `query:"to"` } type WebSocketEntryMessage struct { @@ -56,7 +57,6 @@ type WebSocketEntryMessage struct { Data *BaseEntryDetails `json:"data,omitempty"` } - type WebSocketTappedEntryMessage struct { *shared.WebSocketMessageMetadata Data *tap.OutputChannelItem @@ -82,7 +82,6 @@ func CreateWebsocketTappedEntryMessage(base *tap.OutputChannelItem) ([]byte, err return json.Marshal(message) } - // ExtendedHAR is the top level object of a HAR log. type ExtendedHAR struct { Log *ExtendedLog `json:"log"` @@ -100,5 +99,5 @@ type ExtendedLog struct { type ExtendedCreator struct { *har.Creator - Source string `json:"_source"` + Source string `json:"_source"` } diff --git a/api/pkg/routes/public_routes.go b/api/pkg/routes/public_routes.go index 042ea54c9..df589a62c 100644 --- a/api/pkg/routes/public_routes.go +++ b/api/pkg/routes/public_routes.go @@ -11,8 +11,11 @@ func EntriesRoutes(fiberApp *fiber.App) { routeGroup.Get("/entries", controllers.GetEntries) // get entries (base/thin entries) routeGroup.Get("/entries/:entryId", controllers.GetEntry) // get single (full) entry + routeGroup.Get("/exportEntries", controllers.GetFullEntries) + routeGroup.Get("/har", controllers.GetHARs) + routeGroup.Get("/resetDB", controllers.DeleteAllEntries) // get single (full) entry routeGroup.Get("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB diff --git a/cli/cmd/fetch.go b/cli/cmd/fetch.go index c4e8fb913..9e55f08fa 100644 --- a/cli/cmd/fetch.go +++ b/cli/cmd/fetch.go @@ -5,8 +5,10 @@ import ( ) type MizuFetchOptions struct { - Limit uint16 - Directory string + FromTimestamp int64 + ToTimestamp int64 + Directory string + MizuPort uint } var mizuFetchOptions = MizuFetchOptions{} @@ -23,6 +25,8 @@ var fetchCmd = &cobra.Command{ func init() { rootCmd.AddCommand(fetchCmd) - fetchCmd.Flags().Uint16VarP(&mizuFetchOptions.Limit, "limit", "l", 1000, "Provide a custom limit for entries to fetch") fetchCmd.Flags().StringVarP(&mizuFetchOptions.Directory, "directory", "d", ".", "Provide a custom directory for fetched entries") + fetchCmd.Flags().Int64Var(&mizuFetchOptions.FromTimestamp, "from", 0, "Custom start timestamp for fetched entries") + fetchCmd.Flags().Int64Var(&mizuFetchOptions.ToTimestamp, "to", 0, "Custom end timestamp fetched entries") + fetchCmd.Flags().UintVarP(&mizuFetchOptions.MizuPort, "port", "p", 8899, "Custom port for mizu") } diff --git a/cli/cmd/fetchRunner.go b/cli/cmd/fetchRunner.go index 71db8364d..58db0352b 100644 --- a/cli/cmd/fetchRunner.go +++ b/cli/cmd/fetchRunner.go @@ -14,7 +14,7 @@ import ( ) func RunMizuFetch(fetch *MizuFetchOptions) { - resp, err := http.Get(fmt.Sprintf("http://localhost:8899/api/har?limit=%v", fetch.Limit)) + resp, err := http.Get(fmt.Sprintf("http://localhost:%v/api/har?from=%v&to=%v", fetch.MizuPort, fetch.FromTimestamp, fetch.ToTimestamp)) if err != nil { log.Fatal(err) } @@ -53,7 +53,7 @@ func Unzip(reader *zip.Reader, dest string) error { path := filepath.Join(dest, f.Name) // Check for ZipSlip (Directory traversal) - if !strings.HasPrefix(path, filepath.Clean(dest) + string(os.PathSeparator)) { + if !strings.HasPrefix(path, filepath.Clean(dest)+string(os.PathSeparator)) { return fmt.Errorf("illegal file path: %s", path) } @@ -61,7 +61,7 @@ func Unzip(reader *zip.Reader, dest string) error { _ = os.MkdirAll(path, f.Mode()) } else { _ = os.MkdirAll(filepath.Dir(path), f.Mode()) - fmt.Print("writing HAR file [ ", path, " ] .. ") + fmt.Print("writing HAR file [ ", path, " ] .. ") f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) if err != nil { return err @@ -70,7 +70,7 @@ func Unzip(reader *zip.Reader, dest string) error { if err := f.Close(); err != nil { panic(err) } - fmt.Println(" done") + fmt.Println(" done") }() _, err = io.Copy(f, rc) @@ -90,5 +90,3 @@ func Unzip(reader *zip.Reader, dest string) error { return nil } - -