diff --git a/api/src/main.go b/api/src/main.go index 0abd8b11a..076641162 100644 --- a/api/src/main.go +++ b/api/src/main.go @@ -2,7 +2,7 @@ package main import ( "github.com/gofiber/fiber/v2" - api "mizuserver" + "mizuserver/src/pkg/inserter" "mizuserver/src/pkg/middleware" "mizuserver/src/pkg/routes" "mizuserver/src/pkg/utils" @@ -10,9 +10,9 @@ import ( func main() { // TODO: to generate data - path := "/Users/roeegadot/Downloads/output2" - api.TestHarSavingFromFolder(path) - + //path := "/Users/roeegadot/Downloads/output2" + //api.TestHarSavingFromFolder(path) + go inserter.StartReadingFiles("/var/up9hars") app := fiber.New() middleware.FiberMiddleware(app) // Register Fiber's middleware for app. diff --git a/api/src/pkg/controllers/entries_controller.go b/api/src/pkg/controllers/entries_controller.go index 9cc5f0310..727717fd3 100644 --- a/api/src/pkg/controllers/entries_controller.go +++ b/api/src/pkg/controllers/entries_controller.go @@ -50,3 +50,14 @@ func GetEntry(c *fiber.Ctx) error { return c.Status(fiber.StatusOK).JSON(fullEntry) } + + +func DeleteAllEntries(c *fiber.Ctx) error { + database.GetEntriesTable(). + Where("1 = 1"). + Delete(&models.MizuEntry{}) + + return c.Status(fiber.StatusOK).JSON(fiber.Map{ + "msg": "Success", + }) +} \ No newline at end of file diff --git a/api/src/pkg/database/utils.go b/api/src/pkg/database/main.go similarity index 84% rename from api/src/pkg/database/utils.go rename to api/src/pkg/database/main.go index 4a52eedd0..39b8bec75 100644 --- a/api/src/pkg/database/utils.go +++ b/api/src/pkg/database/main.go @@ -11,10 +11,10 @@ const ( ) var ( - DB = initDataBase(DBPath) + DB = initDataBase(DBPath) ) -func GetEntriesTable() *gorm.DB{ +func GetEntriesTable() *gorm.DB { return DB.Table("mizu_entries") } diff --git a/api/src/pkg/inserter/main.go b/api/src/pkg/inserter/main.go new file mode 100644 index 000000000..7251b072d --- /dev/null +++ b/api/src/pkg/inserter/main.go @@ -0,0 +1,91 @@ +package inserter + +import ( + "bufio" + "encoding/json" + "fmt" + "github.com/google/martian/har" + "go.mongodb.org/mongo-driver/bson/primitive" + "io" + "io/fs" + "mizuserver/src/pkg/database" + "mizuserver/src/pkg/models" + "mizuserver/src/pkg/utils" + "net/url" + "os" + "path" + "sort" + "time" +) + + +func IsEmpty(name string) (bool) { + f, err := os.Open(name) + if err != nil { + return false + } + defer f.Close() + + _, err = f.Readdirnames(1) // Or f.Readdir(1) + if err == io.EOF { + return true + } + return false // Either not empty or error, suits both cases +} + +func StartReadingFiles(workingDir string) { + err := os.MkdirAll(workingDir, fs.ModeDir) + utils.CheckErr(err) + + for true { + if IsEmpty(workingDir) { + fmt.Printf("Waiting for new files\n") + time.Sleep(5 * time.Second) + continue + } + + dir, _ := os.Open(workingDir) + dirFiles, _ := dir.Readdir(-1) + sort.Sort(utils.ByModTime(dirFiles)) + + fileInfo := dirFiles[0] + inputFilePath := path.Join(workingDir, fileInfo.Name()) + file, err := os.Open(inputFilePath) + utils.CheckErr(err) + + var inputHar har.HAR + decErr := json.NewDecoder(bufio.NewReader(file)).Decode(&inputHar) + utils.CheckErr(decErr) + + for _, entry := range inputHar.Log.Entries { + fmt.Printf("Entry inserted") + SaveHarToDb(*entry, "") + } + rmErr := os.Remove(inputFilePath) + utils.CheckErr(rmErr) + } + +} + +func SaveHarToDb(entry har.Entry, source string) { + entryBytes, _ := json.Marshal(entry) + serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL) + mizuEntry := models.MizuEntry{ + EntryId: primitive.NewObjectID().Hex(), + Entry: string(entryBytes), // simple way to store it and not convert to bytes + Service: serviceName, + Url: entry.Request.URL, + Path: urlPath, + Method: entry.Request.Method, + Status: entry.Response.Status, + Source: source, + Timestamp: entry.StartedDateTime.Unix(), + } + database.GetEntriesTable().Create(&mizuEntry) +} + +func getServiceNameFromUrl(inputUrl string) (string, string) { + parsed, err := url.Parse(inputUrl) + utils.CheckErr(err) + return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host), parsed.Path +} diff --git a/api/src/pkg/models/mizuEntry.go b/api/src/pkg/models/mizuEntry.go index b3eb7c854..1c14e6adc 100644 --- a/api/src/pkg/models/mizuEntry.go +++ b/api/src/pkg/models/mizuEntry.go @@ -1,11 +1,11 @@ package models -import ( - "gorm.io/gorm" -) +import "time" type MizuEntry struct { - gorm.Model + ID uint `gorm:"primarykey"` + CreatedAt time.Time + UpdatedAt time.Time Entry string `json:"entry,omitempty" gorm:"column:entry"` EntryId string `json:"entryId" gorm:"column:entryId"` Url string `json:"url" gorm:"column:url"` diff --git a/api/src/pkg/routes/public_routes.go b/api/src/pkg/routes/public_routes.go index 04a4d6a1e..73e07bcca 100644 --- a/api/src/pkg/routes/public_routes.go +++ b/api/src/pkg/routes/public_routes.go @@ -11,4 +11,7 @@ func PublicRoutes(fiberApp *fiber.App) { routeGroup.Get("/entries", controllers.GetEntries) // get entries (base/thin entries) routeGroup.Get("/entries/:entryId", controllers.GetEntry) // get single (full) entry + + routeGroup.Get("/resetDB", controllers.DeleteAllEntries) // get single (full) entry + } diff --git a/api/testing_from_file.go b/api/testing_from_file.go index fff207e64..1b06a86a8 100644 --- a/api/testing_from_file.go +++ b/api/testing_from_file.go @@ -30,7 +30,7 @@ func TestHarSavingFromFolder(inputDir string) { utils.CheckErr(decErr) for _, entry := range inputHar.Log.Entries { - SaveHarToDb(*entry, "source") + SaveHarToDb(*entry, "") } } }