no message

api/src/main.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"github.com/gofiber/fiber/v2"
-	api "mizuserver"
+	"mizuserver/src/pkg/inserter"
 	"mizuserver/src/pkg/middleware"
 	"mizuserver/src/pkg/routes"
 	"mizuserver/src/pkg/utils"
@@ -10,9 +10,9 @@ import (
 
 func main() {
 	// TODO: to generate data
-	path := "/Users/roeegadot/Downloads/output2"
+	//path := "/Users/roeegadot/Downloads/output2"
-	api.TestHarSavingFromFolder(path)
+	//api.TestHarSavingFromFolder(path)
-
+	go inserter.StartReadingFiles("/var/up9hars")
 	app := fiber.New()
 
 	middleware.FiberMiddleware(app) // Register Fiber's middleware for app.

api/src/pkg/controllers/entries_controller.go

@@ -50,3 +50,14 @@ func GetEntry(c *fiber.Ctx) error {
 
 	return c.Status(fiber.StatusOK).JSON(fullEntry)
 }
+
+
+func DeleteAllEntries(c *fiber.Ctx) error {
+	database.GetEntriesTable().
+		Where("1 = 1").
+		Delete(&models.MizuEntry{})
+
+	return c.Status(fiber.StatusOK).JSON(fiber.Map{
+		"msg": "Success",
+	})
+}

api/src/pkg/database/main.go

@@ -11,10 +11,10 @@ const (
 )
 
 var (
-	DB           = initDataBase(DBPath)
+	DB = initDataBase(DBPath)
 )
 
-func GetEntriesTable() *gorm.DB{
+func GetEntriesTable() *gorm.DB {
 	return DB.Table("mizu_entries")
 }
 

api/src/pkg/inserter/main.go

@@ -0,0 +1,91 @@
+package inserter
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"github.com/google/martian/har"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"io"
+	"io/fs"
+	"mizuserver/src/pkg/database"
+	"mizuserver/src/pkg/models"
+	"mizuserver/src/pkg/utils"
+	"net/url"
+	"os"
+	"path"
+	"sort"
+	"time"
+)
+
+
+func IsEmpty(name string) (bool) {
+	f, err := os.Open(name)
+	if err != nil {
+		return false
+	}
+	defer f.Close()
+
+	_, err = f.Readdirnames(1) // Or f.Readdir(1)
+	if err == io.EOF {
+		return true
+	}
+	return false // Either not empty or error, suits both cases
+}
+
+func StartReadingFiles(workingDir string) {
+	err := os.MkdirAll(workingDir, fs.ModeDir)
+	utils.CheckErr(err)
+
+	for true {
+		if IsEmpty(workingDir) {
+			fmt.Printf("Waiting for new files\n")
+			time.Sleep(5 * time.Second)
+			continue
+		}
+
+		dir, _ := os.Open(workingDir)
+		dirFiles, _ := dir.Readdir(-1)
+		sort.Sort(utils.ByModTime(dirFiles))
+
+		fileInfo := dirFiles[0]
+		inputFilePath := path.Join(workingDir, fileInfo.Name())
+		file, err := os.Open(inputFilePath)
+		utils.CheckErr(err)
+
+		var inputHar har.HAR
+		decErr := json.NewDecoder(bufio.NewReader(file)).Decode(&inputHar)
+		utils.CheckErr(decErr)
+
+		for _, entry := range inputHar.Log.Entries 	{
+			fmt.Printf("Entry inserted")
+			SaveHarToDb(*entry, "")
+		}
+		rmErr := os.Remove(inputFilePath)
+		utils.CheckErr(rmErr)
+	}
+
+}
+
+func SaveHarToDb(entry har.Entry, source string) {
+	entryBytes, _ := json.Marshal(entry)
+	serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL)
+	mizuEntry := models.MizuEntry{
+		EntryId:   primitive.NewObjectID().Hex(),
+		Entry:     string(entryBytes), // simple way to store it and not convert to bytes
+		Service:   serviceName,
+		Url:       entry.Request.URL,
+		Path:      urlPath,
+		Method:    entry.Request.Method,
+		Status:    entry.Response.Status,
+		Source:    source,
+		Timestamp: entry.StartedDateTime.Unix(),
+	}
+	database.GetEntriesTable().Create(&mizuEntry)
+}
+
+func getServiceNameFromUrl(inputUrl string) (string, string) {
+	parsed, err := url.Parse(inputUrl)
+	utils.CheckErr(err)
+	return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host), parsed.Path
+}

api/src/pkg/models/mizuEntry.go

@@ -1,11 +1,11 @@
 package models
 
-import (
+import "time"
-	"gorm.io/gorm"
-)
 
 type MizuEntry struct {
-	gorm.Model
+	ID        uint `gorm:"primarykey"`
+	CreatedAt time.Time
+	UpdatedAt time.Time
 	Entry     string `json:"entry,omitempty" gorm:"column:entry"`
 	EntryId   string `json:"entryId" gorm:"column:entryId"`
 	Url       string `json:"url" gorm:"column:url"`

api/src/pkg/routes/public_routes.go

@@ -11,4 +11,7 @@ func PublicRoutes(fiberApp *fiber.App) {
 
 	routeGroup.Get("/entries", controllers.GetEntries)        // get entries (base/thin entries)
 	routeGroup.Get("/entries/:entryId", controllers.GetEntry) // get single (full) entry
+
+	routeGroup.Get("/resetDB", controllers.DeleteAllEntries) // get single (full) entry
+
 }

api/testing_from_file.go

@@ -30,7 +30,7 @@ func TestHarSavingFromFolder(inputDir string) {
 		utils.CheckErr(decErr)
 
 		for _, entry := range inputHar.Log.Entries {
-			SaveHarToDb(*entry, "source")
+			SaveHarToDb(*entry, "")
 		}
 	}
 }