WIP

api/go.mod

@@ -23,6 +23,7 @@ require (
 	k8s.io/api v0.21.0
 	k8s.io/apimachinery v0.21.0
 	k8s.io/client-go v0.21.0
+	github.com/fsnotify/fsnotify v1.4.9
 )
 
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared

api/go.sum

@@ -73,6 +73,8 @@ github.com/fasthttp/websocket v1.4.3-beta.1/go.mod h1:JGrgLaT02bL9NuJkZbHN8mVV2t
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -411,6 +413,7 @@ golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

api/pkg/api/main.go

@@ -160,6 +160,7 @@ func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) {
 		ResolvedDestination: resolvedDestination,
 		IsOutgoing:          connectionInfo.IsOutgoing,
 	}
+	mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry)
 	database.GetEntriesTable().Create(&mizuEntry)
 
 	baseEntry := models.BaseEntryDetails{}
@@ -179,3 +180,22 @@ func getServiceNameFromUrl(inputUrl string) (string, string) {
 func CheckIsServiceIP(address string) bool {
 	return k8sResolver.CheckIsServiceIP(address)
 }
+
+// gives a rough estimate of the size this will take up in the db, good enough for maintaining db size limit accurately
+func getEstimatedEntrySizeBytes(mizuEntry models.MizuEntry) int {
+	sizeBytes := len(mizuEntry.Entry)
+	sizeBytes += len(mizuEntry.EntryId)
+	sizeBytes += len(mizuEntry.Service)
+	sizeBytes += len(mizuEntry.Url)
+	sizeBytes += len(mizuEntry.Method)
+	sizeBytes += len(mizuEntry.RequestSenderIp)
+	sizeBytes += len(mizuEntry.ResolvedDestination)
+	sizeBytes += len(mizuEntry.ResolvedSource)
+	sizeBytes += 8 // Status bytes (sqlite integer is always 8 bytes)
+	sizeBytes += 8 // Timestamp bytes
+	sizeBytes += 8 // SizeBytes bytes
+	sizeBytes += 1 // IsOutgoing bytes
+
+
+	return sizeBytes
+}

api/pkg/database/main.go

@@ -39,6 +39,7 @@ func GetEntriesTable() *gorm.DB {
 }
 
 func initDataBase(databasePath string) *gorm.DB {
+	go StartEnforcingDatabaseSize( 10 * 1000 * 1000)
 	temp, _ := gorm.Open(sqlite.Open(databasePath), &gorm.Config{})
 	_ = temp.AutoMigrate(&models.MizuEntry{}) // this will ensure table is created
 	return temp

api/pkg/database/size_enforcer.go

@@ -0,0 +1,94 @@
+package database
+
+import (
+	"fmt"
+	"github.com/fsnotify/fsnotify"
+	"github.com/up9inc/mizu/shared/debounce"
+	"log"
+	"mizuserver/pkg/models"
+	"os"
+	"time"
+)
+
+const percentageOfMaxSizeBytesToPrune = 5
+
+func StartEnforcingDatabaseSize(maxSizeBytes int64) {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		log.Fatal("Error creating filesystem watcher for db size enforcement:", err)
+	}
+	defer watcher.Close()
+
+	checkFileSizeDebouncer := debounce.NewDebouncer(5 * time.Second, func() {
+		checkFileSize(maxSizeBytes)
+	})
+
+	done := make(chan bool)
+	go func() {
+		for {
+			select {
+			case event, ok := <-watcher.Events:
+				if !ok {
+					return // closed channel
+				}
+				if event.Op&fsnotify.Write == fsnotify.Write {
+					checkFileSizeDebouncer.SetOn()
+				}
+			case err, ok := <-watcher.Errors:
+				if !ok {
+					return // closed channel
+				}
+				fmt.Printf("filesystem watcher encountered error:%v\n", err)
+			}
+		}
+	}()
+
+	err = watcher.Add(DBPath)
+	if err != nil {
+		log.Fatal(fmt.Sprintf("Error adding %s to filesystem watcher for db size enforcement: %v", DBPath, err))
+	}
+	<-done
+}
+
+func checkFileSize(maxSizeBytes int64) {
+	fileStat, err := os.Stat(DBPath)
+	if err != nil {
+		fmt.Printf("Error checking %s file size: %v\n", DBPath, err)
+	} else {
+		if fileStat.Size() > maxSizeBytes {
+			pruneOldEntries(maxSizeBytes)
+		}
+	}
+}
+
+func pruneOldEntries(maxSizeBytes int64) {
+	amountOfBytesToTrim := maxSizeBytes * (percentageOfMaxSizeBytesToPrune / 100)
+
+	rows, err := GetEntriesTable().Limit(100).Order("id").Rows()
+	if err != nil {
+		fmt.Printf("Error getting 100 first db rows: %v\n", err)
+		return
+	}
+
+	entryIdsToRemove := make([]uint, 0)
+	bytesToBeRemoved := int64(0)
+	for rows.Next() {
+		if bytesToBeRemoved >= amountOfBytesToTrim {
+			break
+		}
+		var entry models.MizuEntry
+		err = DB.ScanRows(rows, &entry)
+		if err != nil {
+			fmt.Printf("Error scanning db row: %v\n", err)
+			continue
+		}
+
+		entryIdsToRemove = append(entryIdsToRemove, entry.ID)
+		bytesToBeRemoved += int64(entry.EstimatedSizeBytes)
+	}
+
+	if len(entryIdsToRemove) > 0 {
+		GetEntriesTable().Delete(entryIdsToRemove)
+		fmt.Printf("Removed %d rows and cleared %d bytes", len(entryIdsToRemove), bytesToBeRemoved)
+	}
+}

api/pkg/models/models.go

@@ -33,6 +33,7 @@ type MizuEntry struct {
 	ResolvedSource      string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"`
 	ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
 	IsOutgoing          bool   `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"`
+	EstimatedSizeBytes           int `json:"-" gorm:"column:estimatedSizeBytes"`
 }
 
 type BaseEntryDetails struct {

cli/cmd/tapRunner.go

@@ -3,10 +3,10 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"github.com/up9inc/mizu/cli/debounce"
 	"github.com/up9inc/mizu/cli/kubernetes"
 	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/debounce"
 	core "k8s.io/api/core/v1"
 	"log"
 	"net/http"