From d684dee7a4ce2343a3c924467ed2bd40249663c0 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Mon, 12 Jul 2021 17:47:46 +0300 Subject: [PATCH] WIP --- api/go.mod | 1 + api/go.sum | 3 + api/pkg/api/main.go | 20 ++++++ api/pkg/database/main.go | 1 + api/pkg/database/size_enforcer.go | 94 ++++++++++++++++++++++++++++ api/pkg/models/models.go | 1 + cli/cmd/tapRunner.go | 2 +- {cli => shared}/debounce/debounce.go | 0 8 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 api/pkg/database/size_enforcer.go rename {cli => shared}/debounce/debounce.go (100%) diff --git a/api/go.mod b/api/go.mod index 9f8825215..27833d4e4 100644 --- a/api/go.mod +++ b/api/go.mod @@ -23,6 +23,7 @@ require ( k8s.io/api v0.21.0 k8s.io/apimachinery v0.21.0 k8s.io/client-go v0.21.0 + github.com/fsnotify/fsnotify v1.4.9 ) replace github.com/up9inc/mizu/shared v0.0.0 => ../shared diff --git a/api/go.sum b/api/go.sum index 855914a0f..baf901d28 100644 --- a/api/go.sum +++ b/api/go.sum @@ -73,6 +73,8 @@ github.com/fasthttp/websocket v1.4.3-beta.1/go.mod h1:JGrgLaT02bL9NuJkZbHN8mVV2t github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -411,6 +413,7 @@ golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/api/pkg/api/main.go b/api/pkg/api/main.go index 142f43700..10faec2ad 100644 --- a/api/pkg/api/main.go +++ b/api/pkg/api/main.go @@ -160,6 +160,7 @@ func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) { ResolvedDestination: resolvedDestination, IsOutgoing: connectionInfo.IsOutgoing, } + mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry) database.GetEntriesTable().Create(&mizuEntry) baseEntry := models.BaseEntryDetails{} @@ -179,3 +180,22 @@ func getServiceNameFromUrl(inputUrl string) (string, string) { func CheckIsServiceIP(address string) bool { return k8sResolver.CheckIsServiceIP(address) } + +// gives a rough estimate of the size this will take up in the db, good enough for maintaining db size limit accurately +func getEstimatedEntrySizeBytes(mizuEntry models.MizuEntry) int { + sizeBytes := len(mizuEntry.Entry) + sizeBytes += len(mizuEntry.EntryId) + sizeBytes += len(mizuEntry.Service) + sizeBytes += len(mizuEntry.Url) + sizeBytes += len(mizuEntry.Method) + sizeBytes += len(mizuEntry.RequestSenderIp) + sizeBytes += len(mizuEntry.ResolvedDestination) + sizeBytes += len(mizuEntry.ResolvedSource) + sizeBytes += 8 // Status bytes (sqlite integer is always 8 bytes) + sizeBytes += 8 // Timestamp bytes + sizeBytes += 8 // SizeBytes bytes + sizeBytes += 1 // IsOutgoing bytes + + + return sizeBytes +} diff --git a/api/pkg/database/main.go b/api/pkg/database/main.go index 4f144400e..921b13442 100644 --- a/api/pkg/database/main.go +++ b/api/pkg/database/main.go @@ -39,6 +39,7 @@ func GetEntriesTable() *gorm.DB { } func initDataBase(databasePath string) *gorm.DB { + go StartEnforcingDatabaseSize( 10 * 1000 * 1000) temp, _ := gorm.Open(sqlite.Open(databasePath), &gorm.Config{}) _ = temp.AutoMigrate(&models.MizuEntry{}) // this will ensure table is created return temp diff --git a/api/pkg/database/size_enforcer.go b/api/pkg/database/size_enforcer.go new file mode 100644 index 000000000..f84d461b9 --- /dev/null +++ b/api/pkg/database/size_enforcer.go @@ -0,0 +1,94 @@ +package database + +import ( + "fmt" + "github.com/fsnotify/fsnotify" + "github.com/up9inc/mizu/shared/debounce" + "log" + "mizuserver/pkg/models" + "os" + "time" +) + +const percentageOfMaxSizeBytesToPrune = 5 + +func StartEnforcingDatabaseSize(maxSizeBytes int64) { + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal("Error creating filesystem watcher for db size enforcement:", err) + } + defer watcher.Close() + + checkFileSizeDebouncer := debounce.NewDebouncer(5 * time.Second, func() { + checkFileSize(maxSizeBytes) + }) + + done := make(chan bool) + go func() { + for { + select { + case event, ok := <-watcher.Events: + if !ok { + return // closed channel + } + if event.Op&fsnotify.Write == fsnotify.Write { + checkFileSizeDebouncer.SetOn() + } + case err, ok := <-watcher.Errors: + if !ok { + return // closed channel + } + fmt.Printf("filesystem watcher encountered error:%v\n", err) + } + } + }() + + err = watcher.Add(DBPath) + if err != nil { + log.Fatal(fmt.Sprintf("Error adding %s to filesystem watcher for db size enforcement: %v", DBPath, err)) + } + <-done +} + +func checkFileSize(maxSizeBytes int64) { + fileStat, err := os.Stat(DBPath) + if err != nil { + fmt.Printf("Error checking %s file size: %v\n", DBPath, err) + } else { + if fileStat.Size() > maxSizeBytes { + pruneOldEntries(maxSizeBytes) + } + } +} + +func pruneOldEntries(maxSizeBytes int64) { + amountOfBytesToTrim := maxSizeBytes * (percentageOfMaxSizeBytesToPrune / 100) + + rows, err := GetEntriesTable().Limit(100).Order("id").Rows() + if err != nil { + fmt.Printf("Error getting 100 first db rows: %v\n", err) + return + } + + entryIdsToRemove := make([]uint, 0) + bytesToBeRemoved := int64(0) + for rows.Next() { + if bytesToBeRemoved >= amountOfBytesToTrim { + break + } + var entry models.MizuEntry + err = DB.ScanRows(rows, &entry) + if err != nil { + fmt.Printf("Error scanning db row: %v\n", err) + continue + } + + entryIdsToRemove = append(entryIdsToRemove, entry.ID) + bytesToBeRemoved += int64(entry.EstimatedSizeBytes) + } + + if len(entryIdsToRemove) > 0 { + GetEntriesTable().Delete(entryIdsToRemove) + fmt.Printf("Removed %d rows and cleared %d bytes", len(entryIdsToRemove), bytesToBeRemoved) + } +} diff --git a/api/pkg/models/models.go b/api/pkg/models/models.go index 8c0d07e22..bf2d5b744 100644 --- a/api/pkg/models/models.go +++ b/api/pkg/models/models.go @@ -33,6 +33,7 @@ type MizuEntry struct { ResolvedSource string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"` ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` IsOutgoing bool `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"` + EstimatedSizeBytes int `json:"-" gorm:"column:estimatedSizeBytes"` } type BaseEntryDetails struct { diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index d3a6a69f8..320b37987 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -3,10 +3,10 @@ package cmd import ( "context" "fmt" - "github.com/up9inc/mizu/cli/debounce" "github.com/up9inc/mizu/cli/kubernetes" "github.com/up9inc/mizu/cli/mizu" "github.com/up9inc/mizu/shared" + "github.com/up9inc/mizu/shared/debounce" core "k8s.io/api/core/v1" "log" "net/http" diff --git a/cli/debounce/debounce.go b/shared/debounce/debounce.go similarity index 100% rename from cli/debounce/debounce.go rename to shared/debounce/debounce.go