From 21b91ea6e455b8c67e1e3b5ab46a7fba35392b9c Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Thu, 15 Jul 2021 09:38:12 +0300 Subject: [PATCH 1/4] Update main.go, main.go, and size_enforcer.go --- api/pkg/api/main.go | 2 +- api/pkg/database/main.go | 14 +++++++++----- api/pkg/database/size_enforcer.go | 3 +++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/api/pkg/api/main.go b/api/pkg/api/main.go index 8ae1352af..711b64bf2 100644 --- a/api/pkg/api/main.go +++ b/api/pkg/api/main.go @@ -161,7 +161,7 @@ func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) { IsOutgoing: connectionInfo.IsOutgoing, } mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry) - database.GetEntriesTable().Create(&mizuEntry) + database.CreateEntry(&mizuEntry) baseEntry := models.BaseEntryDetails{} if err := models.GetEntry(&mizuEntry, &baseEntry); err != nil { diff --git a/api/pkg/database/main.go b/api/pkg/database/main.go index ea4b420f9..c3b1d7847 100644 --- a/api/pkg/database/main.go +++ b/api/pkg/database/main.go @@ -12,11 +12,6 @@ import ( const ( DBPath = "./entries.db" -) - - var DB *gorm.DB - -const ( OrderDesc = "desc" OrderAsc = "asc" LT = "lt" @@ -24,6 +19,8 @@ const ( ) var ( + DB *gorm.DB + IsDBLocked = false OperatorToSymbolMapping = map[string]string{ LT: "<", GT: ">", @@ -43,6 +40,13 @@ func GetEntriesTable() *gorm.DB { return DB.Table("mizu_entries") } +func CreateEntry(entry *models.MizuEntry) { + if IsDBLocked { + return + } + GetEntriesTable().Create(entry) +} + func initDataBase(databasePath string) *gorm.DB { temp, _ := gorm.Open(sqlite.Open(databasePath), &gorm.Config{ Logger: &utils.TruncatingLogger{LogLevel: logger.Warn, SlowThreshold: 500 * time.Millisecond}, diff --git a/api/pkg/database/size_enforcer.go b/api/pkg/database/size_enforcer.go index f26b1a4db..08914f0e2 100644 --- a/api/pkg/database/size_enforcer.go +++ b/api/pkg/database/size_enforcer.go @@ -81,6 +81,9 @@ func checkFileSize(maxSizeBytes int64) { } func pruneOldEntries(currentFileSize int64) { + IsDBLocked = true + defer func() {IsDBLocked = false}() + amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune) rows, err := GetEntriesTable().Limit(10000).Order("id").Rows() From 1619df2d5e02360da2522775c662861d8f3decd3 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Thu, 15 Jul 2021 09:54:27 +0300 Subject: [PATCH 2/4] Update size_enforcer.go --- api/pkg/database/size_enforcer.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/api/pkg/database/size_enforcer.go b/api/pkg/database/size_enforcer.go index 08914f0e2..a1e98e4d3 100644 --- a/api/pkg/database/size_enforcer.go +++ b/api/pkg/database/size_enforcer.go @@ -81,9 +81,6 @@ func checkFileSize(maxSizeBytes int64) { } func pruneOldEntries(currentFileSize int64) { - IsDBLocked = true - defer func() {IsDBLocked = false}() - amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune) rows, err := GetEntriesTable().Limit(10000).Order("id").Rows() @@ -110,6 +107,10 @@ func pruneOldEntries(currentFileSize int64) { } if len(entryIdsToRemove) > 0 { + // sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here + IsDBLocked = true + defer func() {IsDBLocked = false}() + GetEntriesTable().Where(entryIdsToRemove).Delete(models.MizuEntry{}) // VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this DB.Exec("VACUUM") From 3b9f5ee32fea3907f2060f06af0f1618b9daffc7 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Thu, 15 Jul 2021 10:41:29 +0300 Subject: [PATCH 3/4] Update size_enforcer.go and tap.go --- api/pkg/database/size_enforcer.go | 8 ++++---- cli/cmd/tap.go | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/api/pkg/database/size_enforcer.go b/api/pkg/database/size_enforcer.go index a1e98e4d3..686775d35 100644 --- a/api/pkg/database/size_enforcer.go +++ b/api/pkg/database/size_enforcer.go @@ -81,6 +81,10 @@ func checkFileSize(maxSizeBytes int64) { } func pruneOldEntries(currentFileSize int64) { + // sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here + IsDBLocked = true + defer func() {IsDBLocked = false}() + amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune) rows, err := GetEntriesTable().Limit(10000).Order("id").Rows() @@ -107,10 +111,6 @@ func pruneOldEntries(currentFileSize int64) { } if len(entryIdsToRemove) > 0 { - // sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here - IsDBLocked = true - defer func() {IsDBLocked = false}() - GetEntriesTable().Where(entryIdsToRemove).Delete(models.MizuEntry{}) // VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this DB.Exec("VACUUM") diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index 46cb4b69e..e63c54c93 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -64,10 +64,8 @@ Supported protocols are HTTP and gRPC.`, mizuTapOptions.MaxEntriesDBSizeBytes, parseHumanDataSizeErr = units.HumanReadableToBytes(humanMaxEntriesDBSize) if parseHumanDataSizeErr != nil { return errors.New(fmt.Sprintf("Could not parse --max-entries-db-size value %s", humanMaxEntriesDBSize)) - } else if cmd.Flags().Changed(maxEntriesDBSizeFlagName) { - // We're parsing human readable file sizes here so its best to be unambiguous - fmt.Printf("Setting max entries db size to %s\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes)) } + fmt.Printf("Mizu will store up to %s in traffic, old traffic will be cleared once the limit is reached.\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes)) directionLowerCase := strings.ToLower(direction) if directionLowerCase == "any" { From 1d6c176c7f78d378490eb345ec8d66284e6f807d Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Thu, 15 Jul 2021 11:10:16 +0300 Subject: [PATCH 4/4] Update tap.go --- cli/cmd/tap.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index e63c54c93..037ad4944 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -65,7 +65,7 @@ Supported protocols are HTTP and gRPC.`, if parseHumanDataSizeErr != nil { return errors.New(fmt.Sprintf("Could not parse --max-entries-db-size value %s", humanMaxEntriesDBSize)) } - fmt.Printf("Mizu will store up to %s in traffic, old traffic will be cleared once the limit is reached.\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes)) + fmt.Printf("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes)) directionLowerCase := strings.ToLower(direction) if directionLowerCase == "any" {