From 3b9f5ee32fea3907f2060f06af0f1618b9daffc7 Mon Sep 17 00:00:00 2001 From: RamiBerm Date: Thu, 15 Jul 2021 10:41:29 +0300 Subject: [PATCH] Update size_enforcer.go and tap.go --- api/pkg/database/size_enforcer.go | 8 ++++---- cli/cmd/tap.go | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/api/pkg/database/size_enforcer.go b/api/pkg/database/size_enforcer.go index a1e98e4d3..686775d35 100644 --- a/api/pkg/database/size_enforcer.go +++ b/api/pkg/database/size_enforcer.go @@ -81,6 +81,10 @@ func checkFileSize(maxSizeBytes int64) { } func pruneOldEntries(currentFileSize int64) { + // sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here + IsDBLocked = true + defer func() {IsDBLocked = false}() + amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune) rows, err := GetEntriesTable().Limit(10000).Order("id").Rows() @@ -107,10 +111,6 @@ func pruneOldEntries(currentFileSize int64) { } if len(entryIdsToRemove) > 0 { - // sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here - IsDBLocked = true - defer func() {IsDBLocked = false}() - GetEntriesTable().Where(entryIdsToRemove).Delete(models.MizuEntry{}) // VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this DB.Exec("VACUUM") diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index 46cb4b69e..e63c54c93 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -64,10 +64,8 @@ Supported protocols are HTTP and gRPC.`, mizuTapOptions.MaxEntriesDBSizeBytes, parseHumanDataSizeErr = units.HumanReadableToBytes(humanMaxEntriesDBSize) if parseHumanDataSizeErr != nil { return errors.New(fmt.Sprintf("Could not parse --max-entries-db-size value %s", humanMaxEntriesDBSize)) - } else if cmd.Flags().Changed(maxEntriesDBSizeFlagName) { - // We're parsing human readable file sizes here so its best to be unambiguous - fmt.Printf("Setting max entries db size to %s\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes)) } + fmt.Printf("Mizu will store up to %s in traffic, old traffic will be cleared once the limit is reached.\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes)) directionLowerCase := strings.ToLower(direction) if directionLowerCase == "any" {