diff --git a/api/pkg/api/main.go b/api/pkg/api/main.go index 8ae1352af..711b64bf2 100644 --- a/api/pkg/api/main.go +++ b/api/pkg/api/main.go @@ -161,7 +161,7 @@ func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) { IsOutgoing: connectionInfo.IsOutgoing, } mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry) - database.GetEntriesTable().Create(&mizuEntry) + database.CreateEntry(&mizuEntry) baseEntry := models.BaseEntryDetails{} if err := models.GetEntry(&mizuEntry, &baseEntry); err != nil { diff --git a/api/pkg/database/main.go b/api/pkg/database/main.go index ea4b420f9..c3b1d7847 100644 --- a/api/pkg/database/main.go +++ b/api/pkg/database/main.go @@ -12,11 +12,6 @@ import ( const ( DBPath = "./entries.db" -) - - var DB *gorm.DB - -const ( OrderDesc = "desc" OrderAsc = "asc" LT = "lt" @@ -24,6 +19,8 @@ const ( ) var ( + DB *gorm.DB + IsDBLocked = false OperatorToSymbolMapping = map[string]string{ LT: "<", GT: ">", @@ -43,6 +40,13 @@ func GetEntriesTable() *gorm.DB { return DB.Table("mizu_entries") } +func CreateEntry(entry *models.MizuEntry) { + if IsDBLocked { + return + } + GetEntriesTable().Create(entry) +} + func initDataBase(databasePath string) *gorm.DB { temp, _ := gorm.Open(sqlite.Open(databasePath), &gorm.Config{ Logger: &utils.TruncatingLogger{LogLevel: logger.Warn, SlowThreshold: 500 * time.Millisecond}, diff --git a/api/pkg/database/size_enforcer.go b/api/pkg/database/size_enforcer.go index f26b1a4db..686775d35 100644 --- a/api/pkg/database/size_enforcer.go +++ b/api/pkg/database/size_enforcer.go @@ -81,6 +81,10 @@ func checkFileSize(maxSizeBytes int64) { } func pruneOldEntries(currentFileSize int64) { + // sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here + IsDBLocked = true + defer func() {IsDBLocked = false}() + amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune) rows, err := GetEntriesTable().Limit(10000).Order("id").Rows() diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index 46cb4b69e..037ad4944 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -64,10 +64,8 @@ Supported protocols are HTTP and gRPC.`, mizuTapOptions.MaxEntriesDBSizeBytes, parseHumanDataSizeErr = units.HumanReadableToBytes(humanMaxEntriesDBSize) if parseHumanDataSizeErr != nil { return errors.New(fmt.Sprintf("Could not parse --max-entries-db-size value %s", humanMaxEntriesDBSize)) - } else if cmd.Flags().Changed(maxEntriesDBSizeFlagName) { - // We're parsing human readable file sizes here so its best to be unambiguous - fmt.Printf("Setting max entries db size to %s\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes)) } + fmt.Printf("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes)) directionLowerCase := strings.ToLower(direction) if directionLowerCase == "any" {