Update size_enforcer.go and tap.go

This commit is contained in:
RamiBerm 2021-07-15 10:41:29 +03:00
parent 1619df2d5e
commit 3b9f5ee32f
2 changed files with 5 additions and 7 deletions

View File

@ -81,6 +81,10 @@ func checkFileSize(maxSizeBytes int64) {
}
func pruneOldEntries(currentFileSize int64) {
// sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here
IsDBLocked = true
defer func() {IsDBLocked = false}()
amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune)
rows, err := GetEntriesTable().Limit(10000).Order("id").Rows()
@ -107,10 +111,6 @@ func pruneOldEntries(currentFileSize int64) {
}
if len(entryIdsToRemove) > 0 {
// sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here
IsDBLocked = true
defer func() {IsDBLocked = false}()
GetEntriesTable().Where(entryIdsToRemove).Delete(models.MizuEntry{})
// VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this
DB.Exec("VACUUM")

View File

@ -64,10 +64,8 @@ Supported protocols are HTTP and gRPC.`,
mizuTapOptions.MaxEntriesDBSizeBytes, parseHumanDataSizeErr = units.HumanReadableToBytes(humanMaxEntriesDBSize)
if parseHumanDataSizeErr != nil {
return errors.New(fmt.Sprintf("Could not parse --max-entries-db-size value %s", humanMaxEntriesDBSize))
} else if cmd.Flags().Changed(maxEntriesDBSizeFlagName) {
// We're parsing human readable file sizes here so its best to be unambiguous
fmt.Printf("Setting max entries db size to %s\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes))
}
fmt.Printf("Mizu will store up to %s in traffic, old traffic will be cleared once the limit is reached.\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes))
directionLowerCase := strings.ToLower(direction)
if directionLowerCase == "any" {