github/kubeshark
1
0
Fork 0
mirror of https://github.com/kubeshark/kubeshark.git synced 2025-07-19 02:40:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

TRA-3437 prevent vacuum bottleneck

Browse Source 
TRA-3437 prevent vacuum bottleneck
This commit is contained in:
RamiBerm 2021-07-15 11:11:51 +03:00 committed by GitHub
commit 5e7ef0fbb9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 15 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/pkg/api/main.go
View File

@ -161,7 +161,7 @@ func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) {
IsOutgoing: connectionInfo.IsOutgoing,
}
mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry)
database.GetEntriesTable().Create(&mizuEntry)
database.CreateEntry(&mizuEntry)
baseEntry := models.BaseEntryDetails{}
if err := models.GetEntry(&mizuEntry, &baseEntry); err != nil {

14
api/pkg/database/main.go
View File

@ -12,11 +12,6 @@ import (
const (
DBPath = "./entries.db"
)
var DB *gorm.DB
const (
OrderDesc = "desc"
OrderAsc = "asc"
LT = "lt"
@ -24,6 +19,8 @@ const (
)
var (
DB *gorm.DB
IsDBLocked = false
OperatorToSymbolMapping = map[string]string{
LT: "<",
GT: ">",
@ -43,6 +40,13 @@ func GetEntriesTable() *gorm.DB {
return DB.Table("mizu_entries")
}
func CreateEntry(entry *models.MizuEntry) {
if IsDBLocked {
return
}
GetEntriesTable().Create(entry)
}
func initDataBase(databasePath string) *gorm.DB {
temp, _ := gorm.Open(sqlite.Open(databasePath), &gorm.Config{
Logger: &utils.TruncatingLogger{LogLevel: logger.Warn, SlowThreshold: 500 * time.Millisecond},

4
api/pkg/database/size_enforcer.go
View File

@ -81,6 +81,10 @@ func checkFileSize(maxSizeBytes int64) {
}
func pruneOldEntries(currentFileSize int64) {
// sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here
IsDBLocked = true
defer func() {IsDBLocked = false}()
amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune)
rows, err := GetEntriesTable().Limit(10000).Order("id").Rows()

4
cli/cmd/tap.go
View File

@ -64,10 +64,8 @@ Supported protocols are HTTP and gRPC.`,
mizuTapOptions.MaxEntriesDBSizeBytes, parseHumanDataSizeErr = units.HumanReadableToBytes(humanMaxEntriesDBSize)
if parseHumanDataSizeErr != nil {
return errors.New(fmt.Sprintf("Could not parse --max-entries-db-size value %s", humanMaxEntriesDBSize))
} else if cmd.Flags().Changed(maxEntriesDBSizeFlagName) {
// We're parsing human readable file sizes here so its best to be unambiguous
fmt.Printf("Setting max entries db size to %s\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes))
}
fmt.Printf("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes))
directionLowerCase := strings.ToLower(direction)
if directionLowerCase == "any" {