github/kubeshark
1
0
Fork 0
mirror of https://github.com/kubeshark/kubeshark.git synced 2025-07-18 02:11:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

⚡ Download files in parallel

Browse Source
This commit is contained in:
M. Mert Yildiran 2023-06-29 16:45:59 +03:00
parent 98738cb5a6
commit aeda619104
No known key found for this signature in database
GPG Key ID: DA5D6DCBB758A461
1 changed files with 29 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
cmd/tapPcapRunner.go
View File

@ -12,11 +12,13 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"sync"
"github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws"
awsConfig "github.com/aws/aws-sdk-go-v2/config" awsConfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3"
s3Types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/docker/docker/api/types" "github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/container"
"github.com/docker/docker/client" "github.com/docker/docker/client"
@ -330,33 +332,39 @@ func downloadTarFromS3(s3Url string) (tarPath string, err error) {
return return
} }
var wg sync.WaitGroup
for _, object := range listObjectsOutput.Contents { for _, object := range listObjectsOutput.Contents {
objectKey := *object.Key wg.Add(1)
go func(object s3Types.Object) {
defer wg.Done()
objectKey := *object.Key
fullPath := filepath.Join(tempDirPath, objectKey) fullPath := filepath.Join(tempDirPath, objectKey)
err = os.MkdirAll(filepath.Dir(fullPath), os.ModePerm) err = os.MkdirAll(filepath.Dir(fullPath), os.ModePerm)
if err != nil { if err != nil {
return return
} }
var objectFile *os.File var objectFile *os.File
objectFile, err = os.Create(fullPath) objectFile, err = os.Create(fullPath)
if err != nil { if err != nil {
return return
} }
defer objectFile.Close() defer objectFile.Close()
log.Info().Str("bucket", bucket).Str("key", objectKey).Msg("Downloading from S3") log.Info().Str("bucket", bucket).Str("key", objectKey).Msg("Downloading from S3")
downloader := manager.NewDownloader(client) downloader := manager.NewDownloader(client)
_, err = downloader.Download(context.TODO(), objectFile, &s3.GetObjectInput{ _, err = downloader.Download(context.TODO(), objectFile, &s3.GetObjectInput{
Bucket: aws.String(bucket), Bucket: aws.String(bucket),
Key: aws.String(objectKey), Key: aws.String(objectKey),
}) })
if err != nil { if err != nil {
return return
} }
}(object)
} }
wg.Wait()
tarPath, err = tarDirectory(tempDirPath) tarPath, err = tarDirectory(tempDirPath)
return return