mirror of
https://github.com/k8sgpt-ai/k8sgpt.git
synced 2025-09-07 18:20:36 +00:00
feat: rework cache package - add gcs cache - add cache purge command (#750)
* feat: rework cache pkg Signed-off-by: Matthis Holleville <matthish29@gmail.com> * feat: Completion of cache pkg rework. Added cache purge command. Signed-off-by: Matthis Holleville <matthish29@gmail.com> * doc: add purgin command note Signed-off-by: Matthis Holleville <matthish29@gmail.com> * fix: disable cache if noCache is set Signed-off-by: Matthis Holleville <matthish29@gmail.com> * feat: improve GetCacheConfiguration lisibility & transform add method to addOrUpdate Signed-off-by: Matthis Holleville <matthish29@gmail.com> * feat: transform server mode to work with new cache configuration Signed-off-by: Matthis Holleville <matthish29@gmail.com> * fix: use 'switch' instead 'if' to evaluate Cache from grpc Signed-off-by: Matthis Holleville <matthish29@gmail.com> * feat: add mutually exclusive flags for command options Signed-off-by: Matthis Holleville <matthish29@gmail.com> * doc: update readme.md Signed-off-by: Matthis Holleville <matthish29@gmail.com> * feat: return err on bucket creation failed Signed-off-by: Matthis Holleville <matthish29@gmail.com> * feat: update dependencies Signed-off-by: Matthis Holleville <matthish29@gmail.com> --------- Signed-off-by: Matthis Holleville <matthish29@gmail.com> Signed-off-by: Matthis <matthish29@gmail.com>
This commit is contained in:
Matthis
GitHub
16
README.md
16
README.md
@@ -595,19 +595,29 @@ _Adding a remote cache_
|
||||
|
||||
* AWS S3
|
||||
* _As a prerequisite `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` are required as environmental variables._
|
||||
* Configuration, ``` k8sgpt cache add --region <aws region> --bucket <name> ```
|
||||
* Configuration, ``` k8sgpt cache add s3 --region <aws region> --bucket <name> ```
|
||||
* K8sGPT will create the bucket if it does not exist
|
||||
* Azure Storage
|
||||
* We support a number of [techniques](https://learn.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication?tabs=bash#2-authenticate-with-azure) to authenticate against Azure
|
||||
* Configuration, ``` k8sgpt cache add --storageacc <storage account name> --container <container name> ```
|
||||
* Configuration, ``` k8sgpt cache add azure --storageacc <storage account name> --container <container name> ```
|
||||
* K8sGPT assumes that the storage account already exist and it will create the container if it does not exist
|
||||
* It's **users'** responsibility have to grant specific permissions to their identity in order to be able to upload blob files and create SA containers (e.g Storage Blob Data Contributor)
|
||||
* It is the **user** responsibility have to grant specific permissions to their identity in order to be able to upload blob files and create SA containers (e.g Storage Blob Data Contributor)
|
||||
* Google Cloud Storage
|
||||
* _As a prerequisite `GOOGLE_APPLICATION_CREDENTIALS` are required as environmental variables._
|
||||
* Configuration, ``` k8sgpt cache add gcs --region <gcp region> --bucket <name> --projectid <project id>```
|
||||
* K8sGPT will create the bucket if it does not exist
|
||||
|
||||
_Listing cache items_
|
||||
```
|
||||
k8sgpt cache list
|
||||
```
|
||||
|
||||
_Purging an object from the cache_
|
||||
Note: purging an object using this command will delete upstream files, so it requires appropriate permissions.
|
||||
```
|
||||
k8sgpt cache purge $OBJECT_NAME
|
||||
```
|
||||
|
||||
_Removing the remote cache_
|
||||
Note: this will not delete the upstream S3 bucket or Azure storage container
|
||||
```
|
||||
|
||||
22
cmd/cache/add.go
vendored
22
cmd/cache/add.go
vendored
@@ -28,20 +28,31 @@ var (
|
||||
bucketName string
|
||||
storageAccount string
|
||||
containerName string
|
||||
projectId string
|
||||
)
|
||||
|
||||
// addCmd represents the add command
|
||||
var addCmd = &cobra.Command{
|
||||
Use: "add",
|
||||
Use: "add [cache type]",
|
||||
Short: "Add a remote cache",
|
||||
Long: `This command allows you to add a remote cache to store the results of an analysis.
|
||||
The supported cache types are:
|
||||
- Azure Blob storage
|
||||
- Google Cloud storage
|
||||
- S3`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
if len(args) == 0 {
|
||||
color.Red("Error: Please provide a value for cache types. Run k8sgpt cache add --help")
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println(color.YellowString("Adding remote based cache"))
|
||||
remoteCache := cache.NewCacheProvider(bucketname, region, storageAccount, containerName)
|
||||
err := cache.AddRemoteCache(remoteCache)
|
||||
cacheType := args[0]
|
||||
remoteCache, err := cache.NewCacheProvider(cacheType, bucketname, region, storageAccount, containerName, projectId)
|
||||
if err != nil {
|
||||
color.Red("Error: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
err = cache.AddRemoteCache(remoteCache)
|
||||
if err != nil {
|
||||
color.Red("Error: %v", err)
|
||||
os.Exit(1)
|
||||
@@ -51,9 +62,10 @@ var addCmd = &cobra.Command{
|
||||
|
||||
func init() {
|
||||
CacheCmd.AddCommand(addCmd)
|
||||
addCmd.Flags().StringVarP(®ion, "region", "r", "", "The region to use for the AWS S3 cache")
|
||||
addCmd.Flags().StringVarP(®ion, "region", "r", "", "The region to use for the AWS S3 or GCS cache")
|
||||
addCmd.Flags().StringVarP(&bucketname, "bucket", "b", "", "The name of the AWS S3 bucket to use for the cache")
|
||||
addCmd.MarkFlagsRequiredTogether("region", "bucket")
|
||||
addCmd.Flags().StringVarP(&projectId, "projectid", "p", "", "The GCP project ID")
|
||||
addCmd.Flags().StringVarP(&storageAccount, "storageacc", "s", "", "The Azure storage account name of the container")
|
||||
addCmd.Flags().StringVarP(&containerName, "container", "c", "", "The Azure container name to use for the cache")
|
||||
addCmd.MarkFlagsRequiredTogether("storageacc", "container")
|
||||
@@ -62,4 +74,6 @@ func init() {
|
||||
addCmd.MarkFlagsMutuallyExclusive("region", "container")
|
||||
addCmd.MarkFlagsMutuallyExclusive("bucket", "storageacc")
|
||||
addCmd.MarkFlagsMutuallyExclusive("bucket", "container")
|
||||
addCmd.MarkFlagsMutuallyExclusive("projectid", "storageacc")
|
||||
addCmd.MarkFlagsMutuallyExclusive("projectid", "container")
|
||||
}
|
||||
|
||||
22
cmd/cache/list.go
vendored
22
cmd/cache/list.go
vendored
@@ -16,9 +16,11 @@ package cache
|
||||
|
||||
import (
|
||||
"os"
|
||||
"reflect"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/k8sgpt-ai/k8sgpt/pkg/cache"
|
||||
"github.com/olekukonko/tablewriter"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
@@ -30,22 +32,32 @@ var listCmd = &cobra.Command{
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
|
||||
// load remote cache if it is configured
|
||||
remoteCacheEnabled, err := cache.RemoteCacheEnabled()
|
||||
c, err := cache.GetCacheConfiguration()
|
||||
if err != nil {
|
||||
color.Red("Error: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
c := cache.New(false, remoteCacheEnabled)
|
||||
// list the contents of the cache
|
||||
names, err := c.List()
|
||||
if err != nil {
|
||||
color.Red("Error: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
for _, name := range names {
|
||||
println(name)
|
||||
var headers []string
|
||||
obj := cache.CacheObjectDetails{}
|
||||
objType := reflect.TypeOf(obj)
|
||||
for i := 0; i < objType.NumField(); i++ {
|
||||
field := objType.Field(i)
|
||||
headers = append(headers, field.Name)
|
||||
}
|
||||
|
||||
table := tablewriter.NewWriter(os.Stdout)
|
||||
table.SetHeader(headers)
|
||||
|
||||
for _, v := range names {
|
||||
table.Append([]string{v.Name, v.UpdatedAt.String()})
|
||||
}
|
||||
table.Render()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
54
cmd/cache/purge.go
vendored
Normal file
54
cmd/cache/purge.go
vendored
Normal file
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
Copyright 2023 The K8sGPT Authors.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package cache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/k8sgpt-ai/k8sgpt/pkg/cache"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var purgeCmd = &cobra.Command{
|
||||
Use: "purge [object name]",
|
||||
Short: "Purge a remote cache",
|
||||
Long: "This command allows you to delete/purge one object from the cache",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
if len(args) == 0 {
|
||||
color.Red("Error: Please provide a value for object name. Run k8sgpt cache purge --help")
|
||||
os.Exit(1)
|
||||
}
|
||||
objectKey := args[0]
|
||||
fmt.Println(color.YellowString("Purging a remote cache."))
|
||||
c, err := cache.GetCacheConfiguration()
|
||||
if err != nil {
|
||||
color.Red("Error: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
err = c.Remove(objectKey)
|
||||
if err != nil {
|
||||
color.Red("Error: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println(color.GreenString("Object deleted."))
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
CacheCmd.AddCommand(purgeCmd)
|
||||
}
|
||||
18
go.mod
18
go.mod
@@ -24,15 +24,22 @@ require (
|
||||
require github.com/adrg/xdg v0.4.0
|
||||
|
||||
require (
|
||||
buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231002095256-194bc640518b.1
|
||||
buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20231002095256-194bc640518b.1
|
||||
buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231116211251-9f5041346631.2
|
||||
buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.28.1-20231116211251-9f5041346631.4
|
||||
cloud.google.com/go/storage v1.30.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0
|
||||
github.com/aws/aws-sdk-go v1.48.0
|
||||
github.com/cohere-ai/cohere-go v0.2.0
|
||||
github.com/olekukonko/tablewriter v0.0.5
|
||||
google.golang.org/api v0.143.0
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.110.7 // indirect
|
||||
cloud.google.com/go/compute v1.23.0 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
cloud.google.com/go/iam v1.1.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect
|
||||
@@ -41,13 +48,20 @@ require (
|
||||
github.com/cohere-ai/tokenizer v1.1.1 // indirect
|
||||
github.com/dlclark/regexp2 v1.4.0 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect
|
||||
github.com/google/s2a-go v0.1.7 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
|
||||
github.com/sagikazarmark/locafero v0.3.0 // indirect
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.0 // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect
|
||||
)
|
||||
|
||||
|
||||
41
go.sum
41
go.sum
@@ -1,8 +1,7 @@
|
||||
buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231002095256-194bc640518b.1 h1:xYEAhcdWh89HNtbM5Uv4b2xu+4/MkNffR9JNrnnEjXU=
|
||||
buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231002095256-194bc640518b.1/go.mod h1:j2QJ3L7VTtI+VeK6t03h9X934FolVTb3FwXUc76bQMQ=
|
||||
buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.28.1-20231002095256-194bc640518b.4/go.mod h1:i/s4ALHwKvjA1oGNKpoHg0FpEOTbufoOm/NdTE6YQAE=
|
||||
buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20231002095256-194bc640518b.1 h1:Bt8mnCodD/BqChxt/r3xYayGLoOAn334qC1tN7VqUTE=
|
||||
buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20231002095256-194bc640518b.1/go.mod h1:gtnk2yAUexdY5nTuUg0SH5WCCGvpKzr7pd3Xbi7MWjE=
|
||||
buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231116211251-9f5041346631.2 h1:lr4kHFzPIzoWIHF6s/B+0Wb/WknraHrbqNguavOzP70=
|
||||
buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231116211251-9f5041346631.2/go.mod h1:g/UdFu0wAAS44ncCfX3zjna7WC1gp6BENVL6LPJ9tow=
|
||||
buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.28.1-20231116211251-9f5041346631.4 h1:f7e94G/tYVs+QDTvjU9sFolC5tuJxlOjzUBJ84XMwkg=
|
||||
buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.28.1-20231116211251-9f5041346631.4/go.mod h1:i/s4ALHwKvjA1oGNKpoHg0FpEOTbufoOm/NdTE6YQAE=
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
@@ -41,6 +40,8 @@ cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRY
|
||||
cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM=
|
||||
cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I=
|
||||
cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY=
|
||||
cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o=
|
||||
cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI=
|
||||
cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4=
|
||||
cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw=
|
||||
cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E=
|
||||
@@ -176,9 +177,12 @@ cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvj
|
||||
cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA=
|
||||
cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs=
|
||||
cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU=
|
||||
cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY=
|
||||
cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
|
||||
cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU=
|
||||
cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
|
||||
cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
|
||||
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
|
||||
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
|
||||
cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY=
|
||||
cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck=
|
||||
@@ -313,6 +317,8 @@ cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGE
|
||||
cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY=
|
||||
cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY=
|
||||
cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0=
|
||||
cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y=
|
||||
cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU=
|
||||
cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc=
|
||||
cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A=
|
||||
cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk=
|
||||
@@ -531,6 +537,8 @@ cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeL
|
||||
cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s=
|
||||
cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y=
|
||||
cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4=
|
||||
cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM=
|
||||
cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E=
|
||||
cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w=
|
||||
cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I=
|
||||
cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4=
|
||||
@@ -839,6 +847,7 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er
|
||||
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
||||
@@ -901,10 +910,13 @@ github.com/google/go-containerregistry v0.16.1/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
|
||||
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/licenseclassifier/v2 v2.0.0 h1:1Y57HHILNf4m0ABuMVb6xk4vAJYEUO0gDxNpog0pyeA=
|
||||
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
|
||||
github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
|
||||
github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
@@ -923,6 +935,8 @@ github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLe
|
||||
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20220608213341-c488b8fa1db3 h1:mpL/HvfIgIejhVwAfxBQkwEjlhP5o0O9RAeTAjpwzxc=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
|
||||
github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
|
||||
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
|
||||
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
@@ -935,6 +949,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
|
||||
@@ -946,6 +962,8 @@ github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqE
|
||||
github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY=
|
||||
github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
|
||||
github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI=
|
||||
github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
|
||||
github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
|
||||
github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
|
||||
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
|
||||
github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4=
|
||||
@@ -1052,6 +1070,7 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
|
||||
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
@@ -1098,8 +1117,10 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
|
||||
github.com/onsi/gomega v1.28.1 h1:MijcGUbfYuznzK/5R4CPNoUP/9Xvuo20sXfEm6XxoTA=
|
||||
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
|
||||
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
|
||||
github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU=
|
||||
github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI=
|
||||
@@ -1705,6 +1726,8 @@ google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/
|
||||
google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI=
|
||||
google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0=
|
||||
google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg=
|
||||
google.golang.org/api v0.143.0 h1:o8cekTkqhywkbZT6p1UHJPZ9+9uuCAJs/KYomxZB8fA=
|
||||
google.golang.org/api v0.143.0/go.mod h1:FoX9DO9hT7DLNn97OuoZAGSDuNAXdJRuGK98rSUgurk=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
@@ -1846,8 +1869,12 @@ google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOl
|
||||
google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=
|
||||
google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY=
|
||||
google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk=
|
||||
google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb h1:XFBgcDwm7irdHTbz4Zk2h7Mh+eis4nfJEFQFYzJzuIA=
|
||||
google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb h1:lK0oleSc7IQsUxO3U5TjL9DWlsxpEBemh+zpB7IqhWI=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 h1:N3bU/SQDCDyD6R528GJ/PwW9KjYcJA3dgyH+MovAkIM=
|
||||
|
||||
@@ -115,18 +115,22 @@ func NewAnalysis(backend string, language string, filters []string, namespace st
|
||||
}
|
||||
|
||||
// load remote cache if it is configured
|
||||
remoteCacheEnabled, err := cache.RemoteCacheEnabled()
|
||||
cache, err := cache.GetCacheConfiguration()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if noCache {
|
||||
cache.DisableCache()
|
||||
}
|
||||
|
||||
return &Analysis{
|
||||
Context: ctx,
|
||||
Filters: filters,
|
||||
Client: client,
|
||||
AIClient: aiClient,
|
||||
Namespace: namespace,
|
||||
Cache: cache.New(noCache, remoteCacheEnabled),
|
||||
Cache: cache,
|
||||
Explain: explain,
|
||||
MaxConcurrency: maxConcurrency,
|
||||
AnalysisAIProvider: backend,
|
||||
|
||||
105
pkg/cache/azuresa_based.go
vendored
105
pkg/cache/azuresa_based.go
vendored
@@ -9,7 +9,7 @@ import (
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
"github.com/spf13/viper"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||
)
|
||||
|
||||
// Generate ICache implementation
|
||||
@@ -20,6 +20,48 @@ type AzureCache struct {
|
||||
session *azblob.Client
|
||||
}
|
||||
|
||||
type AzureCacheConfiguration struct {
|
||||
StorageAccount string `mapstructure:"storageaccount" yaml:"storageaccount,omitempty"`
|
||||
ContainerName string `mapstructure:"container" yaml:"container,omitempty"`
|
||||
}
|
||||
|
||||
func (s *AzureCache) Configure(cacheInfo CacheProvider) error {
|
||||
s.ctx = context.Background()
|
||||
if cacheInfo.Azure.ContainerName == "" {
|
||||
log.Fatal("Azure Container name not configured")
|
||||
}
|
||||
if cacheInfo.Azure.StorageAccount == "" {
|
||||
log.Fatal("Azure Storage account not configured")
|
||||
}
|
||||
|
||||
// We assume that Storage account is already in place
|
||||
blobUrl := fmt.Sprintf("https://%s.blob.core.windows.net/", cacheInfo.Azure.StorageAccount)
|
||||
credential, err := azidentity.NewDefaultAzureCredential(nil)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
client, err := azblob.NewClient(blobUrl, credential, nil)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
// Try to create the blob container
|
||||
_, err = client.CreateContainer(s.ctx, cacheInfo.Azure.ContainerName, nil)
|
||||
if err != nil {
|
||||
// TODO: Maybe there is a better way to check this?
|
||||
// docs: https://pkg.go.dev/github.com/Azure/azure-storage-blob-go/azblob
|
||||
if strings.Contains(err.Error(), "ContainerAlreadyExists") {
|
||||
// do nothing
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
s.containerName = cacheInfo.Azure.ContainerName
|
||||
s.session = client
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func (s *AzureCache) Store(key string, data string) error {
|
||||
// Store the object as a new file in the Azure blob storage with data as the content
|
||||
cacheData := []byte(data)
|
||||
@@ -45,9 +87,9 @@ func (s *AzureCache) Load(key string) (string, error) {
|
||||
return data.String(), nil
|
||||
}
|
||||
|
||||
func (s *AzureCache) List() ([]string, error) {
|
||||
func (s *AzureCache) List() ([]CacheObjectDetails, error) {
|
||||
// List the files in the blob containerName
|
||||
files := []string{}
|
||||
files := []CacheObjectDetails{}
|
||||
|
||||
pager := s.session.NewListBlobsFlatPager(s.containerName, &azblob.ListBlobsFlatOptions{
|
||||
Include: azblob.ListBlobsInclude{Snapshots: false, Versions: false},
|
||||
@@ -60,13 +102,24 @@ func (s *AzureCache) List() ([]string, error) {
|
||||
}
|
||||
|
||||
for _, blob := range resp.Segment.BlobItems {
|
||||
files = append(files, *blob.Name)
|
||||
files = append(files, CacheObjectDetails{
|
||||
Name: *blob.Name,
|
||||
UpdatedAt: *blob.Properties.LastModified,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func (s *AzureCache) Remove(key string) error {
|
||||
_, err := s.session.DeleteBlob(s.ctx, s.containerName, key, &blob.DeleteOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *AzureCache) Exists(key string) bool {
|
||||
// Check if the object exists in the blob storage
|
||||
pager := s.session.NewListBlobsFlatPager(s.containerName, &azblob.ListBlobsFlatOptions{
|
||||
@@ -93,46 +146,10 @@ func (s *AzureCache) IsCacheDisabled() bool {
|
||||
return s.noCache
|
||||
}
|
||||
|
||||
func NewAzureCache(nocache bool) ICache {
|
||||
ctx := context.Background()
|
||||
var cache CacheProvider
|
||||
err := viper.UnmarshalKey("cache", &cache)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if cache.ContainerName == "" {
|
||||
log.Fatal("Azure Container name not configured")
|
||||
}
|
||||
if cache.StorageAccount == "" {
|
||||
log.Fatal("Azure Storage account not configured")
|
||||
func (s *AzureCache) GetName() string {
|
||||
return "azure"
|
||||
}
|
||||
|
||||
// We assume that Storage account is already in place
|
||||
blobUrl := fmt.Sprintf("https://%s.blob.core.windows.net/", cache.StorageAccount)
|
||||
credential, err := azidentity.NewDefaultAzureCredential(nil)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
client, err := azblob.NewClient(blobUrl, credential, nil)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
// Try to create the blob container
|
||||
_, err = client.CreateContainer(ctx, cache.ContainerName, nil)
|
||||
if err != nil {
|
||||
// TODO: Maybe there is a better way to check this?
|
||||
// docs: https://pkg.go.dev/github.com/Azure/azure-storage-blob-go/azblob
|
||||
if strings.Contains(err.Error(), "ContainerAlreadyExists") {
|
||||
// do nothing
|
||||
} else {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
return &AzureCache{
|
||||
ctx: ctx,
|
||||
noCache: nocache,
|
||||
containerName: cache.ContainerName,
|
||||
session: client,
|
||||
}
|
||||
func (s *AzureCache) DisableCache() {
|
||||
s.noCache = true
|
||||
}
|
||||
|
||||
145
pkg/cache/cache.go
vendored
145
pkg/cache/cache.go
vendored
@@ -1,92 +1,108 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
type CacheType string
|
||||
|
||||
const (
|
||||
Azure CacheType = "azure"
|
||||
S3 CacheType = "s3"
|
||||
FileBased CacheType = "file"
|
||||
var (
|
||||
types = []ICache{
|
||||
&AzureCache{},
|
||||
&FileBasedCache{},
|
||||
&GCSCache{},
|
||||
&S3Cache{},
|
||||
}
|
||||
)
|
||||
|
||||
type ICache interface {
|
||||
Configure(cacheInfo CacheProvider) error
|
||||
Store(key string, data string) error
|
||||
Load(key string) (string, error)
|
||||
List() ([]string, error)
|
||||
List() ([]CacheObjectDetails, error)
|
||||
Remove(key string) error
|
||||
Exists(key string) bool
|
||||
IsCacheDisabled() bool
|
||||
GetName() string
|
||||
DisableCache()
|
||||
}
|
||||
|
||||
func New(noCache bool, remoteCache CacheType) ICache {
|
||||
switch remoteCache {
|
||||
case S3:
|
||||
return NewS3Cache(noCache)
|
||||
case Azure:
|
||||
return NewAzureCache(noCache)
|
||||
case FileBased:
|
||||
return &FileBasedCache{
|
||||
noCache: noCache,
|
||||
}
|
||||
default:
|
||||
return &FileBasedCache{
|
||||
noCache: noCache,
|
||||
func New(cacheType string) ICache {
|
||||
for _, t := range types {
|
||||
if cacheType == t.GetName() {
|
||||
return t
|
||||
}
|
||||
}
|
||||
return &FileBasedCache{}
|
||||
}
|
||||
|
||||
// CacheProvider is the configuration for the cache provider when using a remote cache
|
||||
type CacheProvider struct {
|
||||
BucketName string `mapstructure:"bucketname" yaml:"bucketname,omitempty"`
|
||||
Region string `mapstructure:"region" yaml:"region,omitempty"`
|
||||
StorageAccount string `mapstructure:"storageaccount" yaml:"storageaccount,omitempty"`
|
||||
ContainerName string `mapstructure:"container" yaml:"container,omitempty"`
|
||||
}
|
||||
|
||||
// NewCacheProvider constructs a new cache struct
|
||||
func NewCacheProvider(bucketname, region, storageaccount, containername string) CacheProvider {
|
||||
return CacheProvider{
|
||||
BucketName: bucketname,
|
||||
Region: region,
|
||||
StorageAccount: storageaccount,
|
||||
ContainerName: containername,
|
||||
}
|
||||
}
|
||||
|
||||
// If we have set a remote cache, return the remote cache type
|
||||
func RemoteCacheEnabled() (CacheType, error) {
|
||||
// load remote cache if it is configured
|
||||
var cache CacheProvider
|
||||
err := viper.UnmarshalKey("cache", &cache)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if cache.BucketName != "" && cache.Region != "" {
|
||||
return S3, nil
|
||||
} else if cache.StorageAccount != "" && cache.ContainerName != "" {
|
||||
return Azure, nil
|
||||
}
|
||||
return FileBased, nil
|
||||
}
|
||||
|
||||
func AddRemoteCache(cache CacheProvider) error {
|
||||
func ParseCacheConfiguration() (CacheProvider, error) {
|
||||
var cacheInfo CacheProvider
|
||||
err := viper.UnmarshalKey("cache", &cacheInfo)
|
||||
if err != nil {
|
||||
return err
|
||||
return cacheInfo, err
|
||||
}
|
||||
return cacheInfo, nil
|
||||
}
|
||||
|
||||
cacheInfo.BucketName = cache.BucketName
|
||||
cacheInfo.Region = cache.Region
|
||||
cacheInfo.StorageAccount = cache.StorageAccount
|
||||
cacheInfo.ContainerName = cache.ContainerName
|
||||
func NewCacheProvider(cacheType, bucketname, region, storageAccount, containerName, projectId string) (CacheProvider, error) {
|
||||
cProvider := CacheProvider{}
|
||||
|
||||
switch {
|
||||
case cacheType == "azure":
|
||||
cProvider.Azure.ContainerName = containerName
|
||||
cProvider.Azure.StorageAccount = storageAccount
|
||||
case cacheType == "gcs":
|
||||
cProvider.GCS.BucketName = bucketname
|
||||
cProvider.GCS.ProjectId = projectId
|
||||
cProvider.GCS.Region = region
|
||||
case cacheType == "s3":
|
||||
cProvider.S3.BucketName = bucketname
|
||||
cProvider.S3.Region = region
|
||||
default:
|
||||
return CacheProvider{}, status.Error(codes.Internal, fmt.Sprintf("%s is not a valid option", cacheType))
|
||||
}
|
||||
|
||||
cache := New(cacheType)
|
||||
err := cache.Configure(cProvider)
|
||||
if err != nil {
|
||||
return CacheProvider{}, err
|
||||
}
|
||||
return cProvider, nil
|
||||
}
|
||||
|
||||
// If we have set a remote cache, return the remote cache configuration
|
||||
func GetCacheConfiguration() (ICache, error) {
|
||||
cacheInfo, err := ParseCacheConfiguration()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var cache ICache
|
||||
|
||||
switch {
|
||||
case cacheInfo.GCS != GCSCacheConfiguration{}:
|
||||
cache = &GCSCache{}
|
||||
case cacheInfo.Azure != AzureCacheConfiguration{}:
|
||||
cache = &AzureCache{}
|
||||
case cacheInfo.S3 != S3CacheConfiguration{}:
|
||||
cache = &S3Cache{}
|
||||
default:
|
||||
cache = &FileBasedCache{}
|
||||
}
|
||||
|
||||
cache.Configure(cacheInfo)
|
||||
|
||||
return cache, nil
|
||||
}
|
||||
|
||||
func AddRemoteCache(cacheInfo CacheProvider) error {
|
||||
|
||||
viper.Set("cache", cacheInfo)
|
||||
err = viper.WriteConfig()
|
||||
|
||||
err := viper.WriteConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -99,9 +115,6 @@ func RemoveRemoteCache() error {
|
||||
if err != nil {
|
||||
return status.Error(codes.Internal, "cache unmarshal")
|
||||
}
|
||||
if cacheInfo.BucketName == "" && cacheInfo.ContainerName == "" && cacheInfo.StorageAccount == "" {
|
||||
return status.Error(codes.Internal, "no remote cache configured")
|
||||
}
|
||||
|
||||
cacheInfo = CacheProvider{}
|
||||
viper.Set("cache", cacheInfo)
|
||||
|
||||
39
pkg/cache/file_based.go
vendored
39
pkg/cache/file_based.go
vendored
@@ -15,11 +15,15 @@ type FileBasedCache struct {
|
||||
noCache bool
|
||||
}
|
||||
|
||||
func (f *FileBasedCache) Configure(cacheInfo CacheProvider) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FileBasedCache) IsCacheDisabled() bool {
|
||||
return f.noCache
|
||||
}
|
||||
|
||||
func (*FileBasedCache) List() ([]string, error) {
|
||||
func (*FileBasedCache) List() ([]CacheObjectDetails, error) {
|
||||
path, err := xdg.CacheFile("k8sgpt")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -30,9 +34,16 @@ func (*FileBasedCache) List() ([]string, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var result []string
|
||||
var result []CacheObjectDetails
|
||||
for _, file := range files {
|
||||
result = append(result, file.Name())
|
||||
info, err := file.Info()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, CacheObjectDetails{
|
||||
Name: file.Name(),
|
||||
UpdatedAt: info.ModTime(),
|
||||
})
|
||||
}
|
||||
|
||||
return result, nil
|
||||
@@ -72,6 +83,20 @@ func (*FileBasedCache) Load(key string) (string, error) {
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
func (*FileBasedCache) Remove(key string) error {
|
||||
path, err := xdg.CacheFile(filepath.Join("k8sgpt", key))
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := os.Remove(path); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (*FileBasedCache) Store(key string, data string) error {
|
||||
path, err := xdg.CacheFile(filepath.Join("k8sgpt", key))
|
||||
|
||||
@@ -81,3 +106,11 @@ func (*FileBasedCache) Store(key string, data string) error {
|
||||
|
||||
return os.WriteFile(path, []byte(data), 0600)
|
||||
}
|
||||
|
||||
func (s *FileBasedCache) GetName() string {
|
||||
return "file"
|
||||
}
|
||||
|
||||
func (s *FileBasedCache) DisableCache() {
|
||||
s.noCache = true
|
||||
}
|
||||
|
||||
133
pkg/cache/gcs_based.go
vendored
Normal file
133
pkg/cache/gcs_based.go
vendored
Normal file
@@ -0,0 +1,133 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
"google.golang.org/api/iterator"
|
||||
)
|
||||
|
||||
type GCSCache struct {
|
||||
ctx context.Context
|
||||
noCache bool
|
||||
bucketName string
|
||||
projectId string
|
||||
region string
|
||||
session *storage.Client
|
||||
}
|
||||
|
||||
type GCSCacheConfiguration struct {
|
||||
ProjectId string `mapstructure:"projectid" yaml:"projectid,omitempty"`
|
||||
Region string `mapstructure:"region" yaml:"region,omitempty"`
|
||||
BucketName string `mapstructure:"bucketname" yaml:"bucketname,omitempty"`
|
||||
}
|
||||
|
||||
func (s *GCSCache) Configure(cacheInfo CacheProvider) error {
|
||||
s.ctx = context.Background()
|
||||
if cacheInfo.GCS.BucketName == "" {
|
||||
log.Fatal("Bucket name not configured")
|
||||
}
|
||||
if cacheInfo.GCS.Region == "" {
|
||||
log.Fatal("Region not configured")
|
||||
}
|
||||
if cacheInfo.GCS.ProjectId == "" {
|
||||
log.Fatal("ProjectID not configured")
|
||||
}
|
||||
s.bucketName = cacheInfo.GCS.BucketName
|
||||
s.projectId = cacheInfo.GCS.ProjectId
|
||||
s.region = cacheInfo.GCS.Region
|
||||
storageClient, err := storage.NewClient(s.ctx)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = storageClient.Bucket(s.bucketName).Attrs(s.ctx)
|
||||
if err == storage.ErrBucketNotExist {
|
||||
err = storageClient.Bucket(s.bucketName).Create(s.ctx, s.projectId, &storage.BucketAttrs{
|
||||
Location: s.region,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
s.session = storageClient
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *GCSCache) Store(key string, data string) error {
|
||||
wc := s.session.Bucket(s.bucketName).Object(key).NewWriter(s.ctx)
|
||||
|
||||
if _, err := wc.Write([]byte(data)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := wc.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *GCSCache) Load(key string) (string, error) {
|
||||
reader, err := s.session.Bucket(s.bucketName).Object(key).NewReader(s.ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
func (s *GCSCache) Remove(key string) error {
|
||||
bucketClient := s.session.Bucket(s.bucketName)
|
||||
obj := bucketClient.Object(key)
|
||||
if err := obj.Delete(s.ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *GCSCache) List() ([]CacheObjectDetails, error) {
|
||||
var files []CacheObjectDetails
|
||||
|
||||
items := s.session.Bucket(s.bucketName).Objects(s.ctx, nil)
|
||||
for {
|
||||
attrs, err := items.Next()
|
||||
if err == iterator.Done {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, CacheObjectDetails{
|
||||
Name: attrs.Name,
|
||||
UpdatedAt: attrs.Updated,
|
||||
})
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func (s *GCSCache) Exists(key string) bool {
|
||||
obj := s.session.Bucket(s.bucketName).Object(key)
|
||||
_, err := obj.Attrs(s.ctx)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (s *GCSCache) IsCacheDisabled() bool {
|
||||
return s.noCache
|
||||
}
|
||||
|
||||
func (s *GCSCache) GetName() string {
|
||||
return "gcs"
|
||||
}
|
||||
|
||||
func (s *GCSCache) DisableCache() {
|
||||
s.noCache = true
|
||||
}
|
||||
101
pkg/cache/s3_based.go
vendored
101
pkg/cache/s3_based.go
vendored
@@ -7,7 +7,6 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// Generate ICache implementation
|
||||
@@ -17,6 +16,45 @@ type S3Cache struct {
|
||||
session *s3.S3
|
||||
}
|
||||
|
||||
type S3CacheConfiguration struct {
|
||||
Region string `mapstructure:"region" yaml:"region,omitempty"`
|
||||
BucketName string `mapstructure:"bucketname" yaml:"bucketname,omitempty"`
|
||||
}
|
||||
|
||||
func (s *S3Cache) Configure(cacheInfo CacheProvider) error {
|
||||
if cacheInfo.S3.BucketName == "" {
|
||||
log.Fatal("Bucket name not configured")
|
||||
}
|
||||
if cacheInfo.S3.Region == "" {
|
||||
log.Fatal("Region not configured")
|
||||
}
|
||||
s.bucketName = cacheInfo.S3.BucketName
|
||||
|
||||
sess := session.Must(session.NewSessionWithOptions(session.Options{
|
||||
SharedConfigState: session.SharedConfigEnable,
|
||||
Config: aws.Config{
|
||||
Region: aws.String(cacheInfo.S3.Region),
|
||||
},
|
||||
}))
|
||||
|
||||
s3Client := s3.New(sess)
|
||||
|
||||
// Check if the bucket exists, if not create it
|
||||
_, err := s3Client.HeadBucket(&s3.HeadBucketInput{
|
||||
Bucket: aws.String(cacheInfo.S3.BucketName),
|
||||
})
|
||||
if err != nil {
|
||||
_, err = s3Client.CreateBucket(&s3.CreateBucketInput{
|
||||
Bucket: aws.String(cacheInfo.S3.BucketName),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
s.session = s3Client
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *S3Cache) Store(key string, data string) error {
|
||||
// Store the object as a new file in the bucket with data as the content
|
||||
_, err := s.session.PutObject(&s3.PutObjectInput{
|
||||
@@ -28,6 +66,18 @@ func (s *S3Cache) Store(key string, data string) error {
|
||||
|
||||
}
|
||||
|
||||
func (s *S3Cache) Remove(key string) error {
|
||||
_, err := s.session.DeleteObject(&s3.DeleteObjectInput{
|
||||
Bucket: &s.bucketName,
|
||||
Key: aws.String(key),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *S3Cache) Load(key string) (string, error) {
|
||||
|
||||
// Retrieve the object from the bucket and load it into a string
|
||||
@@ -45,7 +95,7 @@ func (s *S3Cache) Load(key string) (string, error) {
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
func (s *S3Cache) List() ([]string, error) {
|
||||
func (s *S3Cache) List() ([]CacheObjectDetails, error) {
|
||||
|
||||
// List the files in the bucket
|
||||
result, err := s.session.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: aws.String(s.bucketName)})
|
||||
@@ -53,9 +103,12 @@ func (s *S3Cache) List() ([]string, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var keys []string
|
||||
var keys []CacheObjectDetails
|
||||
for _, item := range result.Contents {
|
||||
keys = append(keys, *item.Key)
|
||||
keys = append(keys, CacheObjectDetails{
|
||||
Name: *item.Key,
|
||||
UpdatedAt: *item.LastModified,
|
||||
})
|
||||
}
|
||||
|
||||
return keys, nil
|
||||
@@ -75,42 +128,10 @@ func (s *S3Cache) IsCacheDisabled() bool {
|
||||
return s.noCache
|
||||
}
|
||||
|
||||
func NewS3Cache(nocache bool) ICache {
|
||||
|
||||
var cache CacheProvider
|
||||
err := viper.UnmarshalKey("cache", &cache)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if cache.BucketName == "" {
|
||||
log.Fatal("Bucket name not configured")
|
||||
}
|
||||
if cache.Region == "" {
|
||||
log.Fatal("Region not configured")
|
||||
func (s *S3Cache) GetName() string {
|
||||
return "s3"
|
||||
}
|
||||
|
||||
sess := session.Must(session.NewSessionWithOptions(session.Options{
|
||||
SharedConfigState: session.SharedConfigEnable,
|
||||
Config: aws.Config{
|
||||
Region: aws.String(cache.Region),
|
||||
},
|
||||
}))
|
||||
|
||||
s := s3.New(sess)
|
||||
|
||||
// Check if the bucket exists, if not create it
|
||||
_, err = s.HeadBucket(&s3.HeadBucketInput{
|
||||
Bucket: aws.String(cache.BucketName),
|
||||
})
|
||||
if err != nil {
|
||||
_, _ = s.CreateBucket(&s3.CreateBucketInput{
|
||||
Bucket: aws.String(cache.BucketName),
|
||||
})
|
||||
}
|
||||
|
||||
return &S3Cache{
|
||||
noCache: nocache,
|
||||
session: s,
|
||||
bucketName: cache.BucketName,
|
||||
}
|
||||
func (s *S3Cache) DisableCache() {
|
||||
s.noCache = true
|
||||
}
|
||||
|
||||
14
pkg/cache/types.go
vendored
Normal file
14
pkg/cache/types.go
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
package cache
|
||||
|
||||
import "time"
|
||||
|
||||
type CacheProvider struct {
|
||||
GCS GCSCacheConfiguration `mapstructucre:"gcs" yaml:"gcs,omitempty"`
|
||||
Azure AzureCacheConfiguration `mapstructucre:"azure" yaml:"azure,omitempty"`
|
||||
S3 S3CacheConfiguration `mapstructucre:"s3" yaml:"s3,omitempty"`
|
||||
}
|
||||
|
||||
type CacheObjectDetails struct {
|
||||
Name string
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
@@ -18,17 +18,28 @@ func (h *handler) AddConfig(ctx context.Context, i *schemav1.AddConfigRequest) (
|
||||
}
|
||||
|
||||
if i.Cache != nil {
|
||||
// We check if we have a mixed cache configuration
|
||||
CacheConfigured := (i.Cache.Region == "" && i.Cache.BucketName == "") || (i.Cache.ContainerName == "" && i.Cache.StorageAccount == "")
|
||||
if !CacheConfigured {
|
||||
return resp, status.Error(codes.InvalidArgument, "mixed cache arguments")
|
||||
var err error
|
||||
var remoteCache cache.CacheProvider
|
||||
|
||||
switch i.Cache.GetCacheType().(type) {
|
||||
case *schemav1.Cache_AzureCache:
|
||||
remoteCache, err = cache.NewCacheProvider("azure", "", "", i.Cache.GetAzureCache().StorageAccount, i.Cache.GetAzureCache().ContainerName, "")
|
||||
case *schemav1.Cache_S3Cache:
|
||||
remoteCache, err = cache.NewCacheProvider("s3", i.Cache.GetS3Cache().BucketName, i.Cache.GetS3Cache().Region, "", "", "")
|
||||
case *schemav1.Cache_GcsCache:
|
||||
remoteCache, err = cache.NewCacheProvider("gcs", i.Cache.GetGcsCache().BucketName, i.Cache.GetGcsCache().Region, "", "", i.Cache.GetGcsCache().GetProjectId())
|
||||
default:
|
||||
return resp, status.Error(codes.InvalidArgument, "Invalid cache configuration")
|
||||
}
|
||||
|
||||
cacheProvider := cache.NewCacheProvider(i.Cache.BucketName, i.Cache.Region, i.Cache.StorageAccount, i.Cache.ContainerName)
|
||||
err := cache.AddRemoteCache(cacheProvider)
|
||||
if err != nil {
|
||||
return resp, err
|
||||
}
|
||||
err = cache.AddRemoteCache(remoteCache)
|
||||
if err != nil {
|
||||
return resp, err
|
||||
}
|
||||
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user