Merge branch 'main' into feat/add-amazonbedrock

This commit is contained in:
Aris Boutselis
2023-10-23 08:47:17 +03:00
committed by GitHub
9 changed files with 268 additions and 45 deletions

View File

@@ -485,18 +485,22 @@ Config file locations:
<details> <details>
There may be scenarios where caching remotely is preferred. There may be scenarios where caching remotely is preferred.
In these scenarios K8sGPT supports AWS S3 Integration. In these scenarios K8sGPT supports AWS S3 or Azure Blob storage Integration.
<summary> Remote caching </summary> <summary> Remote caching </summary>
<em>Note: You can only configure and use only one remote cache at a time</em>
_As a prerequisite `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` are required as environmental variables._
_Adding a remote cache_ _Adding a remote cache_
Note: this will create the bucket if it does not exist * AWS S3
``` * _As a prerequisite `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` are required as environmental variables._
k8sgpt cache add --region <aws region> --bucket <name> * Configuration, ``` k8sgpt cache add --region <aws region> --bucket <name> ```
``` * K8sGPT will create the bucket if it does not exist
* Azure Storage
* We support a number of [techniques](https://learn.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication?tabs=bash#2-authenticate-with-azure) to authenticate against Azure
* Configuration, ``` k8sgpt cache add --storageacc <storage account name> --container <container name> ```
* K8sGPT assumes that the storage account already exist and it will create the container if it does not exist
* It's **users'** responsibility have to grant specific permissions to their identity in order to be able to upload blob files and create SA containers (e.g Storage Blob Data Contributor)
_Listing cache items_ _Listing cache items_
``` ```
@@ -504,9 +508,9 @@ k8sgpt cache list
``` ```
_Removing the remote cache_ _Removing the remote cache_
Note: this will not delete the bucket Note: this will not delete the upstream S3 bucket or Azure storage container
``` ```
k8sgpt cache remove --bucket <name> k8sgpt cache remove
``` ```
</details> </details>

27
cmd/cache/add.go vendored
View File

@@ -24,7 +24,10 @@ import (
) )
var ( var (
region string region string
bucketName string
storageAccount string
containerName string
) )
// addCmd represents the add command // addCmd represents the add command
@@ -33,10 +36,12 @@ var addCmd = &cobra.Command{
Short: "Add a remote cache", Short: "Add a remote cache",
Long: `This command allows you to add a remote cache to store the results of an analysis. Long: `This command allows you to add a remote cache to store the results of an analysis.
The supported cache types are: The supported cache types are:
- Azure Blob storage
- S3`, - S3`,
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
fmt.Println(color.YellowString("Adding remote S3 based cache")) fmt.Println(color.YellowString("Adding remote based cache"))
err := cache.AddRemoteCache(bucketname, region) remoteCache := cache.NewCacheProvider(bucketname, region, storageAccount, containerName)
err := cache.AddRemoteCache(remoteCache)
if err != nil { if err != nil {
color.Red("Error: %v", err) color.Red("Error: %v", err)
os.Exit(1) os.Exit(1)
@@ -46,9 +51,15 @@ var addCmd = &cobra.Command{
func init() { func init() {
CacheCmd.AddCommand(addCmd) CacheCmd.AddCommand(addCmd)
addCmd.Flags().StringVarP(&region, "region", "r", "", "The region to use for the cache") addCmd.Flags().StringVarP(&region, "region", "r", "", "The region to use for the AWS S3 cache")
addCmd.Flags().StringVarP(&bucketname, "bucket", "b", "", "The name of the bucket to use for the cache") addCmd.Flags().StringVarP(&bucketname, "bucket", "b", "", "The name of the AWS S3 bucket to use for the cache")
addCmd.MarkFlagRequired("bucket") addCmd.MarkFlagsRequiredTogether("region", "bucket")
addCmd.MarkFlagRequired("region") addCmd.Flags().StringVarP(&storageAccount, "storageacc", "s", "", "The Azure storage account name of the container")
addCmd.Flags().StringVarP(&containerName, "container", "c", "", "The Azure container name to use for the cache")
addCmd.MarkFlagsRequiredTogether("storageacc", "container")
// Tedious check to ensure we don't include arguments from different providers
addCmd.MarkFlagsMutuallyExclusive("region", "storageacc")
addCmd.MarkFlagsMutuallyExclusive("region", "container")
addCmd.MarkFlagsMutuallyExclusive("bucket", "storageacc")
addCmd.MarkFlagsMutuallyExclusive("bucket", "container")
} }

2
cmd/cache/remove.go vendored
View File

@@ -29,7 +29,7 @@ var removeCmd = &cobra.Command{
Long: `This command allows you to remove the remote cache and use the default filecache.`, Long: `This command allows you to remove the remote cache and use the default filecache.`,
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
err := cache.RemoveRemoteCache(bucketname) err := cache.RemoveRemoteCache()
if err != nil { if err != nil {
color.Red("Error: %v", err) color.Red("Error: %v", err)
os.Exit(1) os.Exit(1)

8
go.mod
View File

@@ -26,17 +26,25 @@ require github.com/adrg/xdg v0.4.0
require ( require (
buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231002095256-194bc640518b.1 buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231002095256-194bc640518b.1
buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20231002095256-194bc640518b.1 buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20231002095256-194bc640518b.1
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0
github.com/aws/aws-sdk-go v1.45.26 github.com/aws/aws-sdk-go v1.45.26
github.com/cohere-ai/cohere-go v0.2.0 github.com/cohere-ai/cohere-go v0.2.0
) )
require ( require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
github.com/Microsoft/hcsshim v0.11.0 // indirect github.com/Microsoft/hcsshim v0.11.0 // indirect
github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect
github.com/cohere-ai/tokenizer v1.1.1 // indirect github.com/cohere-ai/tokenizer v1.1.1 // indirect
github.com/dlclark/regexp2 v1.4.0 // indirect github.com/dlclark/regexp2 v1.4.0 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
github.com/sagikazarmark/locafero v0.3.0 // indirect github.com/sagikazarmark/locafero v0.3.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect

25
go.sum
View File

@@ -1,6 +1,8 @@
buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231002095256-194bc640518b.1 h1:xYEAhcdWh89HNtbM5Uv4b2xu+4/MkNffR9JNrnnEjXU= buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231002095256-194bc640518b.1 h1:xYEAhcdWh89HNtbM5Uv4b2xu+4/MkNffR9JNrnnEjXU=
buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231002095256-194bc640518b.1/go.mod h1:j2QJ3L7VTtI+VeK6t03h9X934FolVTb3FwXUc76bQMQ= buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231002095256-194bc640518b.1/go.mod h1:j2QJ3L7VTtI+VeK6t03h9X934FolVTb3FwXUc76bQMQ=
buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.28.1-20231002095256-194bc640518b.4/go.mod h1:i/s4ALHwKvjA1oGNKpoHg0FpEOTbufoOm/NdTE6YQAE= buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.28.1-20231002095256-194bc640518b.4/go.mod h1:i/s4ALHwKvjA1oGNKpoHg0FpEOTbufoOm/NdTE6YQAE=
buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20230927080702-a2be8a73637d.1 h1:Snnz9mUZNxMFpd+l5m1zaVdIVAplVmdFxYVn5/f4UoI=
buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20230927080702-a2be8a73637d.1/go.mod h1:gtnk2yAUexdY5nTuUg0SH5WCCGvpKzr7pd3Xbi7MWjE=
buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20231002095256-194bc640518b.1 h1:Bt8mnCodD/BqChxt/r3xYayGLoOAn334qC1tN7VqUTE= buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20231002095256-194bc640518b.1 h1:Bt8mnCodD/BqChxt/r3xYayGLoOAn334qC1tN7VqUTE=
buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20231002095256-194bc640518b.1/go.mod h1:gtnk2yAUexdY5nTuUg0SH5WCCGvpKzr7pd3Xbi7MWjE= buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20231002095256-194bc640518b.1/go.mod h1:gtnk2yAUexdY5nTuUg0SH5WCCGvpKzr7pd3Xbi7MWjE=
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
@@ -601,8 +603,22 @@ gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zum
git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY=
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
@@ -722,6 +738,7 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc
github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc=
github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/docker/cli v23.0.5+incompatible h1:ufWmAOuD3Vmr7JP2G5K3cyuNC4YZWiAsuDEvFVVDafE= github.com/docker/cli v23.0.5+incompatible h1:ufWmAOuD3Vmr7JP2G5K3cyuNC4YZWiAsuDEvFVVDafE=
github.com/docker/cli v23.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v23.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
@@ -818,6 +835,9 @@ github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
@@ -1007,6 +1027,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw=
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=
@@ -1103,6 +1125,8 @@ github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk
github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -1486,6 +1510,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

138
pkg/cache/azuresa_based.go vendored Normal file
View File

@@ -0,0 +1,138 @@
package cache
import (
"bytes"
"context"
"fmt"
"log"
"strings"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/spf13/viper"
)
// Generate ICache implementation
type AzureCache struct {
ctx context.Context
noCache bool
containerName string
session *azblob.Client
}
func (s *AzureCache) Store(key string, data string) error {
// Store the object as a new file in the Azure blob storage with data as the content
cacheData := []byte(data)
_, err := s.session.UploadBuffer(s.ctx, s.containerName, key, cacheData, &azblob.UploadBufferOptions{})
return err
}
func (s *AzureCache) Load(key string) (string, error) {
// Load blob file contents
load, err := s.session.DownloadStream(s.ctx, s.containerName, key, nil)
if err != nil {
return "", err
}
data := bytes.Buffer{}
retryReader := load.NewRetryReader(s.ctx, &azblob.RetryReaderOptions{})
_, err = data.ReadFrom(retryReader)
if err != nil {
return "", err
}
if err := retryReader.Close(); err != nil {
return "", err
}
return data.String(), nil
}
func (s *AzureCache) List() ([]string, error) {
// List the files in the blob containerName
files := []string{}
pager := s.session.NewListBlobsFlatPager(s.containerName, &azblob.ListBlobsFlatOptions{
Include: azblob.ListBlobsInclude{Snapshots: false, Versions: false},
})
for pager.More() {
resp, err := pager.NextPage(s.ctx)
if err != nil {
return nil, err
}
for _, blob := range resp.Segment.BlobItems {
files = append(files, *blob.Name)
}
}
return files, nil
}
func (s *AzureCache) Exists(key string) bool {
// Check if the object exists in the blob storage
pager := s.session.NewListBlobsFlatPager(s.containerName, &azblob.ListBlobsFlatOptions{
Include: azblob.ListBlobsInclude{Snapshots: false, Versions: false},
})
for pager.More() {
resp, err := pager.NextPage(s.ctx)
if err != nil {
return false
}
for _, blob := range resp.Segment.BlobItems {
if *blob.Name == key {
return true
}
}
}
return false
}
func (s *AzureCache) IsCacheDisabled() bool {
return s.noCache
}
func NewAzureCache(nocache bool) ICache {
ctx := context.Background()
var cache CacheProvider
err := viper.UnmarshalKey("cache", &cache)
if err != nil {
panic(err)
}
if cache.ContainerName == "" {
log.Fatal("Azure Container name not configured")
}
if cache.StorageAccount == "" {
log.Fatal("Azure Storage account not configured")
}
// We assume that Storage account is already in place
blobUrl := fmt.Sprintf("https://%s.blob.core.windows.net/", cache.StorageAccount)
credential, err := azidentity.NewDefaultAzureCredential(nil)
if err != nil {
log.Fatal(err)
}
client, err := azblob.NewClient(blobUrl, credential, nil)
if err != nil {
log.Fatal(err)
}
// Try to create the blob container
_, err = client.CreateContainer(ctx, cache.ContainerName, nil)
if err != nil {
// TODO: Maybe there is a better way to check this?
// docs: https://pkg.go.dev/github.com/Azure/azure-storage-blob-go/azblob
if strings.Contains(err.Error(), "ContainerAlreadyExists") {
// do nothing
} else {
log.Fatal(err)
}
}
return &AzureCache{
ctx: ctx,
noCache: nocache,
containerName: cache.ContainerName,
session: client,
}
}

67
pkg/cache/cache.go vendored
View File

@@ -7,6 +7,14 @@ import (
"github.com/spf13/viper" "github.com/spf13/viper"
) )
type CacheType string
const (
Azure CacheType = "azure"
S3 CacheType = "s3"
FileBased CacheType = "file"
)
type ICache interface { type ICache interface {
Store(key string, data string) error Store(key string, data string) error
Load(key string) (string, error) Load(key string) (string, error)
@@ -15,43 +23,68 @@ type ICache interface {
IsCacheDisabled() bool IsCacheDisabled() bool
} }
func New(noCache bool, remoteCache bool) ICache { func New(noCache bool, remoteCache CacheType) ICache {
if remoteCache { switch remoteCache {
case S3:
return NewS3Cache(noCache) return NewS3Cache(noCache)
} case Azure:
return &FileBasedCache{ return NewAzureCache(noCache)
noCache: noCache, case FileBased:
return &FileBasedCache{
noCache: noCache,
}
default:
return &FileBasedCache{
noCache: noCache,
}
} }
} }
// CacheProvider is the configuration for the cache provider when using a remote cache // CacheProvider is the configuration for the cache provider when using a remote cache
type CacheProvider struct { type CacheProvider struct {
BucketName string `mapstructure:"bucketname"` BucketName string `mapstructure:"bucketname" yaml:"bucketname,omitempty"`
Region string `mapstructure:"region"` Region string `mapstructure:"region" yaml:"region,omitempty"`
StorageAccount string `mapstructure:"storageaccount" yaml:"storageaccount,omitempty"`
ContainerName string `mapstructure:"container" yaml:"container,omitempty"`
} }
func RemoteCacheEnabled() (bool, error) { // NewCacheProvider constructs a new cache struct
func NewCacheProvider(bucketname, region, storageaccount, containername string) CacheProvider {
return CacheProvider{
BucketName: bucketname,
Region: region,
StorageAccount: storageaccount,
ContainerName: containername,
}
}
// If we have set a remote cache, return the remote cache type
func RemoteCacheEnabled() (CacheType, error) {
// load remote cache if it is configured // load remote cache if it is configured
var cache CacheProvider var cache CacheProvider
err := viper.UnmarshalKey("cache", &cache) err := viper.UnmarshalKey("cache", &cache)
if err != nil { if err != nil {
return false, err return "", err
} }
if cache.BucketName != "" && cache.Region != "" { if cache.BucketName != "" && cache.Region != "" {
return true, nil return S3, nil
} else if cache.StorageAccount != "" && cache.ContainerName != "" {
return Azure, nil
} }
return false, nil return FileBased, nil
} }
func AddRemoteCache(bucketName string, region string) error { func AddRemoteCache(cache CacheProvider) error {
var cacheInfo CacheProvider var cacheInfo CacheProvider
err := viper.UnmarshalKey("cache", &cacheInfo) err := viper.UnmarshalKey("cache", &cacheInfo)
if err != nil { if err != nil {
return err return err
} }
cacheInfo.BucketName = bucketName cacheInfo.BucketName = cache.BucketName
cacheInfo.Region = region cacheInfo.Region = cache.Region
cacheInfo.StorageAccount = cache.StorageAccount
cacheInfo.ContainerName = cache.ContainerName
viper.Set("cache", cacheInfo) viper.Set("cache", cacheInfo)
err = viper.WriteConfig() err = viper.WriteConfig()
if err != nil { if err != nil {
@@ -60,14 +93,14 @@ func AddRemoteCache(bucketName string, region string) error {
return nil return nil
} }
func RemoveRemoteCache(bucketName string) error { func RemoveRemoteCache() error {
var cacheInfo CacheProvider var cacheInfo CacheProvider
err := viper.UnmarshalKey("cache", &cacheInfo) err := viper.UnmarshalKey("cache", &cacheInfo)
if err != nil { if err != nil {
return status.Error(codes.Internal, "cache unmarshal") return status.Error(codes.Internal, "cache unmarshal")
} }
if cacheInfo.BucketName == "" { if cacheInfo.BucketName == "" && cacheInfo.ContainerName == "" && cacheInfo.StorageAccount == "" {
return status.Error(codes.Internal, "no cache configured") return status.Error(codes.Internal, "no remote cache configured")
} }
cacheInfo = CacheProvider{} cacheInfo = CacheProvider{}

View File

@@ -2,6 +2,7 @@ package cache
import ( import (
"bytes" "bytes"
"log"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/aws/session"
@@ -79,13 +80,13 @@ func NewS3Cache(nocache bool) ICache {
var cache CacheProvider var cache CacheProvider
err := viper.UnmarshalKey("cache", &cache) err := viper.UnmarshalKey("cache", &cache)
if err != nil { if err != nil {
panic(err) log.Fatal(err)
} }
if cache.BucketName == "" { if cache.BucketName == "" {
panic("Bucket name not configured") log.Fatal("Bucket name not configured")
} }
if cache.Region == "" { if cache.Region == "" {
panic("Region not configured") log.Fatal("Region not configured")
} }
sess := session.Must(session.NewSessionWithOptions(session.Options{ sess := session.Must(session.NewSessionWithOptions(session.Options{

View File

@@ -1,8 +1,9 @@
package server package server
import ( import (
schemav1 "buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go/schema/v1"
"context" "context"
schemav1 "buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go/schema/v1"
"github.com/k8sgpt-ai/k8sgpt/pkg/cache" "github.com/k8sgpt-ai/k8sgpt/pkg/cache"
"google.golang.org/grpc/codes" "google.golang.org/grpc/codes"
"google.golang.org/grpc/status" "google.golang.org/grpc/status"
@@ -17,12 +18,14 @@ func (h *handler) AddConfig(ctx context.Context, i *schemav1.AddConfigRequest) (
} }
if i.Cache != nil { if i.Cache != nil {
// Remote cache // We check if we have a mixed cache configuration
if i.Cache.BucketName == "" || i.Cache.Region == "" { CacheConfigured := (i.Cache.Region == "" && i.Cache.BucketName == "") || (i.Cache.ContainerName == "" && i.Cache.StorageAccount == "")
return resp, status.Error(codes.InvalidArgument, "cache arguments") if !CacheConfigured {
return resp, status.Error(codes.InvalidArgument, "mixed cache arguments")
} }
err := cache.AddRemoteCache(i.Cache.BucketName, i.Cache.Region) cacheProvider := cache.NewCacheProvider(i.Cache.BucketName, i.Cache.Region, i.Cache.StorageAccount, i.Cache.ContainerName)
err := cache.AddRemoteCache(cacheProvider)
if err != nil { if err != nil {
return resp, err return resp, err
} }
@@ -32,7 +35,7 @@ func (h *handler) AddConfig(ctx context.Context, i *schemav1.AddConfigRequest) (
func (h *handler) RemoveConfig(ctx context.Context, i *schemav1.RemoveConfigRequest) (*schemav1.RemoveConfigResponse, error, func (h *handler) RemoveConfig(ctx context.Context, i *schemav1.RemoveConfigRequest) (*schemav1.RemoveConfigResponse, error,
) { ) {
err := cache.RemoveRemoteCache(i.Cache.BucketName) err := cache.RemoveRemoteCache()
if err != nil { if err != nil {
return &schemav1.RemoveConfigResponse{}, err return &schemav1.RemoveConfigResponse{}, err
} }