From 928f5804e3b48a4e3057fc383339760c43ed4d7a Mon Sep 17 00:00:00 2001 From: Antoine Pelisse Date: Tue, 8 Aug 2017 15:00:23 -0700 Subject: [PATCH] Revert "Revert "Merge pull request #47353 from apelisse/http-cache"" This reverts commit 4ee72eb300423772020dd1cf208159058ba7dab5. Kubernetes-commit: 332b681bd1d961e2cee16bca10784088a8d308f1 --- Godeps/Godeps.json | 16 +++++++++ rest/config.go | 4 +++ rest/config_test.go | 1 + rest/transport.go | 1 + tools/clientcmd/client_config.go | 5 +++ tools/clientcmd/overrides.go | 8 +++++ transport/BUILD | 3 ++ transport/config.go | 4 +++ transport/round_trippers.go | 17 +++++++++ transport/round_trippers_test.go | 61 ++++++++++++++++++++++++++++++++ 10 files changed, 120 insertions(+) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 1eccbc05..a0425998 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -154,6 +154,10 @@ "ImportPath": "github.com/golang/protobuf/ptypes/timestamp", "Rev": "4bd1920723d7b7c925de087aa32e2187708897f7" }, + { + "ImportPath": "github.com/google/btree", + "Rev": "7d79101e329e5a3adf994758c578dab82b90c017" + }, { "ImportPath": "github.com/google/gofuzz", "Rev": "44d81051d367757e1c7c6a5a86423ece9afcf63c" @@ -198,6 +202,14 @@ "ImportPath": "github.com/gophercloud/gophercloud/pagination", "Rev": "c0406a133c4a74a838baf0ddff3c2fab21155fba" }, + { + "ImportPath": "github.com/gregjones/httpcache", + "Rev": "787624de3eb7bd915c329cba748687a3b22666a6" + }, + { + "ImportPath": "github.com/gregjones/httpcache/diskcache", + "Rev": "787624de3eb7bd915c329cba748687a3b22666a6" + }, { "ImportPath": "github.com/hashicorp/golang-lru", "Rev": "a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4" @@ -234,6 +246,10 @@ "ImportPath": "github.com/mailru/easyjson/jwriter", "Rev": "d5b7844b561a7bc640052f1b935f7b800330d7e0" }, + { + "ImportPath": "github.com/peterbourgon/diskv", + "Rev": "5dfcb07a075adbaaa4094cddfd160b1e1c77a043" + }, { "ImportPath": "github.com/pmezard/go-difflib/difflib", "Rev": "d8ed2627bdf02c080bf22230dbb337003b7aba2d" diff --git a/rest/config.go b/rest/config.go index dca7333d..627a9cc9 100644 --- a/rest/config.go +++ b/rest/config.go @@ -71,6 +71,10 @@ type Config struct { // TODO: demonstrate an OAuth2 compatible client. BearerToken string + // CacheDir is the directory where we'll store HTTP cached responses. + // If set to empty string, no caching mechanism will be used. + CacheDir string + // Impersonate is the configuration that RESTClient will use for impersonation. Impersonate ImpersonationConfig diff --git a/rest/config_test.go b/rest/config_test.go index f04135a4..f20ed722 100644 --- a/rest/config_test.go +++ b/rest/config_test.go @@ -249,6 +249,7 @@ func TestAnonymousConfig(t *testing.T) { expected.BearerToken = "" expected.Username = "" expected.Password = "" + expected.CacheDir = "" expected.AuthProvider = nil expected.AuthConfigPersister = nil expected.TLSClientConfig.CertData = nil diff --git a/rest/transport.go b/rest/transport.go index ba43752b..4c5b1648 100644 --- a/rest/transport.go +++ b/rest/transport.go @@ -89,6 +89,7 @@ func (c *Config) TransportConfig() (*transport.Config, error) { }, Username: c.Username, Password: c.Password, + CacheDir: c.CacheDir, BearerToken: c.BearerToken, Impersonate: transport.ImpersonationConfig{ UserName: c.Impersonate.UserName, diff --git a/tools/clientcmd/client_config.go b/tools/clientcmd/client_config.go index a8698af2..9646c6b7 100644 --- a/tools/clientcmd/client_config.go +++ b/tools/clientcmd/client_config.go @@ -22,6 +22,7 @@ import ( "io/ioutil" "net/url" "os" + "path/filepath" "strings" "github.com/golang/glog" @@ -31,16 +32,19 @@ import ( restclient "k8s.io/client-go/rest" clientauth "k8s.io/client-go/tools/auth" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/client-go/util/homedir" ) var ( // ClusterDefaults has the same behavior as the old EnvVar and DefaultCluster fields // DEPRECATED will be replaced ClusterDefaults = clientcmdapi.Cluster{Server: getDefaultServer()} + cacheDirDefault = filepath.Join(homedir.HomeDir(), ".kube", "http-cache") // DefaultClientConfig represents the legacy behavior of this package for defaulting // DEPRECATED will be replace DefaultClientConfig = DirectClientConfig{*clientcmdapi.NewConfig(), "", &ConfigOverrides{ ClusterDefaults: ClusterDefaults, + CacheDir: cacheDirDefault, }, nil, NewDefaultClientConfigLoadingRules(), promptedCredentials{}} ) @@ -131,6 +135,7 @@ func (config *DirectClientConfig) ClientConfig() (*restclient.Config, error) { clientConfig := &restclient.Config{} clientConfig.Host = configClusterInfo.Server + clientConfig.CacheDir = config.overrides.CacheDir if len(config.overrides.Timeout) > 0 { timeout, err := ParseTimeout(config.overrides.Timeout) diff --git a/tools/clientcmd/overrides.go b/tools/clientcmd/overrides.go index 963ac8fa..25ab1ea1 100644 --- a/tools/clientcmd/overrides.go +++ b/tools/clientcmd/overrides.go @@ -17,11 +17,13 @@ limitations under the License. package clientcmd import ( + "path/filepath" "strconv" "github.com/spf13/pflag" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/client-go/util/homedir" ) // ConfigOverrides holds values that should override whatever information is pulled from the actual Config object. You can't @@ -34,6 +36,7 @@ type ConfigOverrides struct { Context clientcmdapi.Context CurrentContext string Timeout string + CacheDir string } // ConfigOverrideFlags holds the flag names to be used for binding command line flags. Notice that this structure tightly @@ -44,6 +47,7 @@ type ConfigOverrideFlags struct { ContextOverrideFlags ContextOverrideFlags CurrentContext FlagInfo Timeout FlagInfo + CacheDir FlagInfo } // AuthOverrideFlags holds the flag names to be used for binding command line flags for AuthInfo objects @@ -146,10 +150,12 @@ const ( FlagUsername = "username" FlagPassword = "password" FlagTimeout = "request-timeout" + FlagCacheDir = "cachedir" ) // RecommendedConfigOverrideFlags is a convenience method to return recommended flag names prefixed with a string of your choosing func RecommendedConfigOverrideFlags(prefix string) ConfigOverrideFlags { + defaultCacheDir := filepath.Join(homedir.HomeDir(), ".kube", "http-cache") return ConfigOverrideFlags{ AuthOverrideFlags: RecommendedAuthOverrideFlags(prefix), ClusterOverrideFlags: RecommendedClusterOverrideFlags(prefix), @@ -157,6 +163,7 @@ func RecommendedConfigOverrideFlags(prefix string) ConfigOverrideFlags { CurrentContext: FlagInfo{prefix + FlagContext, "", "", "The name of the kubeconfig context to use"}, Timeout: FlagInfo{prefix + FlagTimeout, "", "0", "The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests."}, + CacheDir: FlagInfo{prefix + FlagCacheDir, "", defaultCacheDir, "Path to http-cache directory"}, } } @@ -198,6 +205,7 @@ func BindOverrideFlags(overrides *ConfigOverrides, flags *pflag.FlagSet, flagNam BindContextFlags(&overrides.Context, flags, flagNames.ContextOverrideFlags) flagNames.CurrentContext.BindStringFlag(flags, &overrides.CurrentContext) flagNames.Timeout.BindStringFlag(flags, &overrides.Timeout) + flagNames.CacheDir.BindStringFlag(flags, &overrides.CacheDir) } // BindAuthInfoFlags is a convenience method to bind the specified flags to their associated variables diff --git a/transport/BUILD b/transport/BUILD index 56b6256c..c03ed48c 100644 --- a/transport/BUILD +++ b/transport/BUILD @@ -26,6 +26,9 @@ go_library( ], deps = [ "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/gregjones/httpcache:go_default_library", + "//vendor/github.com/gregjones/httpcache/diskcache:go_default_library", + "//vendor/github.com/peterbourgon/diskv:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", ], ) diff --git a/transport/config.go b/transport/config.go index 820594ba..e34d6e8c 100644 --- a/transport/config.go +++ b/transport/config.go @@ -34,6 +34,10 @@ type Config struct { // Bearer token for authentication BearerToken string + // CacheDir is the directory where we'll store HTTP cached responses. + // If set to empty string, no caching mechanism will be used. + CacheDir string + // Impersonate is the config that this Config will impersonate using Impersonate ImpersonationConfig diff --git a/transport/round_trippers.go b/transport/round_trippers.go index c728b187..2394c42c 100644 --- a/transport/round_trippers.go +++ b/transport/round_trippers.go @@ -23,6 +23,9 @@ import ( "time" "github.com/golang/glog" + "github.com/gregjones/httpcache" + "github.com/gregjones/httpcache/diskcache" + "github.com/peterbourgon/diskv" utilnet "k8s.io/apimachinery/pkg/util/net" ) @@ -56,6 +59,9 @@ func HTTPWrappersForConfig(config *Config, rt http.RoundTripper) (http.RoundTrip len(config.Impersonate.Extra) > 0 { rt = NewImpersonatingRoundTripper(config.Impersonate, rt) } + if len(config.CacheDir) > 0 { + rt = NewCacheRoundTripper(config.CacheDir, rt) + } return rt, nil } @@ -87,6 +93,17 @@ type authProxyRoundTripper struct { rt http.RoundTripper } +// NewCacheRoundTripper creates a roundtripper that reads the ETag on +// response headers and send the If-None-Match header on subsequent +// corresponding requests. +func NewCacheRoundTripper(cacheDir string, rt http.RoundTripper) http.RoundTripper { + d := diskv.New(diskv.Options{BasePath: cacheDir}) + t := httpcache.NewTransport(diskcache.NewWithDiskv(d)) + t.Transport = rt + + return t +} + // NewAuthProxyRoundTripper provides a roundtripper which will add auth proxy fields to requests for // authentication terminating proxy cases // assuming you pull the user from the context: diff --git a/transport/round_trippers_test.go b/transport/round_trippers_test.go index d5ffc6bd..c1e30c3f 100644 --- a/transport/round_trippers_test.go +++ b/transport/round_trippers_test.go @@ -17,7 +17,11 @@ limitations under the License. package transport import ( + "bytes" + "io/ioutil" "net/http" + "net/url" + "os" "reflect" "strings" "testing" @@ -216,3 +220,60 @@ func TestAuthProxyRoundTripper(t *testing.T) { } } } + +func TestCacheRoundTripper(t *testing.T) { + rt := &testRoundTripper{} + cacheDir, err := ioutil.TempDir("", "cache-rt") + defer os.RemoveAll(cacheDir) + if err != nil { + t.Fatal(err) + } + cache := NewCacheRoundTripper(cacheDir, rt) + + // First call, caches the response + req := &http.Request{ + Method: http.MethodGet, + URL: &url.URL{Host: "localhost"}, + } + rt.Response = &http.Response{ + Header: http.Header{"ETag": []string{`"123456"`}}, + Body: ioutil.NopCloser(bytes.NewReader([]byte("Content"))), + StatusCode: http.StatusOK, + } + resp, err := cache.RoundTrip(req) + if err != nil { + t.Fatal(err) + } + content, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if string(content) != "Content" { + t.Errorf(`Expected Body to be "Content", got %q`, string(content)) + } + + // Second call, returns cached response + req = &http.Request{ + Method: http.MethodGet, + URL: &url.URL{Host: "localhost"}, + } + rt.Response = &http.Response{ + StatusCode: http.StatusNotModified, + Body: ioutil.NopCloser(bytes.NewReader([]byte("Other Content"))), + } + + resp, err = cache.RoundTrip(req) + if err != nil { + t.Fatal(err) + } + + // Read body and make sure we have the initial content + content, err = ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + t.Fatal(err) + } + if string(content) != "Content" { + t.Errorf("Invalid content read from cache %q", string(content)) + } +}