github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-26 05:03:09 +00:00
Code Issues Projects Releases Wiki Activity

openapi: Remove cache mechanism

Browse Source 
The cache will be removed and replaced with HTTP Etag caching instead.
This patch is simply removing the existing mechanism.
This commit is contained in:
Antoine Pelisse 2017-06-05 10:28:18 -07:00
parent 90a45b2df3
commit a1d0384e82
10 changed files with 84 additions and 491 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
pkg/kubectl/cmd/get.go
View File

@ -138,7 +138,6 @@ func NewCmdGet(f cmdutil.Factory, out io.Writer, errOut io.Writer) *cobra.Comman
usage := "identifying the resource to get from a server."
cmdutil.AddFilenameOptionFlags(cmd, &options.FilenameOptions, usage)
cmdutil.AddInclude3rdPartyFlags(cmd)
cmdutil.AddOpenAPIFlags(cmd)
cmd.Flags().StringVar(&options.Raw, "raw", options.Raw, "Raw URI to request from the server. Uses the transport specified by the kubeconfig file.")
return cmd
}
@ -457,7 +456,7 @@ func RunGet(f cmdutil.Factory, out, errOut io.Writer, cmd *cobra.Command, args [
// if cmd does not specify output format and useOpenAPIPrintColumnFlagLabel flag is true,
// then get the default output options for this mapping from OpenAPI schema.
if !cmdSpecifiesOutputFmt(cmd) && useOpenAPIPrintColumns {
outputOpts, _ = outputOptsForMappingFromOpenAPI(f, cmdutil.GetOpenAPICacheDir(cmd), mapping)
outputOpts, _ = outputOptsForMappingFromOpenAPI(f, mapping)
}
printer, err = f.PrinterForMapping(cmd, false, outputOpts, mapping, allNamespaces)
@ -556,11 +555,11 @@ func cmdSpecifiesOutputFmt(cmd *cobra.Command) bool {
// outputOptsForMappingFromOpenAPI looks for the output format metatadata in the
// openapi schema and returns the output options for the mapping if found.
func outputOptsForMappingFromOpenAPI(f cmdutil.Factory, openAPIcacheDir string, mapping *meta.RESTMapping) (*printers.OutputOptions, bool) {
func outputOptsForMappingFromOpenAPI(f cmdutil.Factory, mapping *meta.RESTMapping) (*printers.OutputOptions, bool) {
// user has not specified any output format, check if OpenAPI has
// default specification to print this resource type
api, err := f.OpenAPISchema(openAPIcacheDir)
api, err := f.OpenAPISchema()
if err != nil {
// Error getting schema
return nil, false

4
pkg/kubectl/cmd/testing/fake.go
View File

@ -418,7 +418,7 @@ func (f *FakeFactory) SwaggerSchema(schema.GroupVersionKind) (*swagger.ApiDeclar
return nil, nil
}
func (f *FakeFactory) OpenAPISchema(cacheDir string) (openapi.Resources, error) {
func (f *FakeFactory) OpenAPISchema() (openapi.Resources, error) {
return nil, nil
}
@ -756,7 +756,7 @@ func (f *fakeAPIFactory) SwaggerSchema(schema.GroupVersionKind) (*swagger.ApiDec
return nil, nil
}
func (f *fakeAPIFactory) OpenAPISchema(cacheDir string) (openapi.Resources, error) {
func (f *fakeAPIFactory) OpenAPISchema() (openapi.Resources, error) {
if f.tf.OpenAPISchemaFunc != nil {
return f.tf.OpenAPISchemaFunc()
}

2
pkg/kubectl/cmd/util/factory.go
View File

@ -224,7 +224,7 @@ type ObjectMappingFactory interface {
// SwaggerSchema returns the schema declaration for the provided group version kind.
SwaggerSchema(schema.GroupVersionKind) (*swagger.ApiDeclaration, error)
// OpenAPISchema returns the schema openapi schema definiton
OpenAPISchema(cacheDir string) (openapi.Resources, error)
OpenAPISchema() (openapi.Resources, error)
}
// BuilderFactory holds the second level of factory methods. These functions depend upon ObjectMappingFactory and ClientAccessFactory methods.

25
pkg/kubectl/cmd/util/factory_object_mapping.go
View File

@ -439,13 +439,7 @@ func (f *ring1Factory) SwaggerSchema(gvk schema.GroupVersionKind) (*swagger.ApiD
}
// OpenAPISchema returns metadata and structural information about Kubernetes object definitions.
// Will try to cache the data to a local file. Cache is written and read from a
// file created with ioutil.TempFile and obeys the expiration semantics of that file.
// The cache location is a function of the client and server versions so that the open API
// schema will be cached separately for different client / server combinations.
// Note, the cache will not be invalidated if the server changes its open API schema without
// changing the server version.
func (f *ring1Factory) OpenAPISchema(cacheDir string) (openapi.Resources, error) {
func (f *ring1Factory) OpenAPISchema() (openapi.Resources, error) {
discovery, err := f.clientAccessFactory.DiscoveryClient()
if err != nil {
return nil, err
@ -453,23 +447,8 @@ func (f *ring1Factory) OpenAPISchema(cacheDir string) (openapi.Resources, error)
// Lazily initialize the OpenAPIGetter once
f.openAPIGetter.once.Do(func() {
// Get the server version for caching the openapi spec
versionString := ""
version, err := discovery.ServerVersion()
if err != nil {
// Cache the result under the server version
versionString = version.String()
}
// Get the cache directory for caching the openapi spec
cacheDir, err = substituteUserHome(cacheDir)
if err != nil {
// Don't cache the result if we couldn't substitute the home directory
cacheDir = ""
}
// Create the caching OpenAPIGetter
f.openAPIGetter.getter = openapi.NewOpenAPIGetter(cacheDir, versionString, discovery)
f.openAPIGetter.getter = openapi.NewOpenAPIGetter(discovery)
})
// Delegate to the OpenAPIGetter

13
pkg/kubectl/cmd/util/helpers.go
View File

@ -404,19 +404,6 @@ func AddValidateOptionFlags(cmd *cobra.Command, options *ValidateOptions) {
cmd.MarkFlagFilename("schema-cache-dir")
}
func AddOpenAPIFlags(cmd *cobra.Command) {
cmd.Flags().String("schema-cache-dir",
fmt.Sprintf("~/%s/%s", clientcmd.RecommendedHomeDir, clientcmd.RecommendedSchemaName),
fmt.Sprintf("If non-empty, load/store cached API schemas in this directory, default is '$HOME/%s/%s'",
clientcmd.RecommendedHomeDir, clientcmd.RecommendedSchemaName),
)
cmd.MarkFlagFilename("schema-cache-dir")
}
func GetOpenAPICacheDir(cmd *cobra.Command) string {
return GetFlagString(cmd, "schema-cache-dir")
}
func AddFilenameOptionFlags(cmd *cobra.Command, options *resource.FilenameOptions, usage string) {
kubectl.AddJsonFilenameFlag(cmd, &options.Filenames, "Filename, directory, or URL to files "+usage)
cmd.Flags().BoolVarP(&options.Recursive, "recursive", "R", options.Recursive, "Process the directory used in -f, --filename recursively. Useful when you want to manage related manifests organized within the same directory.")

5
pkg/kubectl/cmd/util/openapi/BUILD
View File

@ -15,15 +15,11 @@ go_library(
"document.go",
"extensions.go",
"openapi.go",
"openapi_cache.go",
"openapi_getter.go",
],
tags = ["automanaged"],
deps = [
"//pkg/version:go_default_library",
"//vendor/github.com/go-openapi/spec:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/golang/protobuf/proto:go_default_library",
"//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library",
"//vendor/gopkg.in/yaml.v2:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
@ -35,7 +31,6 @@ go_test(
name = "go_default_xtest",
size = "small",
srcs = [
"openapi_cache_test.go",
"openapi_getter_test.go",
"openapi_suite_test.go",
"openapi_test.go",

163
pkg/kubectl/cmd/util/openapi/openapi_cache.go
View File

@ -1,163 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openapi
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"github.com/golang/glog"
"github.com/golang/protobuf/proto"
openapi_v2 "github.com/googleapis/gnostic/OpenAPIv2"
"k8s.io/client-go/discovery"
"k8s.io/kubernetes/pkg/version"
)
const openapiFileName = "openapi_cache"
type CachingOpenAPIClient struct {
version string
client discovery.OpenAPISchemaInterface
cacheDirName string
}
// NewCachingOpenAPIClient returns a new discovery.OpenAPISchemaInterface
// that will read the openapi spec from a local cache if it exists, and
// if not will then fetch an openapi spec using a client.
// client: used to fetch a new openapi spec if a local cache is not found
// version: the server version and used as part of the cache file location
// cacheDir: the directory under which the cache file will be written
func NewCachingOpenAPIClient(client discovery.OpenAPISchemaInterface, version, cacheDir string) *CachingOpenAPIClient {
return &CachingOpenAPIClient{
client: client,
version: version,
cacheDirName: cacheDir,
}
}
// OpenAPIData returns an openapi spec.
// It will first attempt to read the spec from a local cache
// If it cannot read a local cache, it will read the file
// using the client and then write the cache.
func (c *CachingOpenAPIClient) OpenAPIData() (Resources, error) {
// Try to use the cached version
if c.useCache() {
doc, err := c.readOpenAPICache()
if err == nil {
return NewOpenAPIData(doc)
}
}
// No cached version found, download from server
s, err := c.client.OpenAPISchema()
if err != nil {
glog.V(2).Infof("Failed to download openapi data %v", err)
return nil, err
}
oa, err := NewOpenAPIData(s)
if err != nil {
glog.V(2).Infof("Failed to parse openapi data %v", err)
return nil, err
}
// Try to cache the openapi spec
if c.useCache() {
err = c.writeToCache(s)
if err != nil {
// Just log an message, no need to fail the command since we got the data we need
glog.V(2).Infof("Unable to cache openapi spec %v", err)
}
}
// Return the parsed data
return oa, nil
}
// useCache returns true if the client should try to use the cache file
func (c *CachingOpenAPIClient) useCache() bool {
return len(c.version) > 0 && len(c.cacheDirName) > 0
}
// readOpenAPICache tries to read the openapi spec from the local file cache
func (c *CachingOpenAPIClient) readOpenAPICache() (*openapi_v2.Document, error) {
// Get the filename to read
filename := c.openAPICacheFilename()
// Read the cached file
data, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
doc := &openapi_v2.Document{}
return doc, proto.Unmarshal(data, doc)
}
// writeToCache tries to write the openapi spec to the local file cache.
// writes the data to a new tempfile, and then links the cache file and the tempfile
func (c *CachingOpenAPIClient) writeToCache(doc *openapi_v2.Document) error {
// Get the constant filename used to read the cache.
cacheFile := c.openAPICacheFilename()
// Binary encode the spec. This is 10x as fast as using json encoding. (60ms vs 600ms)
b, err := proto.Marshal(doc)
if err != nil {
return fmt.Errorf("Could not binary encode openapi spec: %v", err)
}
// Create a new temp file for the cached openapi spec.
cacheDir := filepath.Dir(cacheFile)
if err := os.MkdirAll(cacheDir, 0755); err != nil {
return fmt.Errorf("Could not create directory: %v %v", cacheDir, err)
}
tmpFile, err := ioutil.TempFile(cacheDir, "openapi")
if err != nil {
return fmt.Errorf("Could not create temp cache file: %v %v", cacheFile, err)
}
// Write the binary encoded openapi spec to the temp file
if _, err := io.Copy(tmpFile, bytes.NewBuffer(b)); err != nil {
return fmt.Errorf("Could not write temp cache file: %v", err)
}
// Link the temp cache file to the constant cache filepath
return linkFiles(tmpFile.Name(), cacheFile)
}
// openAPICacheFilename returns the filename to read the cache from
func (c *CachingOpenAPIClient) openAPICacheFilename() string {
// Cache using the client and server versions
return filepath.Join(c.cacheDirName, c.version, version.Get().GitVersion, openapiFileName)
}
// linkFiles links the old file to the new file
func linkFiles(old, new string) error {
if err := os.Link(old, new); err != nil {
// If we can't write due to file existing, or permission problems, keep going.
if os.IsExist(err) || os.IsPermission(err) {
return nil
}
return err
}
return nil
}

268
pkg/kubectl/cmd/util/openapi/openapi_cache_test.go
View File

@ -1,268 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openapi_test
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sync"
"gopkg.in/yaml.v2"
"github.com/googleapis/gnostic/OpenAPIv2"
"github.com/googleapis/gnostic/compiler"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"k8s.io/kubernetes/pkg/kubectl/cmd/util/openapi"
)
var _ = Describe("When reading openAPIData", func() {
var tmpDir string
var err error
var client *fakeOpenAPIClient
var instance *openapi.CachingOpenAPIClient
var expectedData openapi.Resources
BeforeEach(func() {
tmpDir, err = ioutil.TempDir("", "openapi_cache_test")
Expect(err).To(BeNil())
client = &fakeOpenAPIClient{}
instance = openapi.NewCachingOpenAPIClient(client, "v1.6", tmpDir)
d, err := data.OpenAPISchema()
Expect(err).To(BeNil())
expectedData, err = openapi.NewOpenAPIData(d)
Expect(err).To(BeNil())
})
AfterEach(func() {
os.RemoveAll(tmpDir)
})
It("should write to the cache", func() {
By("getting the live openapi spec from the server")
result, err := instance.OpenAPIData()
Expect(err).To(BeNil())
Expect(result).To(Equal(expectedData))
Expect(client.calls).To(Equal(1))
By("writing the live openapi spec to a local cache file")
names, err := getFilenames(tmpDir)
Expect(err).To(BeNil())
Expect(names).To(ConsistOf("v1.6"))
names, err = getFilenames(filepath.Join(tmpDir, "v1.6"))
Expect(err).To(BeNil())
Expect(names).To(HaveLen(1))
clientVersion := names[0]
names, err = getFilenames(filepath.Join(tmpDir, "v1.6", clientVersion))
Expect(err).To(BeNil())
Expect(names).To(ContainElement("openapi_cache"))
})
It("should read from the cache", func() {
// First call should use the client
result, err := instance.OpenAPIData()
Expect(err).To(BeNil())
Expect(result).To(Equal(expectedData))
Expect(client.calls).To(Equal(1))
// Second call shouldn't use the client
result, err = instance.OpenAPIData()
Expect(err).To(BeNil())
Expect(result).To(Equal(expectedData))
Expect(client.calls).To(Equal(1))
names, err := getFilenames(tmpDir)
Expect(err).To(BeNil())
Expect(names).To(ConsistOf("v1.6"))
})
It("propagate errors that are encountered", func() {
// Expect an error
client.err = fmt.Errorf("expected error")
result, err := instance.OpenAPIData()
Expect(err.Error()).To(Equal(client.err.Error()))
Expect(result).To(BeNil())
Expect(client.calls).To(Equal(1))
// No cache file is written
files, err := ioutil.ReadDir(tmpDir)
Expect(err).To(BeNil())
Expect(files).To(HaveLen(0))
// Client error is not cached
result, err = instance.OpenAPIData()
Expect(err.Error()).To(Equal(client.err.Error()))
Expect(result).To(BeNil())
Expect(client.calls).To(Equal(2))
})
})
var _ = Describe("Reading openAPIData", func() {
var tmpDir string
var serverVersion string
var cacheDir string
BeforeEach(func() {
var err error
tmpDir, err = ioutil.TempDir("", "openapi_cache_test")
Expect(err).To(BeNil())
})
AfterEach(func() {
os.RemoveAll(tmpDir)
})
// Set the serverVersion to empty
Context("when the server version is empty", func() {
BeforeEach(func() {
serverVersion = ""
cacheDir = tmpDir
})
It("should not cache the result", func() {
client := &fakeOpenAPIClient{}
instance := openapi.NewCachingOpenAPIClient(client, serverVersion, cacheDir)
d, err := data.OpenAPISchema()
Expect(err).To(BeNil())
expectedData, err := openapi.NewOpenAPIData(d)
Expect(err).To(BeNil())
By("getting the live openapi schema")
result, err := instance.OpenAPIData()
Expect(err).To(BeNil())
Expect(result).To(Equal(expectedData))
Expect(client.calls).To(Equal(1))
files, err := ioutil.ReadDir(tmpDir)
Expect(err).To(BeNil())
Expect(files).To(HaveLen(0))
})
})
Context("when the cache directory is empty", func() {
BeforeEach(func() {
serverVersion = "v1.6"
cacheDir = ""
})
It("should not cache the result", func() {
client := &fakeOpenAPIClient{}
instance := openapi.NewCachingOpenAPIClient(client, serverVersion, cacheDir)
d, err := data.OpenAPISchema()
Expect(err).To(BeNil())
expectedData, err := openapi.NewOpenAPIData(d)
Expect(err).To(BeNil())
By("getting the live openapi schema")
result, err := instance.OpenAPIData()
Expect(err).To(BeNil())
Expect(result).To(Equal(expectedData))
Expect(client.calls).To(Equal(1))
files, err := ioutil.ReadDir(tmpDir)
Expect(err).To(BeNil())
Expect(files).To(HaveLen(0))
})
})
})
// Test Utils
func getFilenames(path string) ([]string, error) {
files, err := ioutil.ReadDir(path)
if err != nil {
return nil, err
}
result := []string{}
for _, n := range files {
result = append(result, n.Name())
}
return result, nil
}
type fakeOpenAPIClient struct {
calls int
err error
}
func (f *fakeOpenAPIClient) OpenAPISchema() (*openapi_v2.Document, error) {
f.calls = f.calls + 1
if f.err != nil {
return nil, f.err
}
return data.OpenAPISchema()
}
// Test utils
var data apiData
type apiData struct {
sync.Once
data *openapi_v2.Document
err error
}
func (d *apiData) OpenAPISchema() (*openapi_v2.Document, error) {
d.Do(func() {
// Get the path to the swagger.json file
wd, err := os.Getwd()
if err != nil {
d.err = err
return
}
abs, err := filepath.Abs(wd)
if err != nil {
d.err = err
return
}
root := filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(abs)))))
specpath := filepath.Join(root, "api", "openapi-spec", "swagger.json")
_, err = os.Stat(specpath)
if err != nil {
d.err = err
return
}
spec, err := ioutil.ReadFile(specpath)
if err != nil {
d.err = err
return
}
var info yaml.MapSlice
err = yaml.Unmarshal(spec, &info)
if err != nil {
d.err = err
return
}
d.data, d.err = openapi_v2.NewDocument(info, compiler.NewContext("$root", nil))
})
return d.data, d.err
}

16
pkg/kubectl/cmd/util/openapi/openapi_getter.go
View File

@ -29,8 +29,6 @@ type synchronizedOpenAPIGetter struct {
openAPISchema Resources
err error
serverVersion string
cacheDir string
openAPIClient discovery.OpenAPISchemaInterface
}
@ -42,12 +40,10 @@ type Getter interface {
Get() (Resources, error)
}
// NewOpenAPIGetter returns an object to return OpenAPIDatas which either read from a
// local file cache or read from a server, and then stored in memory for subsequent invocations
func NewOpenAPIGetter(cacheDir, serverVersion string, openAPIClient discovery.OpenAPISchemaInterface) Getter {
// NewOpenAPIGetter returns an object to return OpenAPIDatas which reads
// from a server, and then stores in memory for subsequent invocations
func NewOpenAPIGetter(openAPIClient discovery.OpenAPISchemaInterface) Getter {
return &synchronizedOpenAPIGetter{
serverVersion: serverVersion,
cacheDir: cacheDir,
openAPIClient: openAPIClient,
}
}
@ -55,15 +51,13 @@ func NewOpenAPIGetter(cacheDir, serverVersion string, openAPIClient discovery.Op
// Resources implements Getter
func (g *synchronizedOpenAPIGetter) Get() (Resources, error) {
g.Do(func() {
client := NewCachingOpenAPIClient(g.openAPIClient, g.serverVersion, g.cacheDir)
result, err := client.OpenAPIData()
s, err := g.openAPIClient.OpenAPISchema()
if err != nil {
g.err = err
return
}
// Save the result
g.openAPISchema = result
g.openAPISchema, g.err = NewOpenAPIData(s)
})
// Return the save result

72
pkg/kubectl/cmd/util/openapi/openapi_getter_test.go
View File

@ -18,13 +18,83 @@ package openapi_test
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sync"
"gopkg.in/yaml.v2"
"github.com/googleapis/gnostic/OpenAPIv2"
"github.com/googleapis/gnostic/compiler"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"k8s.io/kubernetes/pkg/kubectl/cmd/util/openapi"
)
// Test utils
var data apiData
type apiData struct {
sync.Once
data *openapi_v2.Document
err error
}
func (d *apiData) OpenAPISchema() (*openapi_v2.Document, error) {
d.Do(func() {
// Get the path to the swagger.json file
wd, err := os.Getwd()
if err != nil {
d.err = err
return
}
abs, err := filepath.Abs(wd)
if err != nil {
d.err = err
return
}
root := filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(abs)))))
specpath := filepath.Join(root, "api", "openapi-spec", "swagger.json")
_, err = os.Stat(specpath)
if err != nil {
d.err = err
return
}
spec, err := ioutil.ReadFile(specpath)
if err != nil {
d.err = err
return
}
var info yaml.MapSlice
err = yaml.Unmarshal(spec, &info)
if err != nil {
d.err = err
return
}
d.data, d.err = openapi_v2.NewDocument(info, compiler.NewContext("$root", nil))
})
return d.data, d.err
}
type fakeOpenAPIClient struct {
calls int
err error
}
func (f *fakeOpenAPIClient) OpenAPISchema() (*openapi_v2.Document, error) {
f.calls = f.calls + 1
if f.err != nil {
return nil, f.err
}
return data.OpenAPISchema()
}
var _ = Describe("Getting the Resources", func() {
var client *fakeOpenAPIClient
var expectedData openapi.Resources
@ -38,7 +108,7 @@ var _ = Describe("Getting the Resources", func() {
expectedData, err = openapi.NewOpenAPIData(d)
Expect(err).To(BeNil())
instance = openapi.NewOpenAPIGetter("", "", client)
instance = openapi.NewOpenAPIGetter(client)
})
Context("when the server returns a successful result", func() {