Merge pull request #115809 from seans3/openapi-file-client

New openapi v3 testing file client

staging/src/k8s.io/client-go/go.mod

@@ -29,6 +29,7 @@ require (
 	k8s.io/klog/v2 v2.80.1
 	k8s.io/kube-openapi v0.0.0-20230123231816-1cb3ae25d79a
 	k8s.io/utils v0.0.0-20230209194617-a36077c30491
+	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3
 	sigs.k8s.io/yaml v1.3.0
 )
@@ -55,7 +56,6 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
 )
 
 replace (

staging/src/k8s.io/client-go/openapi/openapitest/fileclient.go

@@ -0,0 +1,107 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openapitest
+
+import (
+	"embed"
+	"errors"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+
+	"k8s.io/client-go/openapi"
+)
+
+//go:embed testdata/*_openapi.json
+var f embed.FS
+
+// NewFileClient returns a test client implementing the openapi.Client
+// interface, which serves a subset of hard-coded GroupVersion
+// Open API V3 specifications files. The subset of specifications is
+// located in the "testdata" subdirectory.
+func NewFileClient(t *testing.T) openapi.Client {
+	if t == nil {
+		panic("non-nil testing.T required; this package is only for use in tests")
+	}
+	return &fileClient{t: t}
+}
+
+type fileClient struct {
+	t     *testing.T
+	init  sync.Once
+	paths map[string]openapi.GroupVersion
+	err   error
+}
+
+// fileClient implements the openapi.Client interface.
+var _ openapi.Client = &fileClient{}
+
+// Paths returns a map of api path string to openapi.GroupVersion or
+// an error. The OpenAPI V3 GroupVersion specifications are hard-coded
+// in the "testdata" subdirectory. The api path is derived from the
+// spec filename. Example:
+//
+//	apis__apps__v1_openapi.json -> apis/apps/v1
+//
+// The file contents are read only once. All files must parse correctly
+// into an api path, or an error is returned.
+func (t *fileClient) Paths() (map[string]openapi.GroupVersion, error) {
+	t.init.Do(func() {
+		t.paths = map[string]openapi.GroupVersion{}
+		entries, err := f.ReadDir("testdata")
+		if err != nil {
+			t.err = err
+			t.t.Error(err)
+		}
+		for _, e := range entries {
+			// this reverses the transformation done in hack/update-openapi-spec.sh
+			path := strings.ReplaceAll(strings.TrimSuffix(e.Name(), "_openapi.json"), "__", "/")
+			t.paths[path] = &fileGroupVersion{t: t.t, filename: filepath.Join("testdata", e.Name())}
+		}
+	})
+	return t.paths, t.err
+}
+
+type fileGroupVersion struct {
+	t        *testing.T
+	init     sync.Once
+	filename string
+	data     []byte
+	err      error
+}
+
+// fileGroupVersion implements the openapi.GroupVersion interface.
+var _ openapi.GroupVersion = &fileGroupVersion{}
+
+// Schema returns the OpenAPI V3 specification for the GroupVersion as
+// unstructured bytes, or an error if the contentType is not
+// "application/json" or there is an error reading the spec file. The
+// file is read only once. The embedded file is located in the "testdata"
+// subdirectory.
+func (t *fileGroupVersion) Schema(contentType string) ([]byte, error) {
+	if contentType != "application/json" {
+		return nil, errors.New("openapitest only supports 'application/json' contentType")
+	}
+	t.init.Do(func() {
+		t.data, t.err = f.ReadFile(t.filename)
+		if t.err != nil {
+			t.t.Error(t.err)
+		}
+	})
+	return t.data, t.err
+}

staging/src/k8s.io/client-go/openapi/openapitest/fileclient_test.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openapitest
+
+import (
+	"k8s.io/kube-openapi/pkg/spec3"
+	kjson "sigs.k8s.io/json"
+	"testing"
+)
+
+func TestOpenAPITest(t *testing.T) {
+	client := NewFileClient(t)
+
+	// make sure we get paths
+	paths, err := client.Paths()
+	if err != nil {
+		t.Fatalf("error fetching paths: %v", err)
+	}
+	if len(paths) == 0 {
+		t.Error("empty paths")
+	}
+
+	// spot check specific paths
+	expectedPaths := []string{
+		"api/v1",
+		"apis/apps/v1",
+		"apis/batch/v1",
+		"apis/networking.k8s.io/v1alpha1",
+		"apis/discovery.k8s.io/v1",
+	}
+	for _, p := range expectedPaths {
+		if _, ok := paths[p]; !ok {
+			t.Fatalf("expected %s", p)
+		}
+	}
+
+	// make sure all paths can load
+	for path, gv := range paths {
+		data, err := gv.Schema("application/json")
+		if err != nil {
+			t.Fatalf("error reading schema for %v: %v", path, err)
+		}
+		o := &spec3.OpenAPI{}
+		stricterrs, err := kjson.UnmarshalStrict(data, o)
+		if err != nil {
+			t.Fatalf("error unmarshaling schema for %v: %v", path, err)
+		}
+		if len(stricterrs) > 0 {
+			t.Fatalf("strict errors unmarshaling schema for %v: %v", path, stricterrs)
+		}
+	}
+}

staging/src/k8s.io/kubectl/pkg/explain/v2/explain_test.go

@@ -19,118 +19,42 @@ package v2
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
-	"sync"
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/require"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/openapi"
+	"k8s.io/client-go/openapi/openapitest"
 )
 
-var apiDiscoveryJSON string = `{"openapi":"3.0.0","info":{"title":"Kubernetes","version":"v1.26.0"},"paths":{"/apis/discovery.k8s.io/":{"get":{"tags":["discovery"],"description":"get information of a group","operationId":"getDiscoveryAPIGroup","responses":{"200":{"description":"OK","content":{"application/json":{"schema":{"$ref":"#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.APIGroup"}},"application/vnd.kubernetes.protobuf":{"schema":{"$ref":"#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.APIGroup"}},"application/yaml":{"schema":{"$ref":"#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.APIGroup"}}}},"401":{"description":"Unauthorized"}}}}},"components":{"schemas":{"io.k8s.apimachinery.pkg.apis.meta.v1.APIGroup":{"description":"APIGroup contains the name, the supported versions, and the preferred version of a group.","type":"object","required":["name","versions"],"properties":{"apiVersion":{"description":"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"},"kind":{"description":"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds","type":"string"},"name":{"description":"name is the name of the group.","type":"string","default":""},"preferredVersion":{"description":"preferredVersion is the version preferred by the API server, which probably is the storage version.","default":{},"allOf":[{"$ref":"#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.GroupVersionForDiscovery"}]},"serverAddressByClientCIDRs":{"description":"a map of client CIDR to server address that is serving this group. This is to help clients reach servers in the most network-efficient way possible. Clients can use the appropriate server address as per the CIDR that they match. In case of multiple matches, clients should use the longest matching CIDR. The server returns only those CIDRs that it thinks that the client can match. For example: the master will return an internal IP CIDR only, if the client reaches the server using an internal IP. Server looks at X-Forwarded-For header or X-Real-Ip header or request.RemoteAddr (in that order) to get the client IP.","type":"array","items":{"default":{},"allOf":[{"$ref":"#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.ServerAddressByClientCIDR"}]}},"versions":{"description":"versions are the versions supported in this group.","type":"array","items":{"default":{},"allOf":[{"$ref":"#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.GroupVersionForDiscovery"}]}}},"x-kubernetes-group-version-kind":[{"group":"","kind":"APIGroup","version":"v1"}]},"io.k8s.apimachinery.pkg.apis.meta.v1.GroupVersionForDiscovery":{"description":"GroupVersion contains the \"group/version\" and \"version\" string of a version. It is made a struct to keep extensibility.","type":"object","required":["groupVersion","version"],"properties":{"groupVersion":{"description":"groupVersion specifies the API group and version in the form \"group/version\"","type":"string","default":""},"version":{"description":"version specifies the version in the form of \"version\". This is to save the clients the trouble of splitting the GroupVersion.","type":"string","default":""}}},"io.k8s.apimachinery.pkg.apis.meta.v1.ServerAddressByClientCIDR":{"description":"ServerAddressByClientCIDR helps the client to determine the server address that they should use, depending on the clientCIDR that they match.","type":"object","required":["clientCIDR","serverAddress"],"properties":{"clientCIDR":{"description":"The CIDR with which clients can match their IP to figure out the server address that they should use.","type":"string","default":""},"serverAddress":{"description":"Address of this server, suitable for a client that matches the above CIDR. This can be a hostname, hostname:port, IP or IP:port.","type":"string","default":""}}}},"securitySchemes":{"BearerToken":{"type":"apiKey","description":"Bearer Token authentication","name":"authorization","in":"header"}}}}`
-var apiGroupsGVR schema.GroupVersionResource = schema.GroupVersionResource{
+var apiGroupsPath = "apis/discovery.k8s.io/v1"
+
+var apiGroupsGVR = schema.GroupVersionResource{
 	Group:    "discovery.k8s.io",
 	Version:  "v1",
 	Resource: "apigroups",
 }
-var apiGroupsDocument map[string]interface{} = func() map[string]interface{} {
-	var doc map[string]interface{}
-
-	err := json.Unmarshal([]byte(apiDiscoveryJSON), &doc)
-	if err != nil {
-		panic(err)
-	}
-
-	return doc
-}()
-
-type FakeOpenAPIV3Client struct {
-	// Path:
-	//		ContentType:
-	//			OpenAPIV3 Schema bytes
-	Values      map[string]map[string][]byte
-	FetchCounts map[string]map[string]int
-	lock        sync.Mutex
-}
-
-type FakeGroupVersion struct {
-	Data        map[string][]byte
-	FetchCounts map[string]int
-	Lock        *sync.Mutex
-}
-
-func (f *FakeGroupVersion) Schema(contentType string) ([]byte, error) {
-	f.Lock.Lock()
-	defer f.Lock.Unlock()
-
-	if count, ok := f.FetchCounts[contentType]; ok {
-		f.FetchCounts[contentType] = count + 1
-	} else {
-		f.FetchCounts[contentType] = 1
-	}
-
-	data, ok := f.Data[contentType]
-	if !ok {
-		return nil, errors.New("not found")
-	}
-	return data, nil
-}
-
-func (f *FakeOpenAPIV3Client) Paths() (map[string]openapi.GroupVersion, error) {
-	if f.Values == nil {
-		return nil, errors.New("values is nil")
-	}
-
-	res := map[string]openapi.GroupVersion{}
-	if f.FetchCounts == nil {
-		f.FetchCounts = map[string]map[string]int{}
-	}
-
-	for k, v := range f.Values {
-		counts, ok := f.FetchCounts[k]
-		if !ok {
-			counts = map[string]int{}
-			f.FetchCounts[k] = counts
-		}
-		res[k] = &FakeGroupVersion{Data: v, FetchCounts: counts, Lock: &f.lock}
-	}
-	return res, nil
-}
 
 func TestExplainErrors(t *testing.T) {
 	var buf bytes.Buffer
 
-	// A client with nil `Values` will return error on returning paths
-	failFetchPaths := &FakeOpenAPIV3Client{}
-
-	err := PrintModelDescription(nil, &buf, failFetchPaths, apiGroupsGVR, false, "unknown-format")
+	// Validate error when "Paths()" returns error.
+	err := PrintModelDescription(nil, &buf, &forceErrorClient{}, apiGroupsGVR, false, "unknown-format")
 	require.ErrorContains(t, err, "failed to fetch list of groupVersions")
 
-	// Missing Schema
-	fakeClient := &FakeOpenAPIV3Client{
-		Values: map[string]map[string][]byte{
-			"apis/test1.example.com/v1": {
-				"unknown/content-type": []byte(apiDiscoveryJSON),
-			},
-			"apis/test2.example.com/v1": {
-				runtime.ContentTypeJSON: []byte(`<some invalid json!>`),
-			},
-			"apis/discovery.k8s.io/v1": {
-				runtime.ContentTypeJSON: []byte(apiDiscoveryJSON),
-			},
-		},
-	}
-
-	err = PrintModelDescription(nil, &buf, fakeClient, schema.GroupVersionResource{
+	// Validate error when GVR is not found in returned paths map.
+	emptyClient := &emptyPathsClient{}
+	err = PrintModelDescription(nil, &buf, emptyClient, schema.GroupVersionResource{
 		Group:    "test0.example.com",
 		Version:  "v1",
 		Resource: "doesntmatter",
 	}, false, "unknown-format")
 	require.ErrorContains(t, err, "could not locate schema")
 
-	// Missing JSON
+	// Validate error when GroupVersion "Schema()" call returns error.
+	fakeClient := &fakeOpenAPIClient{values: make(map[string]openapi.GroupVersion)}
+	fakeClient.values["apis/test1.example.com/v1"] = &forceErrorGV{}
 	err = PrintModelDescription(nil, &buf, fakeClient, schema.GroupVersionResource{
 		Group:    "test1.example.com",
 		Version:  "v1",
@@ -138,6 +62,8 @@ func TestExplainErrors(t *testing.T) {
 	}, false, "unknown-format")
 	require.ErrorContains(t, err, "failed to fetch openapi schema ")
 
+	// Validate error when returned bytes from GroupVersion "Schema" are invalid.
+	fakeClient.values["apis/test2.example.com/v1"] = &parseErrorGV{}
 	err = PrintModelDescription(nil, &buf, fakeClient, schema.GroupVersionResource{
 		Group:    "test2.example.com",
 		Version:  "v1",
@@ -145,7 +71,9 @@ func TestExplainErrors(t *testing.T) {
 	}, false, "unknown-format")
 	require.ErrorContains(t, err, "failed to parse openapi schema")
 
-	err = PrintModelDescription(nil, &buf, fakeClient, apiGroupsGVR, false, "unknown-format")
+	// Validate error when render template is not recognized.
+	client := openapitest.NewFileClient(t)
+	err = PrintModelDescription(nil, &buf, client, apiGroupsGVR, false, "unknown-format")
 	require.ErrorContains(t, err, "unrecognized format: unknown-format")
 }
 
@@ -154,31 +82,71 @@ func TestExplainErrors(t *testing.T) {
 func TestExplainOpenAPIClient(t *testing.T) {
 	var buf bytes.Buffer
 
-	fakeClient := &FakeOpenAPIV3Client{
-		Values: map[string]map[string][]byte{
-			"apis/discovery.k8s.io/v1": {
-				runtime.ContentTypeJSON: []byte(apiDiscoveryJSON),
-			},
-		},
-	}
+	fileClient := openapitest.NewFileClient(t)
+	paths, err := fileClient.Paths()
+	require.NoError(t, err)
+	gv, found := paths[apiGroupsPath]
+	require.True(t, found)
+	discoveryBytes, err := gv.Schema("application/json")
+	require.NoError(t, err)
+
+	var doc map[string]interface{}
+	err = json.Unmarshal(discoveryBytes, &doc)
+	require.NoError(t, err)
 
 	gen := NewGenerator()
-	err := gen.AddTemplate("Context", "{{ toJson . }}")
+	err = gen.AddTemplate("Context", "{{ toJson . }}")
 	require.NoError(t, err)
 
 	expectedContext := TemplateContext{
-		Document:  apiGroupsDocument,
+		Document:  doc,
 		GVR:       apiGroupsGVR,
 		Recursive: false,
 		FieldPath: nil,
 	}
 
-	err = printModelDescriptionWithGenerator(gen, nil, &buf, fakeClient, apiGroupsGVR, false, "Context")
+	err = printModelDescriptionWithGenerator(gen, nil, &buf, fileClient, apiGroupsGVR, false, "Context")
 	require.NoError(t, err)
 
 	var actualContext TemplateContext
 	err = json.Unmarshal(buf.Bytes(), &actualContext)
 	require.NoError(t, err)
 	require.Equal(t, expectedContext, actualContext)
-	require.Equal(t, fakeClient.FetchCounts["apis/discovery.k8s.io/v1"][runtime.ContentTypeJSON], 1)
+}
+
+// forceErrorClient always returns an error for "Paths()".
+type forceErrorClient struct{}
+
+func (f *forceErrorClient) Paths() (map[string]openapi.GroupVersion, error) {
+	return nil, fmt.Errorf("Always fails")
+}
+
+// emptyPathsClient returns an empty map for "Paths()".
+type emptyPathsClient struct{}
+
+func (f *emptyPathsClient) Paths() (map[string]openapi.GroupVersion, error) {
+	return map[string]openapi.GroupVersion{}, nil
+}
+
+// fakeOpenAPIClient returns hard-coded map for "Paths()".
+type fakeOpenAPIClient struct {
+	values map[string]openapi.GroupVersion
+}
+
+func (f *fakeOpenAPIClient) Paths() (map[string]openapi.GroupVersion, error) {
+	return f.values, nil
+}
+
+// forceErrorGV always returns an error for "Schema()".
+type forceErrorGV struct{}
+
+func (f *forceErrorGV) Schema(contentType string) ([]byte, error) {
+	return nil, fmt.Errorf("Always fails")
+}
+
+// parseErrorGV always returns invalid JSON for "Schema()".
+type parseErrorGV struct{}
+
+func (f *parseErrorGV) Schema(contentType string) ([]byte, error) {
+	return []byte(`<some invalid json!>`), nil
 }

staging/src/k8s.io/kubectl/pkg/explain/v2/generator_test.go

@@ -22,20 +22,31 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/openapi/openapitest"
 )
 
+var appsv1Path = "apis/apps/v1"
+
+var appsDeploymentGVR = schema.GroupVersionResource{
+	Group:    "apps",
+	Version:  "v1",
+	Resource: "deployments",
+}
+
 // Shows generic throws error when attempting to `Render“ an invalid output name
 // And if it is then added as a template, no error is thrown upon `Render`
 func TestGeneratorMissingOutput(t *testing.T) {
 	var buf bytes.Buffer
 	var doc map[string]interface{}
 
-	err := json.Unmarshal([]byte(apiDiscoveryJSON), &doc)
+	appsv1Bytes := bytesForGV(t, appsv1Path)
+	err := json.Unmarshal(appsv1Bytes, &doc)
 	require.NoError(t, err)
 
 	gen := NewGenerator()
 	badTemplateName := "bad-template"
-	err = gen.Render(badTemplateName, doc, apiGroupsGVR, nil, false, &buf)
+	err = gen.Render(badTemplateName, doc, appsDeploymentGVR, nil, false, &buf)
 
 	require.ErrorContains(t, err, "unrecognized format: "+badTemplateName)
 	require.Zero(t, buf.Len())
@@ -43,7 +54,7 @@ func TestGeneratorMissingOutput(t *testing.T) {
 	err = gen.AddTemplate(badTemplateName, "ok")
 	require.NoError(t, err)
 
-	err = gen.Render(badTemplateName, doc, apiGroupsGVR, nil, false, &buf)
+	err = gen.Render(badTemplateName, doc, appsDeploymentGVR, nil, false, &buf)
 	require.NoError(t, err)
 	require.Equal(t, "ok", buf.String())
 }
@@ -53,7 +64,8 @@ func TestGeneratorContext(t *testing.T) {
 	var buf bytes.Buffer
 	var doc map[string]interface{}
 
-	err := json.Unmarshal([]byte(apiDiscoveryJSON), &doc)
+	appsv1Bytes := bytesForGV(t, appsv1Path)
+	err := json.Unmarshal(appsv1Bytes, &doc)
 	require.NoError(t, err)
 
 	gen := NewGenerator()
@@ -62,7 +74,7 @@ func TestGeneratorContext(t *testing.T) {
 
 	expectedContext := TemplateContext{
 		Document:  doc,
-		GVR:       apiGroupsGVR,
+		GVR:       appsDeploymentGVR,
 		Recursive: false,
 		FieldPath: nil,
 	}
@@ -80,3 +92,19 @@ func TestGeneratorContext(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, expectedContext, actualContext)
 }
+
+// bytesForGV returns the OpenAPI V3 spec for the passed
+// group/version as a byte slice. Assumes bytes are in json
+// format. The passed path string looks like:
+//
+//	apis/apps/v1
+func bytesForGV(t *testing.T, gvPath string) []byte {
+	fakeClient := openapitest.NewFileClient(t)
+	paths, err := fakeClient.Paths()
+	require.NoError(t, err)
+	gv, found := paths[gvPath]
+	require.True(t, found)
+	gvBytes, err := gv.Schema("application/json")
+	require.NoError(t, err)
+	return gvBytes
+}