Reducing number of generated schemas

Changes steve to only return top level schema definitions rather
than a full defined schema for every field. Also adds basic docs
and fixes a bug with schema generation on versioned crds

pkg/schema/converter/crd.go

@@ -27,7 +27,9 @@ var (
 	}
 )
 
-func AddCustomResources(crd apiextv1.CustomResourceDefinitionClient, schemas map[string]*types.APISchema) error {
+// addCustomResources uses the openAPISchema defined on CRDs to provide field definitions to previously discovered schemas.
+// Note that this function does not create new schemas - it only adds details to resources already present in the schemas map.
+func addCustomResources(crd apiextv1.CustomResourceDefinitionClient, schemas map[string]*types.APISchema) error {
 	crds, err := crd.List(metav1.ListOptions{})
 	if err != nil {
 		return nil
@@ -41,14 +43,14 @@ func AddCustomResources(crd apiextv1.CustomResourceDefinitionClient, schemas map
 		group, kind := crd.Spec.Group, crd.Status.AcceptedNames.Kind
 
 		for _, version := range crd.Spec.Versions {
-			forVersion(&crd, group, kind, version, schemas)
+			forVersion(group, kind, version, schemas)
 		}
 	}
 
 	return nil
 }
 
-func forVersion(crd *v1.CustomResourceDefinition, group, kind string, version v1.CustomResourceDefinitionVersion, schemasMap map[string]*types.APISchema) {
+func forVersion(group, kind string, version v1.CustomResourceDefinitionVersion, schemasMap map[string]*types.APISchema) {
 	var versionColumns []table.Column
 	for _, col := range version.AdditionalPrinterColumns {
 		versionColumns = append(versionColumns, table.Column{
@@ -73,18 +75,6 @@ func forVersion(crd *v1.CustomResourceDefinition, group, kind string, version v1
 		attributes.SetColumns(schema, versionColumns)
 	}
 	if version.Schema != nil && version.Schema.OpenAPIV3Schema != nil {
-		if fieldsSchema := modelV3ToSchema(id, crd.Spec.Versions[0].Schema.OpenAPIV3Schema, schemasMap); fieldsSchema != nil {
-			for k, v := range staticFields {
-				fieldsSchema.ResourceFields[k] = v
-			}
-			for k, v := range fieldsSchema.ResourceFields {
-				if schema.ResourceFields == nil {
-					schema.ResourceFields = map[string]schemas.Field{}
-				}
-				if _, ok := schema.ResourceFields[k]; !ok {
-					schema.ResourceFields[k] = v
-				}
-			}
-		}
+		schema.Description = version.Schema.OpenAPIV3Schema.Description
 	}
 }

pkg/schema/converter/description.go

@@ -0,0 +1,45 @@
+package converter
+
+import (
+	"github.com/rancher/apiserver/pkg/types"
+	"github.com/sirupsen/logrus"
+	"k8s.io/client-go/discovery"
+	"k8s.io/kube-openapi/pkg/util/proto"
+)
+
+// addDescription adds a description to all schemas in schemas using the openapi v2 definitions from k8s.
+// Will not add new schemas, only mutate existing ones. Returns an error if the definitions could not be retrieved.
+func addDescription(client discovery.DiscoveryInterface, schemas map[string]*types.APISchema) error {
+	openapi, err := client.OpenAPISchema()
+	if err != nil {
+		return err
+	}
+
+	models, err := proto.NewOpenAPIData(openapi)
+	if err != nil {
+		return err
+	}
+
+	for _, modelName := range models.ListModels() {
+		model := models.LookupModel(modelName)
+		if k, ok := model.(*proto.Kind); ok {
+			gvk := GetGVKForKind(k)
+			if gvk == nil {
+				// kind was not for top level gvk, we can skip this resource
+				logrus.Tracef("when adding schema descriptions, will not add description for kind %s, which is not a top level resource", k.Path.String())
+				continue
+			}
+			schemaID := GVKToVersionedSchemaID(*gvk)
+			schema, ok := schemas[schemaID]
+			// some kinds have a gvk but don't correspond to a schema (like a podList). We can
+			// skip these resources as well
+			if !ok {
+				logrus.Tracef("when adding schema descriptions, will not add description for ID %s, which is not in schemas", schemaID)
+				continue
+			}
+			schema.Description = k.GetDescription()
+		}
+	}
+
+	return nil
+}

pkg/schema/converter/discovery.go

@@ -22,7 +22,9 @@ var (
 	}
 )
 
-func AddDiscovery(client discovery.DiscoveryInterface, schemasMap map[string]*types.APISchema) error {
+// addDiscovery uses a k8s discovery client to create very basic schemas for all registered groups/resources. Other
+// functions, such as AddCustomResources are used to add more details to these schemas later on.
+func addDiscovery(client discovery.DiscoveryInterface, schemasMap map[string]*types.APISchema) error {
 	groups, resourceLists, err := client.ServerGroupsAndResources()
 	if gd, ok := err.(*discovery.ErrGroupDiscoveryFailed); ok {
 		logrus.Errorf("Failed to read API for groups %v", gd.Groups)

pkg/schema/converter/k8stonorman.go

@@ -1,3 +1,5 @@
+// Package converter is responsible for converting the types registered with a k8s server to schemas which can be used
+// by the UI (and other consumers) to discover the resources available and the current user's permissions.
 package converter
 
 import (
@@ -5,9 +7,18 @@ import (
 	"strings"
 
 	"github.com/rancher/apiserver/pkg/types"
+	"github.com/rancher/norman/types/convert"
 	v1 "github.com/rancher/wrangler/v2/pkg/generated/controllers/apiextensions.k8s.io/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/discovery"
+	"k8s.io/kube-openapi/pkg/util/proto"
+)
+
+const (
+	gvkExtensionName    = "x-kubernetes-group-version-kind"
+	gvkExtensionGroup   = "group"
+	gvkExtensionVersion = "version"
+	gvkExtensionKind    = "kind"
 )
 
 func GVKToVersionedSchemaID(gvk schema.GroupVersionKind) string {
@@ -38,18 +49,40 @@ func GVRToPluralName(gvr schema.GroupVersionResource) string {
 	return fmt.Sprintf("%s.%s", gvr.Group, gvr.Resource)
 }
 
+// GetGVKForKind attempts to retrieve a GVK for a given Kind. Not all kind represent top level resources,
+// so this function may return nil if the kind did not have a gvk extension
+func GetGVKForKind(kind *proto.Kind) *schema.GroupVersionKind {
+	extensions, ok := kind.Extensions[gvkExtensionName].([]any)
+	if !ok {
+		return nil
+	}
+	for _, extension := range extensions {
+		if gvkExtension, ok := extension.(map[any]any); ok {
+			gvk := schema.GroupVersionKind{
+				Group:   convert.ToString(gvkExtension[gvkExtensionGroup]),
+				Version: convert.ToString(gvkExtension[gvkExtensionVersion]),
+				Kind:    convert.ToString(gvkExtension[gvkExtensionKind]),
+			}
+			return &gvk
+		}
+	}
+	return nil
+}
+
+// ToSchemas creates the schemas for a K8s server, using client to discover groups/resources, and crd to potentially
+// add additional information about new fields/resources. Mostly ties together AddDiscovery and AddCustomResources.
 func ToSchemas(crd v1.CustomResourceDefinitionClient, client discovery.DiscoveryInterface) (map[string]*types.APISchema, error) {
 	result := map[string]*types.APISchema{}
 
-	if err := AddOpenAPI(client, result); err != nil {
+	if err := addDiscovery(client, result); err != nil {
 		return nil, err
 	}
 
-	if err := AddDiscovery(client, result); err != nil {
+	if err := addCustomResources(crd, result); err != nil {
 		return nil, err
 	}
 
-	if err := AddCustomResources(crd, result); err != nil {
+	if err := addDescription(client, result); err != nil {
 		return nil, err
 	}
 

pkg/schema/converter/openapi.go

@@ -1,117 +0,0 @@
-package converter
-
-import (
-	"github.com/rancher/apiserver/pkg/types"
-	"github.com/rancher/steve/pkg/attributes"
-	"github.com/rancher/wrangler/v2/pkg/data/convert"
-	"github.com/rancher/wrangler/v2/pkg/schemas"
-	"github.com/sirupsen/logrus"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/client-go/discovery"
-	"k8s.io/kube-openapi/pkg/util/proto"
-)
-
-func modelToSchema(modelName string, k *proto.Kind) *types.APISchema {
-	s := types.APISchema{
-		Schema: &schemas.Schema{
-			ID:             modelName,
-			ResourceFields: map[string]schemas.Field{},
-			Attributes:     map[string]interface{}{},
-			Description:    k.GetDescription(),
-		},
-	}
-
-	for fieldName, schemaField := range k.Fields {
-		s.ResourceFields[fieldName] = toField(schemaField)
-	}
-
-	for _, fieldName := range k.RequiredFields {
-		if f, ok := s.ResourceFields[fieldName]; ok {
-			f.Required = true
-			s.ResourceFields[fieldName] = f
-		}
-	}
-
-	if ms, ok := k.Extensions["x-kubernetes-group-version-kind"].([]interface{}); ok {
-		for _, mv := range ms {
-			if m, ok := mv.(map[interface{}]interface{}); ok {
-				gvk := schema.GroupVersionKind{
-					Group:   convert.ToString(m["group"]),
-					Version: convert.ToString(m["version"]),
-					Kind:    convert.ToString(m["kind"]),
-				}
-
-				s.ID = GVKToVersionedSchemaID(gvk)
-				attributes.SetGVK(&s, gvk)
-			}
-		}
-	}
-
-	for k, v := range s.ResourceFields {
-		if types.ReservedFields[k] {
-			s.ResourceFields["_"+k] = v
-			delete(s.ResourceFields, k)
-		}
-	}
-
-	return &s
-}
-
-func AddOpenAPI(client discovery.DiscoveryInterface, schemas map[string]*types.APISchema) error {
-	openapi, err := client.OpenAPISchema()
-	if err != nil {
-		return err
-	}
-
-	models, err := proto.NewOpenAPIData(openapi)
-	if err != nil {
-		return err
-	}
-
-	for _, modelName := range models.ListModels() {
-		model := models.LookupModel(modelName)
-		if k, ok := model.(*proto.Kind); ok {
-			schema := modelToSchema(modelName, k)
-			schemas[schema.ID] = schema
-		}
-	}
-
-	return nil
-}
-
-func toField(schema proto.Schema) schemas.Field {
-	f := schemas.Field{
-		Description: schema.GetDescription(),
-		Create:      true,
-		Update:      true,
-	}
-	switch v := schema.(type) {
-	case *proto.Array:
-		f.Type = "array[" + toField(v.SubType).Type + "]"
-	case *proto.Primitive:
-		if v.Type == "number" || v.Type == "integer" {
-			f.Type = "int"
-		} else {
-			f.Type = v.Type
-		}
-	case *proto.Map:
-		f.Type = "map[" + toField(v.SubType).Type + "]"
-	case *proto.Kind:
-		f.Type = v.Path.String()
-	case proto.Reference:
-		sub := v.SubSchema()
-		if p, ok := sub.(*proto.Primitive); ok {
-			f.Type = p.Type
-		} else {
-			f.Type = sub.GetPath().String()
-		}
-	case *proto.Arbitrary:
-		logrus.Debugf("arbitrary type: %v", schema)
-		f.Type = "json"
-	default:
-		logrus.Errorf("unknown type: %v", schema)
-		f.Type = "json"
-	}
-
-	return f
-}

pkg/schema/converter/openapiv3.go

@@ -1,92 +0,0 @@
-package converter
-
-import (
-	"github.com/rancher/apiserver/pkg/types"
-	"github.com/rancher/wrangler/v2/pkg/schemas"
-	v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
-)
-
-func modelV3ToSchema(name string, k *v1.JSONSchemaProps, schemasMap map[string]*types.APISchema) *types.APISchema {
-	s := types.APISchema{
-		Schema: &schemas.Schema{
-			ID:             name,
-			ResourceFields: map[string]schemas.Field{},
-			Attributes:     map[string]interface{}{},
-			Description:    k.Description,
-		},
-	}
-
-	for fieldName, schemaField := range k.Properties {
-		s.ResourceFields[fieldName] = toResourceField(name+"."+fieldName, schemaField, schemasMap)
-	}
-
-	for _, fieldName := range k.Required {
-		if f, ok := s.ResourceFields[fieldName]; ok {
-			f.Required = true
-			s.ResourceFields[fieldName] = f
-		}
-	}
-
-	if existing, ok := schemasMap[s.ID]; ok && len(existing.Attributes) > 0 {
-		s.Attributes = existing.Attributes
-	}
-	schemasMap[s.ID] = &s
-
-	for k, v := range s.ResourceFields {
-		if types.ReservedFields[k] {
-			s.ResourceFields["_"+k] = v
-			delete(s.ResourceFields, k)
-		}
-	}
-
-	return &s
-}
-
-func toResourceField(name string, schema v1.JSONSchemaProps, schemasMap map[string]*types.APISchema) schemas.Field {
-	f := schemas.Field{
-		Description: schema.Description,
-		Nullable:    true,
-		Create:      true,
-		Update:      true,
-	}
-	var itemSchema *v1.JSONSchemaProps
-	if schema.Items != nil {
-		if schema.Items.Schema != nil {
-			itemSchema = schema.Items.Schema
-		} else if len(schema.Items.JSONSchemas) > 0 {
-			itemSchema = &schema.Items.JSONSchemas[0]
-		}
-	}
-
-	switch schema.Type {
-	case "array":
-		if itemSchema == nil {
-			f.Type = "array[json]"
-		} else if itemSchema.Type == "object" {
-			f.Type = "array[" + name + "]"
-			modelV3ToSchema(name, itemSchema, schemasMap)
-		} else {
-			f.Type = "array[" + itemSchema.Type + "]"
-		}
-	case "object":
-		if schema.AdditionalProperties != nil && schema.AdditionalProperties.Schema != nil && schema.AdditionalProperties.Schema.Type == "object" {
-			f.Type = "map[" + name + "]"
-			modelV3ToSchema(name, schema.AdditionalProperties.Schema, schemasMap)
-		} else if schema.AdditionalProperties != nil && schema.AdditionalProperties.Schema != nil {
-			f.Type = "map[" + schema.AdditionalProperties.Schema.Type + "]"
-		} else {
-			f.Type = name
-			modelV3ToSchema(name, &schema, schemasMap)
-		}
-	case "number":
-		f.Type = "int"
-	default:
-		f.Type = schema.Type
-	}
-
-	if f.Type == "" {
-		f.Type = "json"
-	}
-
-	return f
-}