Refactor

2025-09-14 22:10:32 +00:00 · 2020-01-30 22:37:59 -07:00
parent 19c6732de0
commit 8b42d0aff8
71 changed files with 4024 additions and 507 deletions
--- a/pkg/schema/collection.go
+++ b/pkg/schema/collection.go
@@ -3,26 +3,27 @@ package schema
 import (
 	"strings"

-	"github.com/rancher/norman/v2/pkg/data"
-	"github.com/rancher/norman/v2/pkg/types"
 	"github.com/rancher/steve/pkg/accesscontrol"
 	"github.com/rancher/steve/pkg/attributes"
-	"github.com/rancher/steve/pkg/table"
+	"github.com/rancher/steve/pkg/schema/table"
+	"github.com/rancher/steve/pkg/schemaserver/types"
+	"github.com/rancher/wrangler/pkg/data"
 	"github.com/rancher/wrangler/pkg/name"
+	"github.com/rancher/wrangler/pkg/schemas"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/authentication/user"
 )

 type Factory interface {
-	Schemas(user user.Info) (*types.Schemas, error)
+	Schemas(user user.Info) (*types.APISchemas, error)
 	ByGVR(gvr schema.GroupVersionResource) string
 	ByGVK(gvr schema.GroupVersionKind) string
 }

 type Collection struct {
 	toSync     int32
-	baseSchema *types.Schemas
-	schemas    map[string]*types.Schema
+	baseSchema *types.APISchemas
+	schemas    map[string]*types.APISchema
 	templates  map[string]*Template
 	byGVR      map[schema.GroupVersionResource]string
 	byGVK      map[schema.GroupVersionKind]string
@@ -34,20 +35,19 @@ type Template struct {
 	Group           string
 	Kind            string
 	ID              string
-	RegisterType    interface{}
-	Customize       func(*types.Schema)
+	Customize       func(*types.APISchema)
 	Formatter       types.Formatter
 	Store           types.Store
 	StoreFactory    func(types.Store) types.Store
-	Mapper          types.Mapper
+	Mapper          schemas.Mapper
 	Columns         []table.Column
 	ComputedColumns func(data.Object)
 }

-func NewCollection(baseSchema *types.Schemas, access *accesscontrol.AccessStore) *Collection {
+func NewCollection(baseSchema *types.APISchemas, access *accesscontrol.AccessStore) *Collection {
 	return &Collection{
 		baseSchema: baseSchema,
-		schemas:    map[string]*types.Schema{},
+		schemas:    map[string]*types.APISchema{},
 		templates:  map[string]*Template{},
 		byGVR:      map[schema.GroupVersionResource]string{},
 		byGVK:      map[schema.GroupVersionKind]string{},
@@ -55,7 +55,7 @@ func NewCollection(baseSchema *types.Schemas, access *accesscontrol.AccessStore)
 	}
 }

-func (c *Collection) Reset(schemas map[string]*types.Schema) {
+func (c *Collection) Reset(schemas map[string]*types.APISchema) {
 	byGVK := map[schema.GroupVersionKind]string{}
 	byGVR := map[schema.GroupVersionResource]string{}

@@ -75,7 +75,7 @@ func (c *Collection) Reset(schemas map[string]*types.Schema) {
 	c.byGVK = byGVK
 }

-func (c *Collection) Schema(id string) *types.Schema {
+func (c *Collection) Schema(id string) *types.APISchema {
 	return c.schemas[id]
 }

--- a/pkg/schema/converter/crd.go
+++ b/pkg/schema/converter/crd.go
@@ -1,17 +1,18 @@
 package converter

 import (
-	"github.com/rancher/norman/v2/pkg/types"
 	"github.com/rancher/steve/pkg/attributes"
-	"github.com/rancher/steve/pkg/table"
+	"github.com/rancher/steve/pkg/schema/table"
+	"github.com/rancher/steve/pkg/schemaserver/types"
 	"github.com/rancher/wrangler-api/pkg/generated/controllers/apiextensions.k8s.io/v1beta1"
+	"github.com/rancher/wrangler/pkg/schemas"
 	beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 )

 var (
-	staticFields = map[string]types.Field{
+	staticFields = map[string]schemas.Field{
 		"apiVersion": {
 			Type: "string",
 		},
@@ -24,7 +25,7 @@ var (
 	}
 )

-func AddCustomResources(crd v1beta1.CustomResourceDefinitionClient, schemas map[string]*types.Schema) error {
+func AddCustomResources(crd v1beta1.CustomResourceDefinitionClient, schemas map[string]*types.APISchema) error {
 	crds, err := crd.List(metav1.ListOptions{})
 	if err != nil {
 		return nil
@@ -57,7 +58,7 @@ func AddCustomResources(crd v1beta1.CustomResourceDefinitionClient, schemas map[
 	return nil
 }

-func forVersion(crd *beta1.CustomResourceDefinition, group, version, kind string, schemas map[string]*types.Schema, columnDefs []beta1.CustomResourceColumnDefinition, columns []table.Column) {
+func forVersion(crd *beta1.CustomResourceDefinition, group, version, kind string, schemasMap map[string]*types.APISchema, columnDefs []beta1.CustomResourceColumnDefinition, columns []table.Column) {
 	var versionColumns []table.Column
 	for _, col := range columnDefs {
 		versionColumns = append(versionColumns, table.Column{
@@ -77,7 +78,7 @@ func forVersion(crd *beta1.CustomResourceDefinition, group, version, kind string
 		Kind:    kind,
 	})

-	schema := schemas[id]
+	schema := schemasMap[id]
 	if schema == nil {
 		return
 	}
@@ -86,13 +87,13 @@ func forVersion(crd *beta1.CustomResourceDefinition, group, version, kind string
 	}

 	if crd.Spec.Validation != nil && crd.Spec.Validation.OpenAPIV3Schema != nil {
-		if fieldsSchema := modelV3ToSchema(id, crd.Spec.Validation.OpenAPIV3Schema, schemas); fieldsSchema != nil {
+		if fieldsSchema := modelV3ToSchema(id, crd.Spec.Validation.OpenAPIV3Schema, schemasMap); fieldsSchema != nil {
 			for k, v := range staticFields {
 				fieldsSchema.ResourceFields[k] = v
 			}
 			for k, v := range fieldsSchema.ResourceFields {
 				if schema.ResourceFields == nil {
-					schema.ResourceFields = map[string]types.Field{}
+					schema.ResourceFields = map[string]schemas.Field{}
 				}
 				if _, ok := schema.ResourceFields[k]; !ok {
 					schema.ResourceFields[k] = v
--- a/pkg/schema/converter/discovery.go
+++ b/pkg/schema/converter/discovery.go
@@ -3,9 +3,10 @@ package converter
 import (
 	"strings"

-	"github.com/rancher/norman/v2/pkg/types"
 	"github.com/rancher/steve/pkg/attributes"
+	"github.com/rancher/steve/pkg/schemaserver/types"
 	"github.com/rancher/wrangler/pkg/merr"
+	"github.com/rancher/wrangler/pkg/schemas"
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -18,11 +19,13 @@ var (
 	}
 )

-func AddDiscovery(client discovery.DiscoveryInterface, schemas map[string]*types.Schema) error {
+func AddDiscovery(client discovery.DiscoveryInterface, schemasMap map[string]*types.APISchema) error {
 	logrus.Info("Refreshing all schemas")

 	groups, resourceLists, err := client.ServerGroupsAndResources()
-	if err != nil {
+	if gd, ok := err.(*discovery.ErrGroupDiscoveryFailed); ok {
+		logrus.Errorf("Failed to read API for groups %v", gd.Groups)
+	} else if err != nil {
 		return err
 	}

@@ -35,7 +38,7 @@ func AddDiscovery(client discovery.DiscoveryInterface, schemas map[string]*types
 			errs = append(errs, err)
 		}

-		if err := refresh(gv, versions, resourceList, schemas); err != nil {
+		if err := refresh(gv, versions, resourceList, schemasMap); err != nil {
 			errs = append(errs, err)
 		}
 	}
@@ -51,7 +54,7 @@ func indexVersions(groups []*metav1.APIGroup) map[string]string {
 	return result
 }

-func refresh(gv schema.GroupVersion, groupToPreferredVersion map[string]string, resources *metav1.APIResourceList, schemas map[string]*types.Schema) error {
+func refresh(gv schema.GroupVersion, groupToPreferredVersion map[string]string, resources *metav1.APIResourceList, schemasMap map[string]*types.APISchema) error {
 	for _, resource := range resources.APIResources {
 		if strings.Contains(resource.Name, "/") {
 			continue
@@ -66,12 +69,12 @@ func refresh(gv schema.GroupVersion, groupToPreferredVersion map[string]string,

 		logrus.Infof("APIVersion %s/%s Kind %s", gvk.Group, gvk.Version, gvk.Kind)

-		schema := schemas[GVKToSchemaID(gvk)]
+		schema := schemasMap[GVKToSchemaID(gvk)]
 		if schema == nil {
-			schema = &types.Schema{
-				ID:      GVKToSchemaID(gvk),
-				Type:    "schema",
-				Dynamic: true,
+			schema = &types.APISchema{
+				Schema: &schemas.Schema{
+					ID:      GVKToSchemaID(gvk),
+				},
 			}
 			attributes.SetGVK(schema, gvk)
 		}
@@ -85,7 +88,7 @@ func refresh(gv schema.GroupVersion, groupToPreferredVersion map[string]string,
 			attributes.SetPreferredGroup(schema, group)
 		}

-		schemas[schema.ID] = schema
+		schemasMap[schema.ID] = schema
 	}

 	return nil
--- a/pkg/schema/converter/k8stonorman.go
+++ b/pkg/schema/converter/k8stonorman.go
@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"strings"

-	"github.com/rancher/norman/v2/pkg/types"
+	"github.com/rancher/steve/pkg/schemaserver/types"
 	"github.com/rancher/wrangler-api/pkg/generated/controllers/apiextensions.k8s.io/v1beta1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/discovery"
@@ -24,8 +24,8 @@ func GVRToPluralName(gvr schema.GroupVersionResource) string {
 	return fmt.Sprintf("%s.%s.%s", gvr.Group, gvr.Version, gvr.Resource)
 }

-func ToSchemas(crd v1beta1.CustomResourceDefinitionClient, client discovery.DiscoveryInterface) (map[string]*types.Schema, error) {
-	result := map[string]*types.Schema{}
+func ToSchemas(crd v1beta1.CustomResourceDefinitionClient, client discovery.DiscoveryInterface) (map[string]*types.APISchema, error) {
+	result := map[string]*types.APISchema{}

 	if err := AddOpenAPI(client, result); err != nil {
 		return nil, err
--- a/pkg/schema/converter/openapi.go
+++ b/pkg/schema/converter/openapi.go
@@ -1,23 +1,24 @@
 package converter

 import (
-	"github.com/rancher/norman/v2/pkg/types"
-	"github.com/rancher/norman/v2/pkg/types/convert"
 	"github.com/rancher/steve/pkg/attributes"
+	"github.com/rancher/steve/pkg/schemaserver/types"
+	"github.com/rancher/wrangler/pkg/data/convert"
+	"github.com/rancher/wrangler/pkg/schemas"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/discovery"
 	"k8s.io/kube-openapi/pkg/util/proto"
 )

-func modelToSchema(modelName string, k *proto.Kind) *types.Schema {
-	s := types.Schema{
-		ID:             modelName,
-		Type:           "schema",
-		ResourceFields: map[string]types.Field{},
-		Attributes:     map[string]interface{}{},
-		Description:    k.GetDescription(),
-		Dynamic:        true,
+func modelToSchema(modelName string, k *proto.Kind) *types.APISchema {
+	s := types.APISchema{
+		Schema: &schemas.Schema{
+			ID:             modelName,
+			ResourceFields: map[string]schemas.Field{},
+			Attributes:     map[string]interface{}{},
+			Description:    k.GetDescription(),
+		},
 	}

 	for fieldName, schemaField := range k.Fields {
@@ -49,7 +50,7 @@ func modelToSchema(modelName string, k *proto.Kind) *types.Schema {
 	return &s
 }

-func AddOpenAPI(client discovery.DiscoveryInterface, schemas map[string]*types.Schema) error {
+func AddOpenAPI(client discovery.DiscoveryInterface, schemas map[string]*types.APISchema) error {
 	openapi, err := client.OpenAPISchema()
 	if err != nil {
 		return err
@@ -71,10 +72,9 @@ func AddOpenAPI(client discovery.DiscoveryInterface, schemas map[string]*types.S
 	return nil
 }

-func toField(schema proto.Schema) types.Field {
-	f := types.Field{
+func toField(schema proto.Schema) schemas.Field {
+	f := schemas.Field{
 		Description: schema.GetDescription(),
-		Nullable:    true,
 		Create:      true,
 		Update:      true,
 	}
--- a/pkg/schema/converter/openapiv3.go
+++ b/pkg/schema/converter/openapiv3.go
@@ -0,0 +1,80 @@
+package converter
+
+import (
+	"github.com/rancher/steve/pkg/schemaserver/types"
+	"github.com/rancher/wrangler/pkg/schemas"
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+)
+
+func modelV3ToSchema(name string, k *v1beta1.JSONSchemaProps, schemasMap map[string]*types.APISchema) *types.APISchema {
+	s := types.APISchema{
+		Schema: &schemas.Schema{
+			ID:             name,
+			ResourceFields: map[string]schemas.Field{},
+			Attributes:     map[string]interface{}{},
+			Description:    k.Description,
+		},
+	}
+
+	for fieldName, schemaField := range k.Properties {
+		s.ResourceFields[fieldName] = toResourceField(name+"."+fieldName, schemaField, schemasMap)
+	}
+
+	for _, fieldName := range k.Required {
+		if f, ok := s.ResourceFields[fieldName]; ok {
+			f.Required = true
+			s.ResourceFields[fieldName] = f
+		}
+	}
+
+	if _, ok := schemasMap[s.ID]; !ok {
+		schemasMap[s.ID] = &s
+	}
+
+	return &s
+}
+
+func toResourceField(name string, schema v1beta1.JSONSchemaProps, schemasMap map[string]*types.APISchema) schemas.Field {
+	f := schemas.Field{
+		Description: schema.Description,
+		Nullable:    true,
+		Create:      true,
+		Update:      true,
+	}
+	var itemSchema *v1beta1.JSONSchemaProps
+	if schema.Items != nil {
+		if schema.Items.Schema != nil {
+			itemSchema = schema.Items.Schema
+		} else if len(schema.Items.JSONSchemas) > 0 {
+			itemSchema = &schema.Items.JSONSchemas[0]
+		}
+	}
+
+	switch schema.Type {
+	case "array":
+		if itemSchema == nil {
+			f.Type = "array[json]"
+		} else {
+			f.Type = "array[" + name + "]"
+			modelV3ToSchema(name, itemSchema, schemasMap)
+		}
+	case "object":
+		if schema.AdditionalProperties != nil && schema.AdditionalProperties.Schema != nil {
+			f.Type = "map[" + name + "]"
+			modelV3ToSchema(name, schema.AdditionalProperties.Schema, schemasMap)
+		} else {
+			f.Type = name
+			modelV3ToSchema(name, &schema, schemasMap)
+		}
+	case "number":
+		f.Type = "int"
+	default:
+		f.Type = schema.Type
+	}
+
+	if f.Type == "" {
+		f.Type = "json"
+	}
+
+	return f
+}
--- a/pkg/schema/defaultmapper.go
+++ b/pkg/schema/defaultmapper.go
@@ -3,16 +3,18 @@ package schema
 import (
 	"fmt"

-	"github.com/rancher/norman/v2/pkg/data"
-	"github.com/rancher/norman/v2/pkg/types"
+	"github.com/rancher/steve/pkg/schemaserver/types"
+	"github.com/rancher/wrangler/pkg/data"
+	"github.com/rancher/wrangler/pkg/schemas"
+	"github.com/rancher/wrangler/pkg/schemas/mappers"
 )

-func newDefaultMapper() types.Mapper {
+func newDefaultMapper() schemas.Mapper {
 	return &defaultMapper{}
 }

 type defaultMapper struct {
-	types.EmptyMapper
+	mappers.EmptyMapper
 }

 func (d *defaultMapper) FromInternal(data data.Object) {
--- a/pkg/schema/factory.go
+++ b/pkg/schema/factory.go
@@ -4,51 +4,42 @@ import (
 	"fmt"
 	"net/http"

-	"github.com/rancher/norman/v2/pkg/api/builtin"
-	"github.com/rancher/norman/v2/pkg/types"
 	"github.com/rancher/steve/pkg/accesscontrol"
 	"github.com/rancher/steve/pkg/attributes"
-	"github.com/rancher/steve/pkg/table"
+	"github.com/rancher/steve/pkg/schema/table"
+	"github.com/rancher/steve/pkg/schemaserver/builtin"
+	"github.com/rancher/steve/pkg/schemaserver/types"
+	"github.com/rancher/wrangler/pkg/schemas"
 	"k8s.io/apiserver/pkg/authentication/user"
 )

-func newSchemas() (*types.Schemas, error) {
-	s, err := types.NewSchemas(builtin.Schemas)
-	if err != nil {
+func newSchemas() (*types.APISchemas, error) {
+	apiSchemas := types.EmptyAPISchemas()
+	if err := apiSchemas.AddSchemas(builtin.Schemas); err != nil {
 		return nil, err
 	}
-	s.DefaultMapper = func() types.Mapper {
+	apiSchemas.InternalSchemas.DefaultMapper = func() schemas.Mapper {
 		return newDefaultMapper()
 	}

-	return s, nil
+	return apiSchemas, nil
 }

-func (c *Collection) Schemas(user user.Info) (*types.Schemas, error) {
+func (c *Collection) Schemas(user user.Info) (*types.APISchemas, error) {
 	access := c.as.AccessFor(user)
 	return c.schemasForSubject(access)
 }

-func (c *Collection) schemasForSubject(access *accesscontrol.AccessSet) (*types.Schemas, error) {
+func (c *Collection) schemasForSubject(access *accesscontrol.AccessSet) (*types.APISchemas, error) {
 	result, err := newSchemas()
 	if err != nil {
 		return nil, err
 	}

-	if _, err := result.AddSchemas(c.baseSchema); err != nil {
+	if err := result.AddSchemas(c.baseSchema); err != nil {
 		return nil, err
 	}

-	for _, template := range c.templates {
-		if template.RegisterType != nil {
-			s, err := result.Import(template.RegisterType)
-			if err != nil {
-				return nil, err
-			}
-			c.applyTemplates(result, s)
-		}
-	}
-
 	for _, s := range c.schemas {
 		gr := attributes.GR(s)

@@ -60,7 +51,7 @@ func (c *Collection) schemasForSubject(access *accesscontrol.AccessSet) (*types.
 		}

 		verbs := attributes.Verbs(s)
-		verbAccess := accesscontrol.AccessListMap{}
+		verbAccess := accesscontrol.AccessListByVerb{}

 		for _, verb := range verbs {
 			a := access.AccessListFor(verb, gr)
@@ -100,7 +91,7 @@ func (c *Collection) schemasForSubject(access *accesscontrol.AccessSet) (*types.
 	return result, nil
 }

-func (c *Collection) applyTemplates(schemas *types.Schemas, schema *types.Schema) {
+func (c *Collection) applyTemplates(schemas *types.APISchemas, schema *types.APISchema) {
 	templates := []*Template{
 		c.templates[schema.ID],
 		c.templates[fmt.Sprintf("%s/%s", attributes.Group(schema), attributes.Kind(schema))],
@@ -112,7 +103,7 @@ func (c *Collection) applyTemplates(schemas *types.Schemas, schema *types.Schema
 			continue
 		}
 		if t.Mapper != nil {
-			schemas.AddMapper(schema.ID, t.Mapper)
+			schemas.InternalSchemas.AddMapper(schema.ID, t.Mapper)
 		}
 		if schema.Formatter == nil {
 			schema.Formatter = t.Formatter
@@ -128,7 +119,7 @@ func (c *Collection) applyTemplates(schemas *types.Schemas, schema *types.Schema
 			t.Customize(schema)
 		}
 		if len(t.Columns) > 0 {
-			schemas.AddMapper(schema.ID, table.NewColumns(t.ComputedColumns, t.Columns...))
+			schemas.InternalSchemas.AddMapper(schema.ID, table.NewColumns(t.ComputedColumns, t.Columns...))
 		}
 	}
 }