diff --git a/examples/examples_test.go b/examples/examples_test.go index a672401da46..8c5ce092e53 100644 --- a/examples/examples_test.go +++ b/examples/examples_test.go @@ -29,6 +29,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest" "github.com/GoogleCloudPlatform/kubernetes/pkg/api/validation" "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util/yaml" "github.com/golang/glog" ) @@ -88,6 +89,15 @@ func walkJSONFiles(inDir string, fn func(name, path string, data []byte)) error return err } name := strings.TrimSuffix(file, ext) + + if ext == ".yaml" { + out, err := yaml.ToJSON(data) + if err != nil { + return err + } + data = out + } + fn(name, path, data) } return nil @@ -215,14 +225,18 @@ func TestReadme(t *testing.T) { //t.Logf("testing (%s): \n%s", subtype, content) expectedType := &api.Pod{} - if err := latest.Codec.DecodeInto([]byte(content), expectedType); err != nil { + json, err := yaml.ToJSON([]byte(content)) + if err != nil { + t.Errorf("%s could not be converted to JSON: %v\n%s", path, err, string(content)) + } + if err := latest.Codec.DecodeInto(json, expectedType); err != nil { t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(content)) continue } if errors := validateObject(expectedType); len(errors) > 0 { t.Errorf("%s did not validate correctly: %v", path, errors) } - _, err := latest.Codec.Encode(expectedType) + _, err = latest.Codec.Encode(expectedType) if err != nil { t.Errorf("Could not encode object: %v", err) continue diff --git a/examples/limitrange/valid-pod.json b/examples/limitrange/valid-pod.json index 01a79159d9b..8e3d50e6a47 100644 --- a/examples/limitrange/valid-pod.json +++ b/examples/limitrange/valid-pod.json @@ -13,8 +13,8 @@ "name": "kubernetes-serve-hostname", "image": "kubernetes/serve_hostname", "cpu": 1000, - "memory": 1048576, + "memory": 1048576 }] } - }, + } } diff --git a/pkg/api/validation/schema.go b/pkg/api/validation/schema.go index 4bfaa7536e7..37757614757 100644 --- a/pkg/api/validation/schema.go +++ b/pkg/api/validation/schema.go @@ -22,9 +22,9 @@ import ( "reflect" "strings" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util/yaml" "github.com/emicklei/go-restful/swagger" "github.com/golang/glog" - "gopkg.in/yaml.v2" ) type InvalidTypeError struct { @@ -65,11 +65,15 @@ func NewSwaggerSchemaFromBytes(data []byte) (Schema, error) { func (s *SwaggerSchema) ValidateBytes(data []byte) error { var obj interface{} - err := yaml.Unmarshal(data, &obj) + out, err := yaml.ToJSON(data) if err != nil { return err } - fields := obj.(map[interface{}]interface{}) + data = out + if err := json.Unmarshal(data, &obj); err != nil { + return err + } + fields := obj.(map[string]interface{}) apiVersion := fields["apiVersion"].(string) kind := fields["kind"].(string) return s.ValidateObject(obj, apiVersion, "", apiVersion+"."+kind) @@ -84,12 +88,12 @@ func (s *SwaggerSchema) ValidateObject(obj interface{}, apiVersion, fieldName, t return nil } properties := model.Properties - fields := obj.(map[interface{}]interface{}) + fields := obj.(map[string]interface{}) if len(fieldName) > 0 { fieldName = fieldName + "." } for key, value := range fields { - details, ok := properties[key.(string)] + details, ok := properties[key] if !ok { glog.V(2).Infof("couldn't find properties for %s, skipping", key) continue @@ -99,7 +103,7 @@ func (s *SwaggerSchema) ValidateObject(obj interface{}, apiVersion, fieldName, t glog.V(2).Infof("Skipping nil field: %s", key) continue } - err := s.validateField(value, apiVersion, fieldName+key.(string), fieldType, &details) + err := s.validateField(value, apiVersion, fieldName+key, fieldType, &details) if err != nil { glog.Errorf("Validation failed for: %s, %v", key, value) return err diff --git a/pkg/client/clientcmd/api/latest/latest.go b/pkg/client/clientcmd/api/latest/latest.go index be7657ae2b5..260650700e8 100644 --- a/pkg/client/clientcmd/api/latest/latest.go +++ b/pkg/client/clientcmd/api/latest/latest.go @@ -18,6 +18,7 @@ package latest import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd/api/v1" + "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" ) // Version is the string that represents the current external default version. @@ -37,4 +38,4 @@ var Versions = []string{"v1"} // the latest supported version. Use this Codec when writing to // disk, a data store that is not dynamically versioned, or in tests. // This codec can decode any object that Kubernetes is aware of. -var Codec = v1.Codec +var Codec = runtime.YAMLDecoder(v1.Codec) diff --git a/pkg/conversion/decode.go b/pkg/conversion/decode.go index f8ef14d1798..1c98fae9bbf 100644 --- a/pkg/conversion/decode.go +++ b/pkg/conversion/decode.go @@ -17,13 +17,12 @@ limitations under the License. package conversion import ( + "encoding/json" "errors" "fmt" - - "github.com/ghodss/yaml" ) -// Decode converts a YAML or JSON string back into a pointer to an api object. +// Decode converts a JSON string back into a pointer to an api object. // Deduces the type based upon the fields added by the MetaInsertionFactory // technique. The object will be converted, if necessary, into the // s.InternalVersion type before being returned. Decode will not decode @@ -44,16 +43,12 @@ func (s *Scheme) Decode(data []byte) (interface{}, error) { return nil, err } - // yaml is a superset of json, so we use it to decode here. That way, - // we understand both. - err = yaml.Unmarshal(data, obj) - if err != nil { + if err := json.Unmarshal(data, obj); err != nil { return nil, err } // Version and Kind should be blank in memory. - err = s.SetVersionAndKind("", "", obj) - if err != nil { + if err := s.SetVersionAndKind("", "", obj); err != nil { return nil, err } @@ -63,8 +58,7 @@ func (s *Scheme) Decode(data []byte) (interface{}, error) { if err != nil { return nil, err } - err = s.converter.Convert(obj, objOut, 0, s.generateConvertMeta(version, s.InternalVersion)) - if err != nil { + if err := s.converter.Convert(obj, objOut, 0, s.generateConvertMeta(version, s.InternalVersion)); err != nil { return nil, err } obj = objOut @@ -72,16 +66,13 @@ func (s *Scheme) Decode(data []byte) (interface{}, error) { return obj, nil } -// DecodeInto parses a YAML or JSON string and stores it in obj. Returns an error +// DecodeInto parses a JSON string and stores it in obj. Returns an error // if data.Kind is set and doesn't match the type of obj. Obj should be a // pointer to an api type. // If obj's version doesn't match that in data, an attempt will be made to convert // data into obj's version. func (s *Scheme) DecodeInto(data []byte, obj interface{}) error { if len(data) == 0 { - // This is valid YAML, but it's a bad idea not to return an error - // for an empty string-- that's almost certainly not what the caller - // was expecting. return errors.New("empty input") } dataVersion, dataKind, err := s.DataVersionAndKind(data) @@ -107,14 +98,10 @@ func (s *Scheme) DecodeInto(data []byte, obj interface{}) error { if err != nil { return err } - // yaml is a superset of json, so we use it to decode here. That way, - // we understand both. - err = yaml.Unmarshal(data, external) - if err != nil { + if err := json.Unmarshal(data, external); err != nil { return err } - err = s.converter.Convert(external, obj, 0, s.generateConvertMeta(dataVersion, objVersion)) - if err != nil { + if err := s.converter.Convert(external, obj, 0, s.generateConvertMeta(dataVersion, objVersion)); err != nil { return err } diff --git a/pkg/conversion/meta.go b/pkg/conversion/meta.go index 18c0d966e93..5dccd560303 100644 --- a/pkg/conversion/meta.go +++ b/pkg/conversion/meta.go @@ -17,10 +17,9 @@ limitations under the License. package conversion import ( + "encoding/json" "fmt" "reflect" - - "github.com/ghodss/yaml" ) // MetaFactory is used to store and retrieve the version and kind @@ -33,13 +32,13 @@ type MetaFactory interface { Interpret(data []byte) (version, kind string, err error) } -// DefaultMetaFactory is a default factory for versioning objects in JSON/YAML. The object +// DefaultMetaFactory is a default factory for versioning objects in JSON. The object // in memory and in the default JSON serialization will use the "kind" and "apiVersion" // fields. var DefaultMetaFactory = SimpleMetaFactory{KindField: "Kind", VersionField: "APIVersion"} // SimpleMetaFactory provides default methods for retrieving the type and version of objects -// that are identified with an "apiVersion" and "kind" fields in their JSON/YAML +// that are identified with an "apiVersion" and "kind" fields in their JSON // serialization. It may be parameterized with the names of the fields in memory, or an // optional list of base structs to search for those fields in memory. type SimpleMetaFactory struct { @@ -51,16 +50,14 @@ type SimpleMetaFactory struct { BaseFields []string } -// Interpret will return the APIVersion and Kind of the JSON/YAML wire-format +// Interpret will return the APIVersion and Kind of the JSON wire-format // encoding of an object, or an error. func (SimpleMetaFactory) Interpret(data []byte) (version, kind string, err error) { findKind := struct { APIVersion string `json:"apiVersion,omitempty"` Kind string `json:"kind,omitempty"` }{} - // yaml is a superset of json, so we use it to decode here. That way, - // we understand both. - err = yaml.Unmarshal(data, &findKind) + err = json.Unmarshal(data, &findKind) if err != nil { return "", "", fmt.Errorf("couldn't get version/kind: %v", err) } diff --git a/pkg/kubectl/cmd/rollingupdate.go b/pkg/kubectl/cmd/rollingupdate.go index fb655a82a8d..80fa70efad8 100644 --- a/pkg/kubectl/cmd/rollingupdate.go +++ b/pkg/kubectl/cmd/rollingupdate.go @@ -23,6 +23,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl/cmd/util" + "github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl/resource" "github.com/spf13/cobra" ) @@ -72,56 +73,41 @@ func RunRollingUpdate(f *Factory, out io.Writer, cmd *cobra.Command, args []stri return util.UsageError(cmd, "Must specify the controller to update") } oldName := args[0] - schema, err := f.Validator() - if err != nil { - return err - } - - clientConfig, err := f.ClientConfig() - if err != nil { - return err - } - cmdApiVersion := clientConfig.Version - - mapper, typer := f.Object() - // TODO: use resource.Builder instead - mapping, namespace, newName, data, err := util.ResourceFromFile(filename, typer, mapper, schema, cmdApiVersion) - if err != nil { - return err - } - if mapping.Kind != "ReplicationController" { - return util.UsageError(cmd, "%s does not specify a valid ReplicationController", filename) - } - if oldName == newName { - return util.UsageError(cmd, "%s cannot have the same name as the existing ReplicationController %s", - filename, oldName) - } cmdNamespace, err := f.DefaultNamespace() if err != nil { return err } + + mapper, typer := f.Object() // TODO: use resource.Builder instead - err = util.CompareNamespace(cmdNamespace, namespace) + obj, err := resource.NewBuilder(mapper, typer, f.ClientMapperForCommand(cmd)). + NamespaceParam(cmdNamespace).RequireNamespace(). + FilenameParam(filename). + Do(). + Object() if err != nil { return err } + newRc, ok := obj.(*api.ReplicationController) + if !ok { + return util.UsageError(cmd, "%s does not specify a valid ReplicationController", filename) + } + newName := newRc.Name + if oldName == newName { + return util.UsageError(cmd, "%s cannot have the same name as the existing ReplicationController %s", + filename, oldName) + } client, err := f.Client() if err != nil { return err } - obj, err := mapping.Codec.Decode(data) - if err != nil { - return err - } - newRc := obj.(*api.ReplicationController) - - updater := kubectl.NewRollingUpdater(cmdNamespace, client) + updater := kubectl.NewRollingUpdater(newRc.Namespace, client) // fetch rc - oldRc, err := client.ReplicationControllers(cmdNamespace).Get(oldName) + oldRc, err := client.ReplicationControllers(newRc.Namespace).Get(oldName) if err != nil { return err } diff --git a/pkg/kubectl/cmd/util/resource.go b/pkg/kubectl/cmd/util/resource.go index ad39689d9fd..1ceca6279d0 100644 --- a/pkg/kubectl/cmd/util/resource.go +++ b/pkg/kubectl/cmd/util/resource.go @@ -17,13 +17,9 @@ limitations under the License. package util import ( - "fmt" - "github.com/spf13/cobra" "github.com/GoogleCloudPlatform/kubernetes/pkg/api/meta" - "github.com/GoogleCloudPlatform/kubernetes/pkg/api/validation" - "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" ) // ResourceFromArgs expects two arguments with a given type, and extracts the fields necessary @@ -52,69 +48,3 @@ func ResourceFromArgs(cmd *cobra.Command, args []string, mapper meta.RESTMapper, mapping, err = mapper.RESTMapping(kind, version) return } - -// ResourceFromFile retrieves the name and namespace from a valid file. If the file does not -// resolve to a known type an error is returned. The returned mapping can be used to determine -// the correct REST endpoint to modify this resource with. -// DEPRECATED: Use resource.Builder -func ResourceFromFile(filename string, typer runtime.ObjectTyper, mapper meta.RESTMapper, schema validation.Schema, cmdVersion string) (mapping *meta.RESTMapping, namespace, name string, data []byte, err error) { - data, err = ReadConfigData(filename) - if err != nil { - return - } - - objVersion, kind, err := typer.DataVersionAndKind(data) - if err != nil { - return - } - - // TODO: allow unversioned objects? - if len(objVersion) == 0 { - err = fmt.Errorf("the resource in the provided file has no apiVersion defined") - } - - err = schema.ValidateBytes(data) - if err != nil { - return - } - - // decode using the version stored with the object (allows codec to vary across versions) - mapping, err = mapper.RESTMapping(kind, objVersion) - if err != nil { - return - } - - obj, err := mapping.Codec.Decode(data) - if err != nil { - return - } - - meta := mapping.MetadataAccessor - namespace, err = meta.Namespace(obj) - if err != nil { - return - } - name, err = meta.Name(obj) - if err != nil { - return - } - - // if the preferred API version differs, get a different mapper - if cmdVersion != objVersion { - mapping, err = mapper.RESTMapping(kind, cmdVersion) - } - return -} - -// CompareNamespace returns an error if the namespace the user has provided on the CLI -// or via the default namespace file does not match the namespace of an input file. This -// prevents a user from unintentionally updating the wrong namespace. -// DEPRECATED: Use resource.Builder -func CompareNamespace(defaultNamespace, namespace string) error { - if len(namespace) > 0 { - if defaultNamespace != namespace { - return fmt.Errorf("the namespace from the provided file %q does not match the namespace %q. You must pass '--namespace=%s' to perform this operation.", namespace, defaultNamespace, namespace) - } - } - return nil -} diff --git a/pkg/kubectl/resource/mapper.go b/pkg/kubectl/resource/mapper.go index 874eddbe87b..bb97e08c144 100644 --- a/pkg/kubectl/resource/mapper.go +++ b/pkg/kubectl/resource/mapper.go @@ -22,6 +22,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/api/meta" "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util/yaml" ) // Mapper is a convenience struct for holding references to the three interfaces @@ -36,6 +37,11 @@ type Mapper struct { // if any of the decoding or client lookup steps fail. Name and namespace will be // set into Info if the mapping's MetadataAccessor can retrieve them. func (m *Mapper) InfoForData(data []byte, source string) (*Info, error) { + json, err := yaml.ToJSON(data) + if err != nil { + return nil, fmt.Errorf("unable to parse %q: %v", err) + } + data = json version, kind, err := m.DataVersionAndKind(data) if err != nil { return nil, fmt.Errorf("unable to get type info from %q: %v", source, err) diff --git a/pkg/kubectl/resource_printer_test.go b/pkg/kubectl/resource_printer_test.go index cee1ef07a18..ba5f5c4232c 100644 --- a/pkg/kubectl/resource_printer_test.go +++ b/pkg/kubectl/resource_printer_test.go @@ -196,7 +196,7 @@ func testPrinter(t *testing.T, printer ResourcePrinter, unmarshalFunc func(data } // Use real decode function to undo the versioning process. poutput = testStruct{} - err = testapi.Codec().DecodeInto(buf.Bytes(), &poutput) + err = runtime.YAMLDecoder(testapi.Codec()).DecodeInto(buf.Bytes(), &poutput) if err != nil { t.Fatal(err) } @@ -217,7 +217,7 @@ func testPrinter(t *testing.T, printer ResourcePrinter, unmarshalFunc func(data } // Use real decode function to undo the versioning process. objOut = api.Pod{} - err = testapi.Codec().DecodeInto(buf.Bytes(), &objOut) + err = runtime.YAMLDecoder(testapi.Codec()).DecodeInto(buf.Bytes(), &objOut) if err != nil { t.Fatal(err) } diff --git a/pkg/runtime/codec.go b/pkg/runtime/codec.go index 585df989249..640d6c3eb3d 100644 --- a/pkg/runtime/codec.go +++ b/pkg/runtime/codec.go @@ -16,11 +16,47 @@ limitations under the License. package runtime +import ( + "github.com/GoogleCloudPlatform/kubernetes/pkg/util/yaml" +) + // CodecFor returns a Codec that invokes Encode with the provided version. func CodecFor(scheme *Scheme, version string) Codec { return &codecWrapper{scheme, version} } +// yamlCodec converts YAML passed to the Decoder methods to JSON. +type yamlCodec struct { + // a Codec for JSON + Codec +} + +// yamlCodec implements Codec +var _ Codec = yamlCodec{} + +// YAMLDecoder adds YAML decoding support to a codec that supports JSON. +func YAMLDecoder(codec Codec) Codec { + return &yamlCodec{codec} +} + +func (c yamlCodec) Decode(data []byte) (Object, error) { + out, err := yaml.ToJSON(data) + if err != nil { + return nil, err + } + data = out + return c.Codec.Decode(data) +} + +func (c yamlCodec) DecodeInto(data []byte, obj Object) error { + out, err := yaml.ToJSON(data) + if err != nil { + return err + } + data = out + return c.Codec.DecodeInto(data, obj) +} + // EncodeOrDie is a version of Encode which will panic instead of returning an error. For tests. func EncodeOrDie(codec Codec, obj Object) string { bytes, err := codec.Encode(obj) diff --git a/pkg/util/yaml/decoder.go b/pkg/util/yaml/decoder.go index e6e281b1c39..8dfc56cd826 100644 --- a/pkg/util/yaml/decoder.go +++ b/pkg/util/yaml/decoder.go @@ -27,6 +27,17 @@ import ( "github.com/golang/glog" ) +// ToJSON converts a single YAML document into a JSON document +// or returns an error. If the document appears to be JSON the +// YAML decoding path is not used (so that error messages are) +// JSON specific. +func ToJSON(data []byte) ([]byte, error) { + if hasJSONPrefix(data) { + return data, nil + } + return yaml.YAMLToJSON(data) +} + // YAMLToJSONDecoder decodes YAML documents from an io.Reader by // separating individual documents. It first converts the YAML // body to JSON, then unmarshals the JSON. @@ -143,11 +154,19 @@ func (d *YAMLOrJSONDecoder) Decode(into interface{}) error { func guessJSONStream(r io.Reader, size int) (io.Reader, bool) { buffer := bufio.NewReaderSize(r, size) b, _ := buffer.Peek(size) - return buffer, hasPrefix(b, []byte("{")) + return buffer, hasJSONPrefix(b) +} + +var jsonPrefix = []byte("{") + +// hasJSONPrefix returns true if the provided buffer appears to start with +// a JSON open brace. +func hasJSONPrefix(buf []byte) bool { + return hasPrefix(buf, jsonPrefix) } // Return true if the first non-whitespace bytes in buf is -// prefix +// prefix. func hasPrefix(buf []byte, prefix []byte) bool { trim := bytes.TrimLeftFunc(buf, unicode.IsSpace) return bytes.HasPrefix(trim, prefix) diff --git a/test/integration/auth_test.go b/test/integration/auth_test.go index 916cb7be784..93a6bc8f489 100644 --- a/test/integration/auth_test.go +++ b/test/integration/auth_test.go @@ -73,7 +73,7 @@ var aPod string = ` "manifest": { "version": "v1beta1", "id": "a", - "containers": [{ "name": "foo", "image": "bar/foo", }] + "containers": [{ "name": "foo", "image": "bar/foo" }] } }%s } @@ -87,7 +87,7 @@ var aPodInBar string = ` "manifest": { "version": "v1beta1", "id": "a", - "containers": [{ "name": "foo", "image": "bar/foo", }] + "containers": [{ "name": "foo", "image": "bar/foo" }] } }%s } @@ -102,17 +102,18 @@ var aRC string = ` "replicaSelector": {"name": "a"}, "podTemplate": { "desiredState": { - "manifest": { - "version": "v1beta1", - "id": "a", - "containers": [{ - "name": "foo", - "image": "bar/foo", - }] - } - }, - "labels": {"name": "a"} - }}, + "manifest": { + "version": "v1beta1", + "id": "a", + "containers": [{ + "name": "foo", + "image": "bar/foo" + }] + } + }, + "labels": {"name": "a"} + } + }, "labels": {"name": "a"}%s } ` @@ -145,7 +146,7 @@ var aEvent string = ` "kind": "Minion", "name": "a", "namespace": "default", - "apiVersion": "v1beta1", + "apiVersion": "v1beta1" }%s } ` @@ -343,10 +344,11 @@ func TestAuthModeAlwaysAllow(t *testing.T) { sub += fmt.Sprintf(",\r\n\"resourceVersion\": %v", resVersion) } namespace := "default" - sub += fmt.Sprintf(",\r\n\"namespace\": %v", namespace) + sub += fmt.Sprintf(",\r\n\"namespace\": %q", namespace) } bodyStr = fmt.Sprintf(r.body, sub) } + r.body = bodyStr bodyBytes := bytes.NewReader([]byte(bodyStr)) req, err := http.NewRequest(r.verb, s.URL+r.URL, bodyBytes) if err != nil { @@ -514,10 +516,11 @@ func TestAliceNotForbiddenOrUnauthorized(t *testing.T) { sub += fmt.Sprintf(",\r\n\"resourceVersion\": %v", resVersion) } namespace := "default" - sub += fmt.Sprintf(",\r\n\"namespace\": %v", namespace) + sub += fmt.Sprintf(",\r\n\"namespace\": %q", namespace) } bodyStr = fmt.Sprintf(r.body, sub) } + r.body = bodyStr bodyBytes := bytes.NewReader([]byte(bodyStr)) req, err := http.NewRequest(r.verb, s.URL+r.URL, bodyBytes) if err != nil { @@ -767,10 +770,11 @@ func TestNamespaceAuthorization(t *testing.T) { if len(namespace) == 0 { namespace = "default" } - sub += fmt.Sprintf(",\r\n\"namespace\": %v", namespace) + sub += fmt.Sprintf(",\r\n\"namespace\": %q", namespace) } bodyStr = fmt.Sprintf(r.body, sub) } + r.body = bodyStr bodyBytes := bytes.NewReader([]byte(bodyStr)) req, err := http.NewRequest(r.verb, s.URL+r.URL, bodyBytes) if err != nil { @@ -873,6 +877,7 @@ func TestKindAuthorization(t *testing.T) { } } } + r.body = bodyStr bodyBytes := bytes.NewReader([]byte(bodyStr)) req, err := http.NewRequest(r.verb, s.URL+r.URL, bodyBytes) if err != nil {