From 55f28a662d1a0e702f5e2a7ba5367b21bf9b9976 Mon Sep 17 00:00:00 2001 From: liz Date: Tue, 10 Apr 2018 11:11:07 -0700 Subject: [PATCH] Adds migrations to the kubeadm upgrade phase config This fixes a previous issue with kubeadm where a backwards-incompatible struct change broke deserialising configs as part of the upgrade. --- .../kubeadm/v1alpha1/testdata/kubeadm196.yaml | 62 ++++++++ .../app/apis/kubeadm/v1alpha1/upgrade.go | 135 +++++++++++++++++ .../app/apis/kubeadm/v1alpha1/upgrade_test.go | 137 ++++++++++++++++++ .../app/phases/upgrade/configuration.go | 8 +- 4 files changed, 340 insertions(+), 2 deletions(-) create mode 100644 cmd/kubeadm/app/apis/kubeadm/v1alpha1/testdata/kubeadm196.yaml create mode 100644 cmd/kubeadm/app/apis/kubeadm/v1alpha1/upgrade.go create mode 100644 cmd/kubeadm/app/apis/kubeadm/v1alpha1/upgrade_test.go diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/testdata/kubeadm196.yaml b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/testdata/kubeadm196.yaml new file mode 100644 index 00000000000..08ee0b485a6 --- /dev/null +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/testdata/kubeadm196.yaml @@ -0,0 +1,62 @@ +api: + advertiseAddress: 172.31.93.180 + bindPort: 6443 +authorizationModes: +- Node +- RBAC +certificatesDir: /etc/kubernetes/pki +cloudProvider: aws +etcd: + caFile: "" + certFile: "" + dataDir: /var/lib/etcd + endpoints: null + image: "" + keyFile: "" +imageRepository: gcr.io/google_containers +kubeProxy: + config: + bindAddress: 0.0.0.0 + clientConnection: + acceptContentTypes: "" + burst: 10 + contentType: application/vnd.kubernetes.protobuf + kubeconfig: /var/lib/kube-proxy/kubeconfig.conf + qps: 5 + clusterCIDR: 192.168.0.0/16 + configSyncPeriod: 15m0s + conntrack: + max: null + maxPerCore: 32768 + min: 131072 + tcpCloseWaitTimeout: 1h0m0s + tcpEstablishedTimeout: 24h0m0s + enableProfiling: false + featureGates: "" + healthzBindAddress: 0.0.0.0:10256 + hostnameOverride: "" + iptables: + masqueradeAll: false + masqueradeBit: 14 + minSyncPeriod: 0s + syncPeriod: 30s + ipvs: + minSyncPeriod: 0s + scheduler: "" + syncPeriod: 30s + metricsBindAddress: 127.0.0.1:10249 + mode: "" + oomScoreAdj: -999 + portRange: "" + resourceContainer: /kube-proxy + udpTimeoutMilliseconds: 250ms +kubeletConfiguration: {} +kubernetesVersion: v1.9.6 +networking: + dnsDomain: cluster.local + podSubnet: 192.168.0.0/16 + serviceSubnet: 10.96.0.0/12 +nodeName: ip-172-31-93-180.ec2.internal +token: 8d69af.cd3e1c58f6228dfc +tokenTTL: 24h0m0s +unifiedControlPlaneImage: "" diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/upgrade.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/upgrade.go new file mode 100644 index 00000000000..212ef0d7544 --- /dev/null +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/upgrade.go @@ -0,0 +1,135 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "bytes" + "errors" + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/json-iterator/go" + "github.com/ugorji/go/codec" + yaml "gopkg.in/yaml.v2" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/kubernetes/pkg/api/legacyscheme" +) + +var json = jsoniter.ConfigCompatibleWithStandardLibrary + +type configMutationFunc func(map[string]interface{}) error + +// These migrations are a stop-gap until we get a properly-versioned configuration file for MasterConfiguration. +// https://github.com/kubernetes/kubeadm/issues/750 +var migrations = map[string][]configMutationFunc{ + "MasterConfiguration": { + proxyFeatureListToMap, + }, +} + +// Migrate takes a map representing a config file and an object to decode into. +// The map is transformed into a format suitable for encoding into the supplied object, then serialised and decoded. +func Migrate(in map[string]interface{}, obj runtime.Object) error { + kind := reflect.TypeOf(obj).Elem().Name() + migrationsForKind := migrations[kind] + + for _, m := range migrationsForKind { + err := m(in) + if err != nil { + return err + } + } + + // Use codec instead of encoding/json to handle map[interface{}]interface{} + handle := &codec.JsonHandle{} + buf := new(bytes.Buffer) + if err := codec.NewEncoder(buf, handle).Encode(in); err != nil { + return fmt.Errorf("couldn't json encode object: %v", err) + } + + return runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), buf.Bytes(), obj) +} + +func proxyFeatureListToMap(m map[string]interface{}) error { + featureGatePath := []string{"kubeProxy", "config", "featureGates"} + + // If featureGatePath is already a map, we don't need to do anything. + _, _, err := unstructured.NestedMap(m, featureGatePath...) + if err == nil { + return nil + } + + gates, _, err := unstructured.NestedString(m, featureGatePath...) + if err != nil { + return fmt.Errorf("couldn't get featureGates: %v", err) + } + + gateMap := make(map[string]interface{}) + for _, gate := range strings.Split(gates, ",") { + if gate == "" { + continue + } + parts := strings.SplitN(gate, "=", 2) + if len(parts) != 2 { + return fmt.Errorf("unparsable kubeproxy feature gate %q", gate) + } + val, err := strconv.ParseBool(parts[1]) + if err != nil { + return fmt.Errorf("unparsable kubeproxy feature gate %q: %v", gate, err) + } + gateMap[parts[0]] = val + } + + unstructured.SetNestedMap(m, gateMap, featureGatePath...) + return nil +} + +// LoadYAML is a small wrapper around go-yaml that ensures all nested structs are map[string]interface{} instead of map[interface{}]interface{}. +func LoadYAML(bytes []byte) (map[string]interface{}, error) { + var decoded map[interface{}]interface{} + if err := yaml.Unmarshal(bytes, &decoded); err != nil { + return map[string]interface{}{}, fmt.Errorf("couldn't unmarshal YAML: %v", err) + } + + converted, ok := convert(decoded).(map[string]interface{}) + if !ok { + return map[string]interface{}{}, errors.New("yaml is not a map") + } + + return converted, nil +} + +// https://stackoverflow.com/questions/40737122/convert-yaml-to-json-without-struct-golang +func convert(i interface{}) interface{} { + switch x := i.(type) { + case map[interface{}]interface{}: + m2 := map[string]interface{}{} + for k, v := range x { + m2[k.(string)] = convert(v) + } + return m2 + case []interface{}: + for i, v := range x { + x[i] = convert(v) + } + } + return i +} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/upgrade_test.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/upgrade_test.go new file mode 100644 index 00000000000..6a1fca7e1d3 --- /dev/null +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/upgrade_test.go @@ -0,0 +1,137 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "io/ioutil" + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +const test196 = "testdata/kubeadm196.yaml" + +func TestUpgrade(t *testing.T) { + testYAML, err := ioutil.ReadFile(test196) + if err != nil { + t.Fatalf("couldn't read test data: %v", err) + } + + decoded, err := LoadYAML(testYAML) + if err != nil { + t.Fatalf("couldn't unmarshal test yaml: %v", err) + } + + var obj MasterConfiguration + if err := Migrate(decoded, &obj); err != nil { + t.Fatalf("couldn't decode migrated object: %v", err) + } +} + +func TestProxyFeatureListToMap(t *testing.T) { + + cases := []struct { + name string + featureGates interface{} + expected map[string]interface{} + shouldError bool + }{ + { + name: "multiple features", + featureGates: "feature1=true,feature2=false", + expected: map[string]interface{}{ + "feature1": true, + "feature2": false, + }, + }, + { + name: "single feature", + featureGates: "feature1=true", + expected: map[string]interface{}{ + "feature1": true, + }, + }, + { + name: "already a map", + featureGates: map[string]interface{}{ + "feature1": true, + }, + expected: map[string]interface{}{ + "feature1": true, + }, + }, + { + name: "single feature", + featureGates: "", + expected: map[string]interface{}{}, + }, + { + name: "malformed string", + featureGates: "test,", + shouldError: true, + }, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + + cfg := map[string]interface{}{ + "kubeProxy": map[string]interface{}{ + "config": map[string]interface{}{ + "featureGates": testCase.featureGates, + }, + }, + } + + err := proxyFeatureListToMap(cfg) + if testCase.shouldError { + if err == nil { + t.Error("expected error, got nil") + } + return + } + + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + gates, ok, err := unstructured.NestedMap(cfg, "kubeProxy", "config", "featureGates") + if !ok { + t.Errorf("missing map keys in nested map") + } + if err != nil { + t.Errorf("unexpected error in map: %v", err) + } + + if len(testCase.expected) != len(gates) { + t.Errorf("expected feature gate size %d, got %d", len(testCase.expected), len(gates)) + } + + for k, v := range testCase.expected { + gateVal, ok := gates[k] + if !ok { + t.Errorf("featureGates missing key %q", k) + continue + } + + if v != gateVal { + t.Errorf("expected value %v, got %v", v, gateVal) + } + } + }) + } +} diff --git a/cmd/kubeadm/app/phases/upgrade/configuration.go b/cmd/kubeadm/app/phases/upgrade/configuration.go index 21218bb63af..1d3fd5797e3 100644 --- a/cmd/kubeadm/app/phases/upgrade/configuration.go +++ b/cmd/kubeadm/app/phases/upgrade/configuration.go @@ -23,7 +23,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" @@ -103,9 +102,14 @@ func bytesToValidatedMasterConfig(b []byte) (*kubeadmapiext.MasterConfiguration, finalCfg := &kubeadmapiext.MasterConfiguration{} internalcfg := &kubeadmapi.MasterConfiguration{} - if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), b, cfg); err != nil { + decoded, err := kubeadmapiext.LoadYAML(b) + if err != nil { return nil, fmt.Errorf("unable to decode config from bytes: %v", err) } + + if err := kubeadmapiext.Migrate(decoded, cfg); err != nil { + return nil, fmt.Errorf("unable to migrate config from previous version: %v", err) + } // Default and convert to the internal version legacyscheme.Scheme.Default(cfg) legacyscheme.Scheme.Convert(cfg, internalcfg, nil)