diff --git a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go index b5e2483c94e..391ea4bad0e 100644 --- a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go +++ b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go @@ -73,7 +73,7 @@ func TestMutatePodSpec(t *testing.T) { } for _, rt := range tests { - mutatePodSpec(getDefaultMutators(), rt.component, rt.podSpec) + mutatePodSpec(GetDefaultMutators(), rt.component, rt.podSpec) if !reflect.DeepEqual(*rt.podSpec, rt.expected) { t.Errorf("failed mutatePodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go index 3ac5a6e6ad7..1c8ed984860 100644 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go +++ b/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go @@ -182,7 +182,8 @@ spec: - hostPath: path: /etc/pki name: ca-certs-etc-pki - updateStrategy: {} + updateStrategy: + type: RollingUpdate status: currentNumberScheduled: 0 desiredNumberScheduled: 0 @@ -330,7 +331,8 @@ spec: - hostPath: path: /etc/pki name: ca-certs-etc-pki - updateStrategy: {} + updateStrategy: + type: RollingUpdate status: currentNumberScheduled: 0 desiredNumberScheduled: 0 @@ -430,7 +432,8 @@ spec: path: /etc/kubernetes/scheduler.conf type: FileOrCreate name: kubeconfig - updateStrategy: {} + updateStrategy: + type: RollingUpdate status: currentNumberScheduled: 0 desiredNumberScheduled: 0 @@ -471,7 +474,7 @@ func TestBuildDaemonSet(t *testing.T) { t.Fatalf("couldn't load the specified Pod") } - ds := buildDaemonSet(rt.component, podSpec, getDefaultMutators()) + ds := BuildDaemonSet(rt.component, podSpec, GetDefaultMutators()) dsBytes, err := yaml.Marshal(ds) if err != nil { t.Fatalf("failed to marshal daemonset to YAML: %v", err) diff --git a/cmd/kubeadm/app/phases/upgrade/compute_test.go b/cmd/kubeadm/app/phases/upgrade/compute_test.go new file mode 100644 index 00000000000..5466fe64e54 --- /dev/null +++ b/cmd/kubeadm/app/phases/upgrade/compute_test.go @@ -0,0 +1,481 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "reflect" + "testing" + + versionutil "k8s.io/kubernetes/pkg/util/version" +) + +type fakeVersionGetter struct { + clusterVersion, kubeadmVersion, stableVersion, latestVersion, latestDevBranchVersion, stablePatchVersion, kubeletVersion string +} + +var _ VersionGetter = &fakeVersionGetter{} + +// ClusterVersion gets a fake API server version +func (f *fakeVersionGetter) ClusterVersion() (string, *versionutil.Version, error) { + return f.clusterVersion, versionutil.MustParseSemantic(f.clusterVersion), nil +} + +// KubeadmVersion gets a fake kubeadm version +func (f *fakeVersionGetter) KubeadmVersion() (string, *versionutil.Version, error) { + return f.kubeadmVersion, versionutil.MustParseSemantic(f.kubeadmVersion), nil +} + +// VersionFromCILabel gets fake latest versions from CI +func (f *fakeVersionGetter) VersionFromCILabel(ciVersionLabel, _ string) (string, *versionutil.Version, error) { + if ciVersionLabel == "stable" { + return f.stableVersion, versionutil.MustParseSemantic(f.stableVersion), nil + } + if ciVersionLabel == "latest" { + return f.latestVersion, versionutil.MustParseSemantic(f.latestVersion), nil + } + if ciVersionLabel == "latest-1.8" { + return f.latestDevBranchVersion, versionutil.MustParseSemantic(f.latestDevBranchVersion), nil + } + return f.stablePatchVersion, versionutil.MustParseSemantic(f.stablePatchVersion), nil +} + +// KubeletVersions gets the versions of the kubelets in the cluster +func (f *fakeVersionGetter) KubeletVersions() (map[string]uint16, error) { + return map[string]uint16{ + f.kubeletVersion: 1, + }, nil +} + +func TestGetAvailableUpgrades(t *testing.T) { + tests := []struct { + vg *fakeVersionGetter + expectedUpgrades []Upgrade + allowExperimental, allowRCs bool + errExpected bool + }{ + { // no action needed, already up-to-date + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.3", + kubeletVersion: "v1.7.3", + kubeadmVersion: "v1.7.3", + + stablePatchVersion: "v1.7.3", + stableVersion: "v1.7.3", + }, + expectedUpgrades: []Upgrade{}, + allowExperimental: false, + errExpected: false, + }, + { // simple patch version upgrade + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.1", + kubeletVersion: "v1.7.1", // the kubelet are on the same version as the control plane + kubeadmVersion: "v1.7.2", + + stablePatchVersion: "v1.7.3", + stableVersion: "v1.7.3", + }, + expectedUpgrades: []Upgrade{ + { + Description: "version in the v1.7 series", + Before: ClusterState{ + KubeVersion: "v1.7.1", + KubeletVersions: map[string]uint16{ + "v1.7.1": 1, + }, + KubeadmVersion: "v1.7.2", + DNSVersion: "1.14.4", + }, + After: ClusterState{ + KubeVersion: "v1.7.3", + KubeadmVersion: "v1.7.3", + DNSVersion: "1.14.4", + }, + }, + }, + allowExperimental: false, + errExpected: false, + }, + { // minor version upgrade only + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.3", + kubeletVersion: "v1.7.3", // the kubelet are on the same version as the control plane + kubeadmVersion: "v1.8.0", + + stablePatchVersion: "v1.7.3", + stableVersion: "v1.8.0", + }, + expectedUpgrades: []Upgrade{ + { + Description: "stable version", + Before: ClusterState{ + KubeVersion: "v1.7.3", + KubeletVersions: map[string]uint16{ + "v1.7.3": 1, + }, + KubeadmVersion: "v1.8.0", + DNSVersion: "1.14.4", + }, + After: ClusterState{ + KubeVersion: "v1.8.0", + KubeadmVersion: "v1.8.0", + DNSVersion: "1.14.4", + }, + }, + }, + allowExperimental: false, + errExpected: false, + }, + { // both minor version upgrade and patch version upgrade available + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.3", + kubeletVersion: "v1.7.3", // the kubelet are on the same version as the control plane + kubeadmVersion: "v1.8.1", + + stablePatchVersion: "v1.7.5", + stableVersion: "v1.8.2", + }, + expectedUpgrades: []Upgrade{ + { + Description: "version in the v1.7 series", + Before: ClusterState{ + KubeVersion: "v1.7.3", + KubeletVersions: map[string]uint16{ + "v1.7.3": 1, + }, + KubeadmVersion: "v1.8.1", + DNSVersion: "1.14.4", + }, + After: ClusterState{ + KubeVersion: "v1.7.5", + KubeadmVersion: "v1.8.1", // Note: The kubeadm version mustn't be "downgraded" here + DNSVersion: "1.14.4", + }, + }, + { + Description: "stable version", + Before: ClusterState{ + KubeVersion: "v1.7.3", + KubeletVersions: map[string]uint16{ + "v1.7.3": 1, + }, + KubeadmVersion: "v1.8.1", + DNSVersion: "1.14.4", + }, + After: ClusterState{ + KubeVersion: "v1.8.2", + KubeadmVersion: "v1.8.2", + DNSVersion: "1.14.4", + }, + }, + }, + allowExperimental: false, + errExpected: false, + }, + { // allow experimental upgrades, but no upgrade available + vg: &fakeVersionGetter{ + clusterVersion: "v1.8.0-alpha.2", + kubeletVersion: "v1.7.5", + kubeadmVersion: "v1.7.5", + + stablePatchVersion: "v1.7.5", + stableVersion: "v1.7.5", + latestVersion: "v1.8.0-alpha.2", + }, + expectedUpgrades: []Upgrade{}, + allowExperimental: true, + errExpected: false, + }, + { // upgrade to an unstable version should be supported + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.5", + kubeletVersion: "v1.7.5", + kubeadmVersion: "v1.7.5", + + stablePatchVersion: "v1.7.5", + stableVersion: "v1.7.5", + latestVersion: "v1.8.0-alpha.2", + }, + expectedUpgrades: []Upgrade{ + { + Description: "experimental version", + Before: ClusterState{ + KubeVersion: "v1.7.5", + KubeletVersions: map[string]uint16{ + "v1.7.5": 1, + }, + KubeadmVersion: "v1.7.5", + DNSVersion: "1.14.4", + }, + After: ClusterState{ + KubeVersion: "v1.8.0-alpha.2", + KubeadmVersion: "v1.8.0-alpha.2", + DNSVersion: "1.14.4", + }, + }, + }, + allowExperimental: true, + errExpected: false, + }, + { // upgrade from an unstable version to an unstable version should be supported + vg: &fakeVersionGetter{ + clusterVersion: "v1.8.0-alpha.1", + kubeletVersion: "v1.7.5", + kubeadmVersion: "v1.7.5", + + stablePatchVersion: "v1.7.5", + stableVersion: "v1.7.5", + latestVersion: "v1.8.0-alpha.2", + }, + expectedUpgrades: []Upgrade{ + { + Description: "experimental version", + Before: ClusterState{ + KubeVersion: "v1.8.0-alpha.1", + KubeletVersions: map[string]uint16{ + "v1.7.5": 1, + }, + KubeadmVersion: "v1.7.5", + DNSVersion: "1.14.4", + }, + After: ClusterState{ + KubeVersion: "v1.8.0-alpha.2", + KubeadmVersion: "v1.8.0-alpha.2", + DNSVersion: "1.14.4", + }, + }, + }, + allowExperimental: true, + errExpected: false, + }, + { // v1.X.0-alpha.0 should be ignored + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.5", + kubeletVersion: "v1.7.5", + kubeadmVersion: "v1.7.5", + + stablePatchVersion: "v1.7.5", + stableVersion: "v1.7.5", + latestDevBranchVersion: "v1.8.0-beta.1", + latestVersion: "v1.9.0-alpha.0", + }, + expectedUpgrades: []Upgrade{ + { + Description: "experimental version", + Before: ClusterState{ + KubeVersion: "v1.7.5", + KubeletVersions: map[string]uint16{ + "v1.7.5": 1, + }, + KubeadmVersion: "v1.7.5", + DNSVersion: "1.14.4", + }, + After: ClusterState{ + KubeVersion: "v1.8.0-beta.1", + KubeadmVersion: "v1.8.0-beta.1", + DNSVersion: "1.14.4", + }, + }, + }, + allowExperimental: true, + errExpected: false, + }, + { // upgrade to an RC version should be supported + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.5", + kubeletVersion: "v1.7.5", + kubeadmVersion: "v1.7.5", + + stablePatchVersion: "v1.7.5", + stableVersion: "v1.7.5", + latestDevBranchVersion: "v1.8.0-rc.1", + latestVersion: "v1.9.0-alpha.1", + }, + expectedUpgrades: []Upgrade{ + { + Description: "release candidate version", + Before: ClusterState{ + KubeVersion: "v1.7.5", + KubeletVersions: map[string]uint16{ + "v1.7.5": 1, + }, + KubeadmVersion: "v1.7.5", + DNSVersion: "1.14.4", + }, + After: ClusterState{ + KubeVersion: "v1.8.0-rc.1", + KubeadmVersion: "v1.8.0-rc.1", + DNSVersion: "1.14.4", + }, + }, + }, + allowRCs: true, + errExpected: false, + }, + { // it is possible (but very uncommon) that the latest version from the previous branch is an rc and the current latest version is alpha.0. In that case, show the RC + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.5", + kubeletVersion: "v1.7.5", + kubeadmVersion: "v1.7.5", + + stablePatchVersion: "v1.7.5", + stableVersion: "v1.7.5", + latestDevBranchVersion: "v1.8.0-rc.1", + latestVersion: "v1.9.0-alpha.0", + }, + expectedUpgrades: []Upgrade{ + { + Description: "experimental version", // Note that this is considered an experimental version in this uncommon scenario + Before: ClusterState{ + KubeVersion: "v1.7.5", + KubeletVersions: map[string]uint16{ + "v1.7.5": 1, + }, + KubeadmVersion: "v1.7.5", + DNSVersion: "1.14.4", + }, + After: ClusterState{ + KubeVersion: "v1.8.0-rc.1", + KubeadmVersion: "v1.8.0-rc.1", + DNSVersion: "1.14.4", + }, + }, + }, + allowExperimental: true, + errExpected: false, + }, + { // upgrade to an RC version should be supported. There may also be an even newer unstable version. + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.5", + kubeletVersion: "v1.7.5", + kubeadmVersion: "v1.7.5", + + stablePatchVersion: "v1.7.5", + stableVersion: "v1.7.5", + latestDevBranchVersion: "v1.8.0-rc.1", + latestVersion: "v1.9.0-alpha.1", + }, + expectedUpgrades: []Upgrade{ + { + Description: "release candidate version", + Before: ClusterState{ + KubeVersion: "v1.7.5", + KubeletVersions: map[string]uint16{ + "v1.7.5": 1, + }, + KubeadmVersion: "v1.7.5", + DNSVersion: "1.14.4", + }, + After: ClusterState{ + KubeVersion: "v1.8.0-rc.1", + KubeadmVersion: "v1.8.0-rc.1", + DNSVersion: "1.14.4", + }, + }, + { + Description: "experimental version", + Before: ClusterState{ + KubeVersion: "v1.7.5", + KubeletVersions: map[string]uint16{ + "v1.7.5": 1, + }, + KubeadmVersion: "v1.7.5", + DNSVersion: "1.14.4", + }, + After: ClusterState{ + KubeVersion: "v1.9.0-alpha.1", + KubeadmVersion: "v1.9.0-alpha.1", + DNSVersion: "1.14.4", + }, + }, + }, + allowRCs: true, + allowExperimental: true, + errExpected: false, + }, + } + + for _, rt := range tests { + + actualUpgrades, actualErr := GetAvailableUpgrades(rt.vg, rt.allowExperimental, rt.allowRCs) + if !reflect.DeepEqual(actualUpgrades, rt.expectedUpgrades) { + t.Errorf("failed TestGetAvailableUpgrades\n\texpected upgrades: %v\n\tgot: %v", rt.expectedUpgrades, actualUpgrades) + } + if (actualErr != nil) != rt.errExpected { + t.Errorf("failed TestGetAvailableUpgrades\n\texpected error: %t\n\tgot error: %t", rt.errExpected, (actualErr != nil)) + } + } +} + +func TestKubeletUpgrade(t *testing.T) { + tests := []struct { + before map[string]uint16 + after string + expected bool + }{ + { // upgrade available + before: map[string]uint16{ + "v1.7.1": 1, + }, + after: "v1.7.3", + expected: true, + }, + { // upgrade available + before: map[string]uint16{ + "v1.7.1": 1, + "v1.7.3": 100, + }, + after: "v1.7.3", + expected: true, + }, + { // upgrade not available + before: map[string]uint16{ + "v1.7.3": 1, + }, + after: "v1.7.3", + expected: false, + }, + { // upgrade not available + before: map[string]uint16{ + "v1.7.3": 100, + }, + after: "v1.7.3", + expected: false, + }, + { // upgrade not available if we don't know anything about the earlier state + before: map[string]uint16{}, + after: "v1.7.3", + expected: false, + }, + } + + for _, rt := range tests { + + upgrade := Upgrade{ + Before: ClusterState{ + KubeletVersions: rt.before, + }, + After: ClusterState{ + KubeVersion: rt.after, + }, + } + actual := upgrade.CanUpgradeKubelets() + if actual != rt.expected { + t.Errorf("failed TestKubeletUpgrade\n\texpected: %t\n\tgot: %t\n\ttest object: %v", rt.expected, actual, upgrade) + } + } +} diff --git a/cmd/kubeadm/app/phases/upgrade/policy_test.go b/cmd/kubeadm/app/phases/upgrade/policy_test.go new file mode 100644 index 00000000000..659a5fe990a --- /dev/null +++ b/cmd/kubeadm/app/phases/upgrade/policy_test.go @@ -0,0 +1,186 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "testing" + + "k8s.io/kubernetes/pkg/util/version" +) + +func TestEnforceVersionPolicies(t *testing.T) { + tests := []struct { + vg *fakeVersionGetter + expectedMandatoryErrs int + expectedSkippableErrs int + allowExperimental, allowRCs bool + newK8sVersion string + }{ + { // everything ok + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.3", + kubeletVersion: "v1.7.3", + kubeadmVersion: "v1.7.5", + }, + newK8sVersion: "v1.7.5", + }, + { // everything ok + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.3", + kubeletVersion: "v1.7.2", + kubeadmVersion: "v1.8.1", + }, + newK8sVersion: "v1.8.0", + }, + { // downgrades not supported + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.3", + kubeletVersion: "v1.7.3", + kubeadmVersion: "v1.7.3", + }, + newK8sVersion: "v1.7.2", + expectedSkippableErrs: 1, + }, + { // upgrades without bumping the version number not supported yet. TODO: Change this? + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.3", + kubeletVersion: "v1.7.3", + kubeadmVersion: "v1.7.3", + }, + newK8sVersion: "v1.7.3", + expectedSkippableErrs: 1, + }, + { // new version must be higher than v1.7.0 + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.3", + kubeletVersion: "v1.7.3", + kubeadmVersion: "v1.7.3", + }, + newK8sVersion: "v1.6.10", + expectedMandatoryErrs: 1, // version must be higher than v1.7.0 + expectedSkippableErrs: 1, // version shouldn't be downgraded + }, + { // upgrading two minor versions in one go is not supported + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.3", + kubeletVersion: "v1.7.3", + kubeadmVersion: "v1.9.0", + }, + newK8sVersion: "v1.9.0", + expectedMandatoryErrs: 1, // can't upgrade two minor versions + expectedSkippableErrs: 1, // kubelet <-> apiserver skew too large + }, + { // kubeadm version must be higher than the new kube version. However, patch version skews may be forced + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.3", + kubeletVersion: "v1.7.3", + kubeadmVersion: "v1.7.3", + }, + newK8sVersion: "v1.7.5", + expectedSkippableErrs: 1, + }, + { // kubeadm version must be higher than the new kube version. Trying to upgrade k8s to a higher minor version than kubeadm itself should never be supported + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.3", + kubeletVersion: "v1.7.3", + kubeadmVersion: "v1.7.3", + }, + newK8sVersion: "v1.8.0", + expectedMandatoryErrs: 1, + }, + { // the maximum skew between the cluster version and the kubelet versions should be one minor version. This may be forced through though. + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.3", + kubeletVersion: "v1.6.8", + kubeadmVersion: "v1.8.0", + }, + newK8sVersion: "v1.8.0", + expectedSkippableErrs: 1, + }, + { // experimental upgrades supported if the flag is set + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.3", + kubeletVersion: "v1.7.3", + kubeadmVersion: "v1.8.0-beta.1", + }, + newK8sVersion: "v1.8.0-beta.1", + allowExperimental: true, + }, + { // release candidate upgrades supported if the flag is set + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.3", + kubeletVersion: "v1.7.3", + kubeadmVersion: "v1.8.0-rc.1", + }, + newK8sVersion: "v1.8.0-rc.1", + allowRCs: true, + }, + { // release candidate upgrades supported if the flag is set + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.3", + kubeletVersion: "v1.7.3", + kubeadmVersion: "v1.8.0-rc.1", + }, + newK8sVersion: "v1.8.0-rc.1", + allowExperimental: true, + }, + { // the user should not be able to upgrade to an experimental version if they haven't opted into that + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.3", + kubeletVersion: "v1.7.3", + kubeadmVersion: "v1.8.0-beta.1", + }, + newK8sVersion: "v1.8.0-beta.1", + allowRCs: true, + expectedSkippableErrs: 1, + }, + { // the user should not be able to upgrade to an release candidate version if they haven't opted into that + vg: &fakeVersionGetter{ + clusterVersion: "v1.7.3", + kubeletVersion: "v1.7.3", + kubeadmVersion: "v1.8.0-rc.1", + }, + newK8sVersion: "v1.8.0-rc.1", + expectedSkippableErrs: 1, + }, + } + + for _, rt := range tests { + + newK8sVer, err := version.ParseSemantic(rt.newK8sVersion) + if err != nil { + t.Fatalf("couldn't parse version %s: %v", rt.newK8sVersion, err) + } + + actualSkewErrs := EnforceVersionPolicies(rt.vg, rt.newK8sVersion, newK8sVer, rt.allowExperimental, rt.allowRCs) + if actualSkewErrs == nil { + // No errors were seen. Report unit test failure if we expected to see errors + if rt.expectedMandatoryErrs+rt.expectedSkippableErrs > 0 { + t.Errorf("failed TestEnforceVersionPolicies\n\texpected errors but got none") + } + // Otherwise, just move on with the next test + continue + } + + if len(actualSkewErrs.Skippable) != rt.expectedSkippableErrs { + t.Errorf("failed TestEnforceVersionPolicies\n\texpected skippable errors: %d\n\tgot skippable errors: %d %v", rt.expectedSkippableErrs, len(actualSkewErrs.Skippable), *rt.vg) + } + if len(actualSkewErrs.Mandatory) != rt.expectedMandatoryErrs { + t.Errorf("failed TestEnforceVersionPolicies\n\texpected mandatory errors: %d\n\tgot mandatory errors: %d %v", rt.expectedMandatoryErrs, len(actualSkewErrs.Mandatory), *rt.vg) + } + } +} diff --git a/cmd/kubeadm/app/phases/upgrade/prepull_test.go b/cmd/kubeadm/app/phases/upgrade/prepull_test.go new file mode 100644 index 00000000000..2789d8e644c --- /dev/null +++ b/cmd/kubeadm/app/phases/upgrade/prepull_test.go @@ -0,0 +1,145 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "fmt" + "testing" + "time" + //"k8s.io/kubernetes/pkg/util/version" +) + +// failedCreatePrepuller is a fake prepuller that errors for kube-controller-manager in the CreateFunc call +type failedCreatePrepuller struct{} + +func NewFailedCreatePrepuller() Prepuller { + return &failedCreatePrepuller{} +} + +func (p *failedCreatePrepuller) CreateFunc(component string) error { + if component == "kube-controller-manager" { + return fmt.Errorf("boo") + } + return nil +} + +func (p *failedCreatePrepuller) WaitFunc(component string) {} + +func (p *failedCreatePrepuller) DeleteFunc(component string) error { + return nil +} + +// foreverWaitPrepuller is a fake prepuller that basically waits "forever" (10 mins, but longer than the 10sec timeout) +type foreverWaitPrepuller struct{} + +func NewForeverWaitPrepuller() Prepuller { + return &foreverWaitPrepuller{} +} + +func (p *foreverWaitPrepuller) CreateFunc(component string) error { + return nil +} + +func (p *foreverWaitPrepuller) WaitFunc(component string) { + time.Sleep(10 * time.Minute) +} + +func (p *foreverWaitPrepuller) DeleteFunc(component string) error { + return nil +} + +// failedDeletePrepuller is a fake prepuller that errors for kube-scheduler in the DeleteFunc call +type failedDeletePrepuller struct{} + +func NewFailedDeletePrepuller() Prepuller { + return &failedDeletePrepuller{} +} + +func (p *failedDeletePrepuller) CreateFunc(component string) error { + return nil +} + +func (p *failedDeletePrepuller) WaitFunc(component string) {} + +func (p *failedDeletePrepuller) DeleteFunc(component string) error { + if component == "kube-scheduler" { + return fmt.Errorf("boo") + } + return nil +} + +// goodPrepuller is a fake prepuller that works as expected +type goodPrepuller struct{} + +func NewGoodPrepuller() Prepuller { + return &goodPrepuller{} +} + +func (p *goodPrepuller) CreateFunc(component string) error { + time.Sleep(300 * time.Millisecond) + return nil +} + +func (p *goodPrepuller) WaitFunc(component string) { + time.Sleep(300 * time.Millisecond) +} + +func (p *goodPrepuller) DeleteFunc(component string) error { + time.Sleep(300 * time.Millisecond) + return nil +} + +func TestPrepullImagesInParallel(t *testing.T) { + tests := []struct { + p Prepuller + timeout time.Duration + expectedErr bool + }{ + { // should error out; create failed + p: NewFailedCreatePrepuller(), + timeout: 10 * time.Second, + expectedErr: true, + }, + { // should error out; timeout exceeded + p: NewForeverWaitPrepuller(), + timeout: 10 * time.Second, + expectedErr: true, + }, + { // should error out; delete failed + p: NewFailedDeletePrepuller(), + timeout: 10 * time.Second, + expectedErr: true, + }, + { // should work just fine + p: NewGoodPrepuller(), + timeout: 10 * time.Second, + expectedErr: false, + }, + } + + for _, rt := range tests { + + actualErr := PrepullImagesInParallel(rt.p, rt.timeout) + if (actualErr != nil) != rt.expectedErr { + t.Errorf( + "failed TestPrepullImagesInParallel\n\texpected error: %t\n\tgot: %t", + rt.expectedErr, + (actualErr != nil), + ) + } + } +} diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go new file mode 100644 index 00000000000..301fc20a54b --- /dev/null +++ b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go @@ -0,0 +1,352 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "crypto/sha256" + "fmt" + "io/ioutil" + "os" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/runtime" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + "k8s.io/kubernetes/pkg/api" +) + +const ( + waitForHashes = "wait-for-hashes" + waitForHashChange = "wait-for-hash-change" + waitForPodsWithLabel = "wait-for-pods-with-label" + + testConfiguration = ` +api: + advertiseAddress: 1.2.3.4 + bindPort: 6443 +apiServerCertSANs: null +apiServerExtraArgs: null +authorizationModes: +- Node +- RBAC +certificatesDir: /etc/kubernetes/pki +cloudProvider: "" +controllerManagerExtraArgs: null +etcd: + caFile: "" + certFile: "" + dataDir: /var/lib/etcd + endpoints: null + extraArgs: null + image: "" + keyFile: "" +featureFlags: null +imageRepository: gcr.io/google_containers +kubernetesVersion: %s +networking: + dnsDomain: cluster.local + podSubnet: "" + serviceSubnet: 10.96.0.0/12 +nodeName: thegopher +schedulerExtraArgs: null +token: ce3aa5.5ec8455bb76b379f +tokenTTL: 86400000000000 +unifiedControlPlaneImage: "" +` +) + +// fakeWaiter is a fake apiclient.Waiter that returns errors it was initialized with +type fakeWaiter struct { + errsToReturn map[string]error +} + +func NewFakeStaticPodWaiter(errsToReturn map[string]error) apiclient.Waiter { + return &fakeWaiter{ + errsToReturn: errsToReturn, + } +} + +// WaitForAPI just returns a dummy nil, to indicate that the program should just proceed +func (w *fakeWaiter) WaitForAPI() error { + return nil +} + +// WaitForPodsWithLabel just returns an error if set from errsToReturn +func (w *fakeWaiter) WaitForPodsWithLabel(kvLabel string) error { + return w.errsToReturn[waitForPodsWithLabel] +} + +// WaitForPodToDisappear just returns a dummy nil, to indicate that the program should just proceed +func (w *fakeWaiter) WaitForPodToDisappear(podName string) error { + return nil +} + +// SetTimeout is a no-op; we don't use it in this implementation +func (w *fakeWaiter) SetTimeout(_ time.Duration) {} + +// WaitForStaticPodControlPlaneHashes returns an error if set from errsToReturn +func (w *fakeWaiter) WaitForStaticPodControlPlaneHashes(_ string) (map[string]string, error) { + return map[string]string{}, w.errsToReturn[waitForHashes] +} + +// WaitForStaticPodControlPlaneHashChange returns an error if set from errsToReturn +func (w *fakeWaiter) WaitForStaticPodControlPlaneHashChange(_, _, _ string) error { + return w.errsToReturn[waitForHashChange] +} + +type fakeStaticPodPathManager struct { + realManifestDir string + tempManifestDir string + backupManifestDir string + MoveFileFunc func(string, string) error +} + +func NewFakeStaticPodPathManager(moveFileFunc func(string, string) error) (StaticPodPathManager, error) { + realManifestsDir, err := ioutil.TempDir("", "kubeadm-upgraded-manifests") + if err != nil { + return nil, fmt.Errorf("couldn't create a temporary directory for the upgrade: %v", err) + } + + upgradedManifestsDir, err := ioutil.TempDir("", "kubeadm-upgraded-manifests") + if err != nil { + return nil, fmt.Errorf("couldn't create a temporary directory for the upgrade: %v", err) + } + + backupManifestsDir, err := ioutil.TempDir("", "kubeadm-backup-manifests") + if err != nil { + return nil, fmt.Errorf("couldn't create a temporary directory for the upgrade: %v", err) + } + + return &fakeStaticPodPathManager{ + realManifestDir: realManifestsDir, + tempManifestDir: upgradedManifestsDir, + backupManifestDir: backupManifestsDir, + MoveFileFunc: moveFileFunc, + }, nil +} + +func (spm *fakeStaticPodPathManager) MoveFile(oldPath, newPath string) error { + return spm.MoveFileFunc(oldPath, newPath) +} + +func (spm *fakeStaticPodPathManager) RealManifestPath(component string) string { + return constants.GetStaticPodFilepath(component, spm.realManifestDir) +} +func (spm *fakeStaticPodPathManager) RealManifestDir() string { + return spm.realManifestDir +} + +func (spm *fakeStaticPodPathManager) TempManifestPath(component string) string { + return constants.GetStaticPodFilepath(component, spm.tempManifestDir) +} +func (spm *fakeStaticPodPathManager) TempManifestDir() string { + return spm.tempManifestDir +} + +func (spm *fakeStaticPodPathManager) BackupManifestPath(component string) string { + return constants.GetStaticPodFilepath(component, spm.backupManifestDir) +} +func (spm *fakeStaticPodPathManager) BackupManifestDir() string { + return spm.backupManifestDir +} + +func TestStaticPodControlPlane(t *testing.T) { + tests := []struct { + waitErrsToReturn map[string]error + moveFileFunc func(string, string) error + expectedErr bool + manifestShouldChange bool + }{ + { // error-free case should succeed + waitErrsToReturn: map[string]error{ + waitForHashes: nil, + waitForHashChange: nil, + waitForPodsWithLabel: nil, + }, + moveFileFunc: func(oldPath, newPath string) error { + return os.Rename(oldPath, newPath) + }, + expectedErr: false, + manifestShouldChange: true, + }, + { // any wait error should result in a rollback and an abort + waitErrsToReturn: map[string]error{ + waitForHashes: fmt.Errorf("boo! failed"), + waitForHashChange: nil, + waitForPodsWithLabel: nil, + }, + moveFileFunc: func(oldPath, newPath string) error { + return os.Rename(oldPath, newPath) + }, + expectedErr: true, + manifestShouldChange: false, + }, + { // any wait error should result in a rollback and an abort + waitErrsToReturn: map[string]error{ + waitForHashes: nil, + waitForHashChange: fmt.Errorf("boo! failed"), + waitForPodsWithLabel: nil, + }, + moveFileFunc: func(oldPath, newPath string) error { + return os.Rename(oldPath, newPath) + }, + expectedErr: true, + manifestShouldChange: false, + }, + { // any wait error should result in a rollback and an abort + waitErrsToReturn: map[string]error{ + waitForHashes: nil, + waitForHashChange: nil, + waitForPodsWithLabel: fmt.Errorf("boo! failed"), + }, + moveFileFunc: func(oldPath, newPath string) error { + return os.Rename(oldPath, newPath) + }, + expectedErr: true, + manifestShouldChange: false, + }, + { // any path-moving error should result in a rollback and an abort + waitErrsToReturn: map[string]error{ + waitForHashes: nil, + waitForHashChange: nil, + waitForPodsWithLabel: nil, + }, + moveFileFunc: func(oldPath, newPath string) error { + // fail for kube-apiserver move + if strings.Contains(newPath, "kube-apiserver") { + return fmt.Errorf("moving the kube-apiserver file failed") + } + return os.Rename(oldPath, newPath) + }, + expectedErr: true, + manifestShouldChange: false, + }, + { // any path-moving error should result in a rollback and an abort + waitErrsToReturn: map[string]error{ + waitForHashes: nil, + waitForHashChange: nil, + waitForPodsWithLabel: nil, + }, + moveFileFunc: func(oldPath, newPath string) error { + // fail for kube-controller-manager move + if strings.Contains(newPath, "kube-controller-manager") { + return fmt.Errorf("moving the kube-apiserver file failed") + } + return os.Rename(oldPath, newPath) + }, + expectedErr: true, + manifestShouldChange: false, + }, + { // any path-moving error should result in a rollback and an abort; even though this is the last component (kube-apiserver and kube-controller-manager healthy) + waitErrsToReturn: map[string]error{ + waitForHashes: nil, + waitForHashChange: nil, + waitForPodsWithLabel: nil, + }, + moveFileFunc: func(oldPath, newPath string) error { + // fail for kube-scheduler move + if strings.Contains(newPath, "kube-scheduler") { + return fmt.Errorf("moving the kube-apiserver file failed") + } + return os.Rename(oldPath, newPath) + }, + expectedErr: true, + manifestShouldChange: false, + }, + } + + for _, rt := range tests { + + waiter := NewFakeStaticPodWaiter(rt.waitErrsToReturn) + pathMgr, err := NewFakeStaticPodPathManager(rt.moveFileFunc) + if err != nil { + t.Fatalf("couldn't run NewFakeStaticPodPathManager: %v", err) + } + defer os.RemoveAll(pathMgr.RealManifestDir()) + defer os.RemoveAll(pathMgr.TempManifestDir()) + defer os.RemoveAll(pathMgr.BackupManifestDir()) + + oldcfg, err := getConfig("v1.7.0") + if err != nil { + t.Fatalf("couldn't create config: %v", err) + } + // Initialize the directory with v1.7 manifests; should then be upgraded to v1.8 using the method + err = controlplane.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), oldcfg) + if err != nil { + t.Fatalf("couldn't run CreateInitStaticPodManifestFiles: %v", err) + } + // Get a hash of the v1.7 API server manifest to compare later (was the file re-written) + oldHash, err := getAPIServerHash(pathMgr.RealManifestDir()) + if err != nil { + t.Fatalf("couldn't read temp file: %v", err) + } + + newcfg, err := getConfig("v1.8.0") + if err != nil { + t.Fatalf("couldn't create config: %v", err) + } + + actualErr := StaticPodControlPlane(waiter, pathMgr, newcfg) + if (actualErr != nil) != rt.expectedErr { + t.Errorf( + "failed UpgradeStaticPodControlPlane\n\texpected error: %t\n\tgot: %t", + rt.expectedErr, + (actualErr != nil), + ) + } + + newHash, err := getAPIServerHash(pathMgr.RealManifestDir()) + if err != nil { + t.Fatalf("couldn't read temp file: %v", err) + } + + if (oldHash != newHash) != rt.manifestShouldChange { + t.Errorf( + "failed StaticPodControlPlane\n\texpected manifest change: %t\n\tgot: %t", + rt.manifestShouldChange, + (oldHash != newHash), + ) + } + + } +} + +func getAPIServerHash(dir string) (string, error) { + manifestPath := constants.GetStaticPodFilepath(constants.KubeAPIServer, dir) + + fileBytes, err := ioutil.ReadFile(manifestPath) + if err != nil { + return "", err + } + + return fmt.Sprintf("%x", sha256.Sum256(fileBytes)), nil +} + +func getConfig(version string) (*kubeadmapi.MasterConfiguration, error) { + externalcfg := &kubeadmapiext.MasterConfiguration{} + internalcfg := &kubeadmapi.MasterConfiguration{} + if err := runtime.DecodeInto(api.Codecs.UniversalDecoder(), []byte(fmt.Sprintf(testConfiguration, version)), externalcfg); err != nil { + return nil, fmt.Errorf("unable to decode config: %v", err) + } + api.Scheme.Convert(externalcfg, internalcfg, nil) + return internalcfg, nil +}