diff --git a/cmd/kubeadm/app/cmd/phases/init/controlplane.go b/cmd/kubeadm/app/cmd/phases/init/controlplane.go index 39348a0cb0a..cdcbe11f9e5 100644 --- a/cmd/kubeadm/app/cmd/phases/init/controlplane.go +++ b/cmd/kubeadm/app/cmd/phases/init/controlplane.go @@ -144,6 +144,10 @@ func runControlPlaneSubphase(component string) func(c workflow.RunData) error { cfg := data.Cfg() fmt.Printf("[control-plane] Creating static Pod manifest for %q\n", component) - return controlplane.CreateStaticPodFiles(data.ManifestDir(), &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component) + + // TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580 + kustomizeDir := "" + + return controlplane.CreateStaticPodFiles(data.ManifestDir(), kustomizeDir, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component) } } diff --git a/cmd/kubeadm/app/cmd/phases/init/etcd.go b/cmd/kubeadm/app/cmd/phases/init/etcd.go index c7c2b963a90..d77b581d7d4 100644 --- a/cmd/kubeadm/app/cmd/phases/init/etcd.go +++ b/cmd/kubeadm/app/cmd/phases/init/etcd.go @@ -92,7 +92,11 @@ func runEtcdPhaseLocal() func(c workflow.RunData) error { fmt.Printf("[dryrun] Would ensure that %q directory is present\n", cfg.Etcd.Local.DataDir) } fmt.Printf("[etcd] Creating static Pod manifest for local etcd in %q\n", data.ManifestDir()) - if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(data.ManifestDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil { + + // TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580 + kustomizeDir := "" + + if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(data.ManifestDir(), kustomizeDir, cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil { return errors.Wrap(err, "error creating local etcd static pod manifest file") } } else { diff --git a/cmd/kubeadm/app/cmd/phases/join/controlplaneprepare.go b/cmd/kubeadm/app/cmd/phases/join/controlplaneprepare.go index f2464f9f834..0be0df52a57 100644 --- a/cmd/kubeadm/app/cmd/phases/join/controlplaneprepare.go +++ b/cmd/kubeadm/app/cmd/phases/join/controlplaneprepare.go @@ -184,10 +184,14 @@ func runControlPlanePrepareControlPlaneSubphase(c workflow.RunData) error { fmt.Printf("[control-plane] Using manifest folder %q\n", kubeadmconstants.GetStaticPodDirectory()) + // TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580 + kustomizeDir := "" + for _, component := range kubeadmconstants.ControlPlaneComponents { fmt.Printf("[control-plane] Creating static Pod manifest for %q\n", component) err := controlplane.CreateStaticPodFiles( kubeadmconstants.GetStaticPodDirectory(), + kustomizeDir, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component, diff --git a/cmd/kubeadm/app/phases/controlplane/BUILD b/cmd/kubeadm/app/phases/controlplane/BUILD index 9763d90f066..8a68cfdd5fa 100644 --- a/cmd/kubeadm/app/phases/controlplane/BUILD +++ b/cmd/kubeadm/app/phases/controlplane/BUILD @@ -17,9 +17,11 @@ go_test( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/phases/certs:go_default_library", + "//cmd/kubeadm/app/util/staticpod:go_default_library", "//cmd/kubeadm/test:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/github.com/lithammer/dedent:go_default_library", ], ) diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 354ff074d3d..e48b64db8d9 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -39,9 +39,9 @@ import ( ) // CreateInitStaticPodManifestFiles will write all static pod manifest files needed to bring up the control plane. -func CreateInitStaticPodManifestFiles(manifestDir string, cfg *kubeadmapi.InitConfiguration) error { +func CreateInitStaticPodManifestFiles(manifestDir, kustomizeDir string, cfg *kubeadmapi.InitConfiguration) error { klog.V(1).Infoln("[control-plane] creating static Pod files") - return CreateStaticPodFiles(manifestDir, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler) + return CreateStaticPodFiles(manifestDir, kustomizeDir, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler) } // GetStaticPodSpecs returns all staticPodSpecs actualized to the context of the current configuration @@ -103,7 +103,7 @@ func livenessProbe(host string, port int, scheme v1.URIScheme) *v1.Probe { } // CreateStaticPodFiles creates all the requested static pod files. -func CreateStaticPodFiles(manifestDir string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, componentNames ...string) error { +func CreateStaticPodFiles(manifestDir, kustomizeDir string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, componentNames ...string) error { // gets the StaticPodSpecs, actualized for the current ClusterConfiguration klog.V(1).Infoln("[control-plane] getting StaticPodSpecs") specs := GetStaticPodSpecs(cfg, endpoint) @@ -116,6 +116,15 @@ func CreateStaticPodFiles(manifestDir string, cfg *kubeadmapi.ClusterConfigurati return errors.Errorf("couldn't retrieve StaticPodSpec for %q", componentName) } + // if kustomizeDir is defined, customize the static pod manifest + if kustomizeDir != "" { + kustomizedSpec, err := staticpodutil.KustomizeStaticPod(&spec, kustomizeDir) + if err != nil { + return errors.Wrapf(err, "failed to kustomize static pod manifest file for %q", componentName) + } + spec = *kustomizedSpec + } + // writes the StaticPodSpec to disk if err := staticpodutil.WriteStaticPodToDisk(componentName, manifestDir, spec); err != nil { return errors.Wrapf(err, "failed to create static pod manifest file for %q", componentName) diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go index da689d9fa9d..6fbc1a88139 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests_test.go @@ -18,6 +18,7 @@ package controlplane import ( "fmt" + "io/ioutil" "os" "path/filepath" "reflect" @@ -25,11 +26,13 @@ import ( "strings" "testing" + "github.com/lithammer/dedent" + "k8s.io/apimachinery/pkg/util/sets" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - + staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" testutil "k8s.io/kubernetes/cmd/kubeadm/test" ) @@ -121,7 +124,7 @@ func TestCreateStaticPodFilesAndWrappers(t *testing.T) { // Execute createStaticPodFunction manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName) - err := CreateStaticPodFiles(manifestPath, cfg, &kubeadmapi.APIEndpoint{}, test.components...) + err := CreateStaticPodFiles(manifestPath, "", cfg, &kubeadmapi.APIEndpoint{}, test.components...) if err != nil { t.Errorf("Error executing createStaticPodFunction: %v", err) return @@ -137,6 +140,56 @@ func TestCreateStaticPodFilesAndWrappers(t *testing.T) { } } +func TestCreateStaticPodFilesKustomize(t *testing.T) { + // Create temp folder for the test case + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + // Creates a Cluster Configuration + cfg := &kubeadmapi.ClusterConfiguration{ + KubernetesVersion: "v1.9.0", + } + + kustomizePath := filepath.Join(tmpdir, "kustomize") + err := os.MkdirAll(kustomizePath, 0777) + if err != nil { + t.Fatalf("Couldn't create %s", kustomizePath) + } + + patchString := dedent.Dedent(` + apiVersion: v1 + kind: Pod + metadata: + name: kube-apiserver + namespace: kube-system + annotations: + kustomize: patch for kube-apiserver + `) + + err = ioutil.WriteFile(filepath.Join(kustomizePath, "patch.yaml"), []byte(patchString), 0644) + if err != nil { + t.Fatalf("WriteFile returned unexpected error: %v", err) + } + + // Execute createStaticPodFunction with kustomizations + manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName) + err = CreateStaticPodFiles(manifestPath, kustomizePath, cfg, &kubeadmapi.APIEndpoint{}, kubeadmconstants.KubeAPIServer) + if err != nil { + t.Errorf("Error executing createStaticPodFunction: %v", err) + return + } + + pod, err := staticpodutil.ReadStaticPodFromDisk(filepath.Join(manifestPath, fmt.Sprintf("%s.yaml", kubeadmconstants.KubeAPIServer))) + if err != nil { + t.Errorf("Error executing ReadStaticPodFromDisk: %v", err) + return + } + + if _, ok := pod.ObjectMeta.Annotations["kustomize"]; !ok { + t.Error("Kustomize did not apply patches corresponding to the resource") + } +} + func TestGetAPIServerCommand(t *testing.T) { var tests = []struct { name string diff --git a/cmd/kubeadm/app/phases/etcd/BUILD b/cmd/kubeadm/app/phases/etcd/BUILD index c8182e8cb74..5da59b22551 100644 --- a/cmd/kubeadm/app/phases/etcd/BUILD +++ b/cmd/kubeadm/app/phases/etcd/BUILD @@ -14,7 +14,9 @@ go_test( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/util/etcd:go_default_library", + "//cmd/kubeadm/app/util/staticpod:go_default_library", "//cmd/kubeadm/test:go_default_library", + "//vendor/github.com/lithammer/dedent:go_default_library", ], ) diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go index 03c6ce25f41..459963be2c7 100644 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -25,7 +25,7 @@ import ( "github.com/pkg/errors" "k8s.io/klog" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" @@ -45,13 +45,22 @@ const ( // CreateLocalEtcdStaticPodManifestFile will write local etcd static pod manifest file. // This function is used by init - when the etcd cluster is empty - or by kubeadm // upgrade - when the etcd cluster is already up and running (and the --initial-cluster flag have no impact) -func CreateLocalEtcdStaticPodManifestFile(manifestDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error { +func CreateLocalEtcdStaticPodManifestFile(manifestDir, kustomizeDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error { if cfg.Etcd.External != nil { return errors.New("etcd static pod manifest cannot be generated for cluster using external etcd") } // gets etcd StaticPodSpec spec := GetEtcdPodSpec(cfg, endpoint, nodeName, []etcdutil.Member{}) + // if kustomizeDir is defined, customize the static pod manifest + if kustomizeDir != "" { + kustomizedSpec, err := staticpodutil.KustomizeStaticPod(&spec, kustomizeDir) + if err != nil { + return errors.Wrapf(err, "failed to kustomize static pod manifest file for %q", kubeadmconstants.Etcd) + } + spec = *kustomizedSpec + } + // writes etcd StaticPod to disk if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil { return err diff --git a/cmd/kubeadm/app/phases/etcd/local_test.go b/cmd/kubeadm/app/phases/etcd/local_test.go index aac26a7ea9d..098413abaa8 100644 --- a/cmd/kubeadm/app/phases/etcd/local_test.go +++ b/cmd/kubeadm/app/phases/etcd/local_test.go @@ -18,15 +18,19 @@ package etcd import ( "fmt" + "io/ioutil" "os" "path/filepath" "reflect" "sort" "testing" + "github.com/lithammer/dedent" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" + staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" testutil "k8s.io/kubernetes/cmd/kubeadm/test" ) @@ -92,7 +96,7 @@ func TestCreateLocalEtcdStaticPodManifestFile(t *testing.T) { for _, test := range tests { // Execute createStaticPodFunction manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName) - err := CreateLocalEtcdStaticPodManifestFile(manifestPath, "", test.cfg, &kubeadmapi.APIEndpoint{}) + err := CreateLocalEtcdStaticPodManifestFile(manifestPath, "", "", test.cfg, &kubeadmapi.APIEndpoint{}) if !test.expectedError { if err != nil { @@ -107,6 +111,61 @@ func TestCreateLocalEtcdStaticPodManifestFile(t *testing.T) { } } +func TestCreateLocalEtcdStaticPodManifestFileKustomize(t *testing.T) { + // Create temp folder for the test case + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + // Creates a Cluster Configuration + cfg := &kubeadmapi.ClusterConfiguration{ + KubernetesVersion: "v1.7.0", + Etcd: kubeadmapi.Etcd{ + Local: &kubeadmapi.LocalEtcd{ + DataDir: tmpdir + "/etcd", + }, + }, + } + + kustomizePath := filepath.Join(tmpdir, "kustomize") + err := os.MkdirAll(kustomizePath, 0777) + if err != nil { + t.Fatalf("Couldn't create %s", kustomizePath) + } + + patchString := dedent.Dedent(` + apiVersion: v1 + kind: Pod + metadata: + name: etcd + namespace: kube-system + annotations: + kustomize: patch for etcd + `) + + err = ioutil.WriteFile(filepath.Join(kustomizePath, "patch.yaml"), []byte(patchString), 0644) + if err != nil { + t.Fatalf("WriteFile returned unexpected error: %v", err) + } + + // Execute createStaticPodFunction with kustomizations + manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName) + err = CreateLocalEtcdStaticPodManifestFile(manifestPath, kustomizePath, "", cfg, &kubeadmapi.APIEndpoint{}) + if err != nil { + t.Errorf("Error executing createStaticPodFunction: %v", err) + return + } + + pod, err := staticpodutil.ReadStaticPodFromDisk(filepath.Join(manifestPath, fmt.Sprintf("%s.yaml", kubeadmconstants.Etcd))) + if err != nil { + t.Errorf("Error executing ReadStaticPodFromDisk: %v", err) + return + } + + if _, ok := pod.ObjectMeta.Annotations["kustomize"]; !ok { + t.Error("Kustomize did not apply patches corresponding to the resource") + } +} + func TestGetEtcdCommand(t *testing.T) { var tests = []struct { name string diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods.go b/cmd/kubeadm/app/phases/upgrade/staticpods.go index 2c3ac934a7d..78427f62ed8 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods.go @@ -314,7 +314,11 @@ func performEtcdStaticPodUpgrade(certsRenewMgr *renewal.Manager, client clientse // Write the updated etcd static Pod manifest into the temporary directory, at this point no etcd change // has occurred in any aspects. - if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.TempManifestDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil { + + // TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580 + kustomizeDir := "" + + if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.TempManifestDir(), kustomizeDir, cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil { return true, errors.Wrap(err, "error creating local etcd static pod manifest file") } @@ -460,7 +464,11 @@ func StaticPodControlPlane(client clientset.Interface, waiter apiclient.Waiter, // Write the updated static Pod manifests into the temporary directory fmt.Printf("[upgrade/staticpods] Writing new Static Pod manifests to %q\n", pathMgr.TempManifestDir()) - err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.TempManifestDir(), cfg) + + // TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580 + kustomizeDir := "" + + err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.TempManifestDir(), kustomizeDir, cfg) if err != nil { return errors.Wrap(err, "error creating init static pod manifest files") } @@ -612,7 +620,10 @@ func DryRunStaticPodUpgrade(internalcfg *kubeadmapi.InitConfiguration) error { } defer os.RemoveAll(dryRunManifestDir) - if err := controlplane.CreateInitStaticPodManifestFiles(dryRunManifestDir, internalcfg); err != nil { + // TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580 + kustomizeDir := "" + + if err := controlplane.CreateInitStaticPodManifestFiles(dryRunManifestDir, kustomizeDir, internalcfg); err != nil { return err } diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go index 08dfa863f68..023e8c48826 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go @@ -512,11 +512,11 @@ func TestStaticPodControlPlane(t *testing.T) { } // Initialize the directory with v1.7 manifests; should then be upgraded to v1.8 using the method - err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), oldcfg) + err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), "", oldcfg) if err != nil { t.Fatalf("couldn't run CreateInitStaticPodManifestFiles: %v", err) } - err = etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.RealManifestDir(), oldcfg.NodeRegistration.Name, &oldcfg.ClusterConfiguration, &oldcfg.LocalAPIEndpoint) + err = etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.RealManifestDir(), "", oldcfg.NodeRegistration.Name, &oldcfg.ClusterConfiguration, &oldcfg.LocalAPIEndpoint) if err != nil { t.Fatalf("couldn't run CreateLocalEtcdStaticPodManifestFile: %v", err) } diff --git a/cmd/kubeadm/app/util/BUILD b/cmd/kubeadm/app/util/BUILD index 001ed73ca6b..9a50c4c2e30 100644 --- a/cmd/kubeadm/app/util/BUILD +++ b/cmd/kubeadm/app/util/BUILD @@ -83,6 +83,7 @@ filegroup( "//cmd/kubeadm/app/util/etcd:all-srcs", "//cmd/kubeadm/app/util/initsystem:all-srcs", "//cmd/kubeadm/app/util/kubeconfig:all-srcs", + "//cmd/kubeadm/app/util/kustomize:all-srcs", "//cmd/kubeadm/app/util/pkiutil:all-srcs", "//cmd/kubeadm/app/util/pubkeypin:all-srcs", "//cmd/kubeadm/app/util/runtime:all-srcs", diff --git a/cmd/kubeadm/app/util/kustomize/BUILD b/cmd/kubeadm/app/util/kustomize/BUILD new file mode 100644 index 00000000000..5156028d257 --- /dev/null +++ b/cmd/kubeadm/app/util/kustomize/BUILD @@ -0,0 +1,49 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "kustomize.go", + "unstructured.go", + ], + importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/kustomize", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library", + "//staging/src/k8s.io/cli-runtime/pkg/kustomize:go_default_library", + "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/sigs.k8s.io/kustomize/pkg/fs:go_default_library", + "//vendor/sigs.k8s.io/kustomize/pkg/ifc:go_default_library", + "//vendor/sigs.k8s.io/kustomize/pkg/loader:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = [ + "kustomize_test.go", + "unstructured_test.go", + ], + embed = [":go_default_library"], + deps = [ + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//vendor/github.com/lithammer/dedent:go_default_library", + ], +) diff --git a/cmd/kubeadm/app/util/kustomize/kustomize.go b/cmd/kubeadm/app/util/kustomize/kustomize.go new file mode 100644 index 00000000000..a58f7ec16d0 --- /dev/null +++ b/cmd/kubeadm/app/util/kustomize/kustomize.go @@ -0,0 +1,181 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package kustomize contains helpers for working with embedded kustomize commands +package kustomize + +import ( + "bytes" + "fmt" + "io/ioutil" + "path/filepath" + "runtime" + "sync" + + "k8s.io/cli-runtime/pkg/kustomize" + "sigs.k8s.io/kustomize/pkg/fs" + "sigs.k8s.io/kustomize/pkg/loader" +) + +// Manager define a manager that allow access to kustomize capabilities +type Manager struct { + kustomizeDir string + us UnstructuredSlice +} + +var ( + lock = &sync.Mutex{} + instances = map[string]*Manager{} +) + +// GetManager return the KustomizeManager singleton instance +func GetManager(kustomizeDir string) (*Manager, error) { + lock.Lock() + defer lock.Unlock() + + // if the instance does not exists, create it + if _, ok := instances[kustomizeDir]; !ok { + km := &Manager{ + kustomizeDir: kustomizeDir, + } + + // loads the UnstructuredSlice with all the patches into the Manager + // NB. this is done at singleton instance level because kubeadm has a unique pool + // of patches that are applied to different content, at different time + if err := km.getUnstructuredSlice(); err != nil { + return nil, err + } + + instances[kustomizeDir] = km + } + + return instances[kustomizeDir], nil +} + +// getUnstructuredSlice returns a UnstructuredSlice with all the patches. +func (km *Manager) getUnstructuredSlice() error { + // kubeadm does not require a kustomization.yaml file listing all the resources/patches, so it is necessary + // to rebuild the list of patches manually + // TODO: make this git friendly - currently this works only for patches in local folders - + files, err := ioutil.ReadDir(km.kustomizeDir) + if err != nil { + return err + } + + var paths = []string{} + for _, file := range files { + if file.IsDir() { + continue + } + paths = append(paths, file.Name()) + } + + // Create a loader that mimics the behavior of kubectl kustomize, including support for reading from + // a local git repository like git@github.com:someOrg/someRepo.git or https://github.com/someOrg/someRepo?ref=someHash + fSys := fs.MakeRealFS() + ldr, err := loader.NewLoader(km.kustomizeDir, fSys) + if err != nil { + return err + } + defer ldr.Cleanup() + + // read all the kustomizations and build the UnstructuredSlice + us, err := NewUnstructuredSliceFromFiles(ldr, paths) + if err != nil { + return err + } + + km.us = us + return nil +} + +// Kustomize apply a set of patches to a resource. +// Portions of the kustomize logic in this function are taken from the kubernetes-sigs/kind project +func (km *Manager) Kustomize(res []byte) ([]byte, error) { + // create a loader that mimics the behavior of kubectl kustomize + // and converts the resource into a UnstructuredSlice + // Nb. in kubeadm we are controlling resource generation, and so we + // we are expecting 1 object into each resource, eg. the static pod. + // Nevertheless, this code is ready for more than one object per resource + resList, err := NewUnstructuredSliceFromBytes(res) + if err != nil { + return nil, err + } + + // create a list of resource and corresponding patches + var resources, patches UnstructuredSlice + for _, r := range resList { + resources = append(resources, r) + + resourcePatches := km.us.FilterResource(r.GroupVersionKind(), r.GetNamespace(), r.GetName()) + patches = append(patches, resourcePatches...) + } + + fmt.Printf("[kustomize] Applying %d patches\n", len(patches)) + + // if there are no patches, for the target resources, exit + if len(patches) == 0 { + return res, nil + } + + // create an in memory fs to use for the kustomization + memFS := fs.MakeFakeFS() + + var kustomization bytes.Buffer + fakeDir := "/" + // for Windows we need this to be a drive because kustomize uses filepath.Abs() + // which will add a drive letter if there is none. which drive letter is + // unimportant as the path is on the fake filesystem anyhow + if runtime.GOOS == "windows" { + fakeDir = `C:\` + } + + // write resources and patches to the in memory fs, generate the kustomization.yaml + // that ties everything together + kustomization.WriteString("resources:\n") + for i, r := range resources { + b, err := r.MarshalJSON() + if err != nil { + return nil, err + } + + name := fmt.Sprintf("resource-%d.json", i) + _ = memFS.WriteFile(filepath.Join(fakeDir, name), b) + fmt.Fprintf(&kustomization, " - %s\n", name) + } + + kustomization.WriteString("patches:\n") + for i, p := range patches { + b, err := p.MarshalJSON() + if err != nil { + return nil, err + } + + name := fmt.Sprintf("patch-%d.json", i) + _ = memFS.WriteFile(filepath.Join(fakeDir, name), b) + fmt.Fprintf(&kustomization, " - %s\n", name) + } + + memFS.WriteFile(filepath.Join(fakeDir, "kustomization.yaml"), kustomization.Bytes()) + + // Finally customize the target resource + var out bytes.Buffer + if err := kustomize.RunKustomizeBuild(&out, memFS, fakeDir); err != nil { + return nil, err + } + + return out.Bytes(), nil +} diff --git a/cmd/kubeadm/app/util/kustomize/kustomize_test.go b/cmd/kubeadm/app/util/kustomize/kustomize_test.go new file mode 100644 index 00000000000..0175a68a4c7 --- /dev/null +++ b/cmd/kubeadm/app/util/kustomize/kustomize_test.go @@ -0,0 +1,88 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kustomize + +import ( + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/lithammer/dedent" +) + +func TestKustomize(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal("Couldn't create tmpdir") + } + defer os.RemoveAll(tmpdir) + + resourceString := dedent.Dedent(` + apiVersion: v1 + kind: Pod + metadata: + name: kube-apiserver + `) + + patch1String := dedent.Dedent(` + apiVersion: v1 + kind: Pod + metadata: + name: kube-apiserver + annotations: + kustomize: patch for kube-apiserver + `) + + err = ioutil.WriteFile(filepath.Join(tmpdir, "patch-1.yaml"), []byte(patch1String), 0644) + if err != nil { + t.Fatalf("WriteFile returned unexpected error: %v", err) + } + + patch2String := dedent.Dedent(` + apiVersion: v1 + kind: Pod + metadata: + name: kube-scheduler + annotations: + kustomize: patch for kube-scheduler + `) + + err = ioutil.WriteFile(filepath.Join(tmpdir, "patch-2.yaml"), []byte(patch2String), 0644) + if err != nil { + t.Fatalf("WriteFile returned unexpected error: %v", err) + } + + km, err := GetManager(tmpdir) + if err != nil { + t.Errorf("GetManager returned unexpected error: %v", err) + } + + kustomized, err := km.Kustomize([]byte(resourceString)) + if err != nil { + t.Errorf("Kustomize returned unexpected error: %v", err) + } + + if !strings.Contains(string(kustomized), "kustomize: patch for kube-apiserver") { + t.Error("Kustomize did not apply patches corresponding to the resource") + } + + if strings.Contains(string(kustomized), "kustomize: patch for kube-scheduler") { + t.Error("Kustomize did apply patches not corresponding to the resource") + } +} diff --git a/cmd/kubeadm/app/util/kustomize/unstructured.go b/cmd/kubeadm/app/util/kustomize/unstructured.go new file mode 100644 index 00000000000..64bec38a3df --- /dev/null +++ b/cmd/kubeadm/app/util/kustomize/unstructured.go @@ -0,0 +1,148 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kustomize + +import ( + "bytes" + "encoding/json" + "io" + "strings" + + "github.com/pkg/errors" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/kustomize/pkg/ifc" +) + +// UnstructuredSlice is a slice of Unstructured objects. +// Unstructured objects are used to represent both resources and patches of any group/version/kind. +type UnstructuredSlice []*unstructured.Unstructured + +// NewUnstructuredSliceFromFiles returns a ResMap given a resource path slice. +// This func use a Loader to mimic the behavior of kubectl kustomize, and most specifically support for reading from +// a local git repository like git@github.com:someOrg/someRepo.git or https://github.com/someOrg/someRepo?ref=someHash +func NewUnstructuredSliceFromFiles(loader ifc.Loader, paths []string) (UnstructuredSlice, error) { + var result UnstructuredSlice + for _, path := range paths { + content, err := loader.Load(path) + if err != nil { + return nil, errors.Wrapf(err, "load from path %q failed", path) + } + res, err := NewUnstructuredSliceFromBytes(content) + if err != nil { + return nil, errors.Wrapf(err, "convert %q to Unstructured failed", path) + } + + result = append(result, res...) + } + return result, nil +} + +// NewUnstructuredSliceFromBytes returns a slice of Unstructured. +// This functions handles all the nuances of Kubernetes yaml (e.g. many yaml +// documents in one file, List of objects) +func NewUnstructuredSliceFromBytes(in []byte) (UnstructuredSlice, error) { + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(in), 1024) + var result UnstructuredSlice + var err error + // Parse all the yaml documents in the file + for err == nil || isEmptyYamlError(err) { + var u unstructured.Unstructured + err = decoder.Decode(&u) + // if the yaml document is a valid unstructured object + if err == nil { + // it the unstructured object is empty, move to the next + if len(u.Object) == 0 { + continue + } + + // validate the object has kind, metadata.name as required by Kustomize + if err := validate(u); err != nil { + return nil, err + } + + // if the document is a list of objects + if strings.HasSuffix(u.GetKind(), "List") { + // for each item in the list of objects + if err := u.EachListItem(func(item runtime.Object) error { + // Marshal the object + itemJSON, err := json.Marshal(item) + if err != nil { + return err + } + + // Get the UnstructuredSlice for the item + itemU, err := NewUnstructuredSliceFromBytes(itemJSON) + if err != nil { + return err + } + + // append the UnstructuredSlice for the item to the UnstructuredSlice + result = append(result, itemU...) + + return nil + }); err != nil { + return nil, err + } + + continue + } + + // append the object to the UnstructuredSlice + result = append(result, &u) + } + } + if err != io.EOF { + return nil, err + } + return result, nil +} + +// FilterResource returns all the Unstructured items in the UnstructuredSlice corresponding to a given resource +func (rs *UnstructuredSlice) FilterResource(gvk schema.GroupVersionKind, namespace, name string) UnstructuredSlice { + var result UnstructuredSlice + for _, r := range *rs { + if r.GroupVersionKind() == gvk && + r.GetNamespace() == namespace && + r.GetName() == name { + result = append(result, r) + } + } + return result +} + +// validate validates that u has kind and name +// except for kind `List`, which doesn't require a name +func validate(u unstructured.Unstructured) error { + kind := u.GetKind() + if kind == "" { + return errors.New("missing kind in object") + } else if strings.HasSuffix(kind, "List") { + return nil + } + if u.GetName() == "" { + return errors.New("missing metadata.name in object") + } + return nil +} + +func isEmptyYamlError(err error) bool { + return strings.Contains(err.Error(), "is missing in 'null'") +} diff --git a/cmd/kubeadm/app/util/kustomize/unstructured_test.go b/cmd/kubeadm/app/util/kustomize/unstructured_test.go new file mode 100644 index 00000000000..6802f0a186f --- /dev/null +++ b/cmd/kubeadm/app/util/kustomize/unstructured_test.go @@ -0,0 +1,222 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kustomize + +import ( + "testing" + + "github.com/lithammer/dedent" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func TestNewUnstructuredSliceFromBytes(t *testing.T) { + var useCases = []struct { + name string + in string + expectedUnctructured int + expectedError bool + }{ + { + name: "empty", + in: "", + expectedUnctructured: 0, + }, + { + name: "single patch", + in: dedent.Dedent(` + apiVersion: v1 + kind: Pod + metadata: + name: kube-apiserver + `), + expectedUnctructured: 1, + }, + { + name: "two patches as separated yaml documents", + in: dedent.Dedent(` + apiVersion: v1 + kind: Pod + metadata: + name: kube-apiserver + --- + apiVersion: v1 + kind: Pod + metadata: + name: kube-apiserver + `), + expectedUnctructured: 2, + }, + { + name: "two patches as a k8s list", + in: dedent.Dedent(` + apiVersion: v1 + kind: List + items: + - apiVersion: v1 + kind: Pod + metadata: + name: kube-apiserver + - apiVersion: v1 + kind: Pod + metadata: + name: kube-apiserver + `), + expectedUnctructured: 2, + }, + { + name: "nested k8s lists", + in: dedent.Dedent(` + apiVersion: v1 + kind: List + items: + - apiVersion: v1 + kind: Pod + metadata: + name: kube-apiserver + - apiVersion: v1 + kind: List + items: + - apiVersion: v1 + kind: Pod + metadata: + name: kube-apiserver + `), + expectedUnctructured: 2, + }, + { + name: "invalid yaml", + in: "$$$", + expectedError: true, + }, + { + name: "invalid patch (missing kind)", + in: dedent.Dedent(` + apiVersion: v1 + #kind: Pod + metadata: + name: kube-apiserver + `), + expectedError: true, + }, + { + name: "invalid patch (missing name)", + in: dedent.Dedent(` + apiVersion: v1 + kind: Pod + metadata: + #name: kube-apiserver + `), + expectedError: true, + }, + } + for _, rt := range useCases { + t.Run(rt.name, func(t *testing.T) { + r, err := NewUnstructuredSliceFromBytes([]byte(rt.in)) + if err != nil { + if !rt.expectedError { + t.Errorf("NewUnstructuredSliceFromBytes returned unexpected error: %v", err) + } + return + } + if err == nil && rt.expectedError { + t.Error("NewUnstructuredSliceFromBytes does not returned expected error") + } + if len(r) != rt.expectedUnctructured { + t.Errorf("Expected %d Unstructured items in the slice, actual %d", rt.expectedUnctructured, len(r)) + } + }) + } +} + +func TestFilterResource(t *testing.T) { + in := dedent.Dedent(` + apiVersion: v1 + kind: Pod + metadata: + name: kube-apiserver + namespace: kube-system + --- + apiVersion: v1 + kind: Pod + metadata: + name: kube-scheduler + namespace: kube-system + --- + apiVersion: v1 + kind: Pod + metadata: + name: kube-scheduler + namespace: kube-system + `) + u, err := NewUnstructuredSliceFromBytes([]byte(in)) + if err != nil { + t.Fatalf("NewUnstructuredSliceFromBytes returned unexpected error: %v", err) + } + + var useCases = []struct { + name string + rgvk schema.GroupVersionKind + rnamespace string + rname string + expectedUnctructured int + }{ + { + name: "match 1", + rgvk: schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}, + rnamespace: "kube-system", + rname: "kube-apiserver", + expectedUnctructured: 1, + }, + { + name: "match 2", + rgvk: schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}, + rnamespace: "kube-system", + rname: "kube-scheduler", + expectedUnctructured: 2, + }, + { + name: "match 0 (wrong gvk)", + rgvk: schema.GroupVersionKind{Group: "something", Version: "v1", Kind: "Pod"}, + rnamespace: "kube-system", + rname: "kube-scheduler", + expectedUnctructured: 0, + }, + { + name: "match 0 (wrong namespace)", + rgvk: schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}, + rnamespace: "kube-something", + rname: "kube-scheduler", + expectedUnctructured: 0, + }, + { + name: "match 0 (wrong namr)", + rgvk: schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}, + rnamespace: "kube-system", + rname: "kube-something", + expectedUnctructured: 0, + }, + } + for _, rt := range useCases { + t.Run(rt.name, func(t *testing.T) { + r := u.FilterResource(rt.rgvk, rt.rnamespace, rt.rname) + + if len(r) != rt.expectedUnctructured { + t.Errorf("Expected %d Unstructured items in the slice, actual %d", rt.expectedUnctructured, len(r)) + } + }) + } +} diff --git a/cmd/kubeadm/app/util/staticpod/BUILD b/cmd/kubeadm/app/util/staticpod/BUILD index 0af544da422..9d93d2f8265 100644 --- a/cmd/kubeadm/app/util/staticpod/BUILD +++ b/cmd/kubeadm/app/util/staticpod/BUILD @@ -15,6 +15,7 @@ go_test( "//cmd/kubeadm/test:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/github.com/lithammer/dedent:go_default_library", ], ) @@ -26,6 +27,7 @@ go_library( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/util:go_default_library", + "//cmd/kubeadm/app/util/kustomize:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go index c7cc66041ac..4f000c09a2d 100644 --- a/cmd/kubeadm/app/util/staticpod/utils.go +++ b/cmd/kubeadm/app/util/staticpod/utils.go @@ -28,12 +28,13 @@ import ( "github.com/pkg/errors" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/cmd/kubeadm/app/util/kustomize" ) const ( @@ -165,6 +166,38 @@ func GetExtraParameters(overrides map[string]string, defaults map[string]string) return command } +// KustomizeStaticPod applies patches defined in kustomizeDir to a static Pod manifest +func KustomizeStaticPod(pod *v1.Pod, kustomizeDir string) (*v1.Pod, error) { + // marshal the pod manifest into yaml + serialized, err := util.MarshalToYaml(pod, v1.SchemeGroupVersion) + if err != nil { + return pod, errors.Wrapf(err, "failed to marshal manifest to YAML") + } + + km, err := kustomize.GetManager(kustomizeDir) + if err != nil { + return pod, errors.Wrapf(err, "failed to GetPatches from %q", kustomizeDir) + } + + kustomized, err := km.Kustomize(serialized) + if err != nil { + return pod, errors.Wrap(err, "failed to kustomize static Pod manifest") + } + + // unmarshal kustomized yaml back into a pod manifest + obj, err := util.UnmarshalFromYaml(kustomized, v1.SchemeGroupVersion) + if err != nil { + return pod, errors.Wrap(err, "failed to unmarshal kustomize manifest from YAML") + } + + pod2, ok := obj.(*v1.Pod) + if !ok { + return pod, errors.Wrap(err, "kustomized manifest is not a valid Pod object") + } + + return pod2, nil +} + // WriteStaticPodToDisk writes a static pod file to disk func WriteStaticPodToDisk(componentName, manifestDir string, pod v1.Pod) error { diff --git a/cmd/kubeadm/app/util/staticpod/utils_test.go b/cmd/kubeadm/app/util/staticpod/utils_test.go index 125a496b682..76ec3130da5 100644 --- a/cmd/kubeadm/app/util/staticpod/utils_test.go +++ b/cmd/kubeadm/app/util/staticpod/utils_test.go @@ -25,7 +25,9 @@ import ( "strconv" "testing" - "k8s.io/api/core/v1" + "github.com/lithammer/dedent" + + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" testutil "k8s.io/kubernetes/cmd/kubeadm/test" @@ -604,3 +606,44 @@ func TestManifestFilesAreEqual(t *testing.T) { }) } } + +func TestKustomizeStaticPod(t *testing.T) { + // Create temp folder for the test case + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + patchString := dedent.Dedent(` + apiVersion: v1 + kind: Pod + metadata: + name: kube-apiserver + namespace: kube-system + annotations: + kustomize: patch for kube-apiserver + `) + + err := ioutil.WriteFile(filepath.Join(tmpdir, "patch.yaml"), []byte(patchString), 0644) + if err != nil { + t.Fatalf("WriteFile returned unexpected error: %v", err) + } + + pod := &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-apiserver", + Namespace: "kube-system", + }, + } + + kpod, err := KustomizeStaticPod(pod, tmpdir) + if err != nil { + t.Errorf("KustomizeStaticPod returned unexpected error: %v", err) + } + + if _, ok := kpod.ObjectMeta.Annotations["kustomize"]; !ok { + t.Error("Kustomize did not apply patches corresponding to the resource") + } +} diff --git a/go.mod b/go.mod index ab6b57985fc..b7c89056184 100644 --- a/go.mod +++ b/go.mod @@ -168,6 +168,7 @@ require ( k8s.io/repo-infra v0.0.0-20181204233714-00fe14e3d1a3 k8s.io/sample-apiserver v0.0.0 k8s.io/utils v0.0.0-20190801114015-581e00157fb1 + sigs.k8s.io/kustomize v2.0.3+incompatible sigs.k8s.io/yaml v1.1.0 vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc )