Merge pull request #80905 from fabriziopandini/kubeadm-kustomize-core

Kubeadm: kustomize core

cmd/kubeadm/app/cmd/phases/init/controlplane.go

@@ -144,6 +144,10 @@ func runControlPlaneSubphase(component string) func(c workflow.RunData) error {
 		cfg := data.Cfg()
 
 		fmt.Printf("[control-plane] Creating static Pod manifest for %q\n", component)
-		return controlplane.CreateStaticPodFiles(data.ManifestDir(), &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component)
+
+		// TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580
+		kustomizeDir := ""
+
+		return controlplane.CreateStaticPodFiles(data.ManifestDir(), kustomizeDir, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component)
 	}
 }

cmd/kubeadm/app/cmd/phases/init/etcd.go

@@ -92,7 +92,11 @@ func runEtcdPhaseLocal() func(c workflow.RunData) error {
 				fmt.Printf("[dryrun] Would ensure that %q directory is present\n", cfg.Etcd.Local.DataDir)
 			}
 			fmt.Printf("[etcd] Creating static Pod manifest for local etcd in %q\n", data.ManifestDir())
-			if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(data.ManifestDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil {
+
+			// TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580
+			kustomizeDir := ""
+
+			if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(data.ManifestDir(), kustomizeDir, cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil {
 				return errors.Wrap(err, "error creating local etcd static pod manifest file")
 			}
 		} else {

cmd/kubeadm/app/cmd/phases/join/controlplaneprepare.go

@@ -184,10 +184,14 @@ func runControlPlanePrepareControlPlaneSubphase(c workflow.RunData) error {
 
 	fmt.Printf("[control-plane] Using manifest folder %q\n", kubeadmconstants.GetStaticPodDirectory())
 
+	// TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580
+	kustomizeDir := ""
+
 	for _, component := range kubeadmconstants.ControlPlaneComponents {
 		fmt.Printf("[control-plane] Creating static Pod manifest for %q\n", component)
 		err := controlplane.CreateStaticPodFiles(
 			kubeadmconstants.GetStaticPodDirectory(),
+			kustomizeDir,
 			&cfg.ClusterConfiguration,
 			&cfg.LocalAPIEndpoint,
 			component,

cmd/kubeadm/app/phases/controlplane/BUILD

@@ -17,9 +17,11 @@ go_test(
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/phases/certs:go_default_library",
+        "//cmd/kubeadm/app/util/staticpod:go_default_library",
         "//cmd/kubeadm/test:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/github.com/lithammer/dedent:go_default_library",
     ],
 )
 

cmd/kubeadm/app/phases/controlplane/manifests.go

@@ -39,9 +39,9 @@ import (
 )
 
 // CreateInitStaticPodManifestFiles will write all static pod manifest files needed to bring up the control plane.
-func CreateInitStaticPodManifestFiles(manifestDir string, cfg *kubeadmapi.InitConfiguration) error {
+func CreateInitStaticPodManifestFiles(manifestDir, kustomizeDir string, cfg *kubeadmapi.InitConfiguration) error {
 	klog.V(1).Infoln("[control-plane] creating static Pod files")
-	return CreateStaticPodFiles(manifestDir, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler)
+	return CreateStaticPodFiles(manifestDir, kustomizeDir, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler)
 }
 
 // GetStaticPodSpecs returns all staticPodSpecs actualized to the context of the current configuration
@@ -103,7 +103,7 @@ func livenessProbe(host string, port int, scheme v1.URIScheme) *v1.Probe {
 }
 
 // CreateStaticPodFiles creates all the requested static pod files.
-func CreateStaticPodFiles(manifestDir string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, componentNames ...string) error {
+func CreateStaticPodFiles(manifestDir, kustomizeDir string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, componentNames ...string) error {
 	// gets the StaticPodSpecs, actualized for the current ClusterConfiguration
 	klog.V(1).Infoln("[control-plane] getting StaticPodSpecs")
 	specs := GetStaticPodSpecs(cfg, endpoint)
@@ -116,6 +116,15 @@ func CreateStaticPodFiles(manifestDir string, cfg *kubeadmapi.ClusterConfigurati
 			return errors.Errorf("couldn't retrieve StaticPodSpec for %q", componentName)
 		}
 
+		// if kustomizeDir is defined, customize the static pod manifest
+		if kustomizeDir != "" {
+			kustomizedSpec, err := staticpodutil.KustomizeStaticPod(&spec, kustomizeDir)
+			if err != nil {
+				return errors.Wrapf(err, "failed to kustomize static pod manifest file for %q", componentName)
+			}
+			spec = *kustomizedSpec
+		}
+
 		// writes the StaticPodSpec to disk
 		if err := staticpodutil.WriteStaticPodToDisk(componentName, manifestDir, spec); err != nil {
 			return errors.Wrapf(err, "failed to create static pod manifest file for %q", componentName)

cmd/kubeadm/app/phases/controlplane/manifests_test.go

@@ -18,6 +18,7 @@ package controlplane
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -25,11 +26,13 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/lithammer/dedent"
+
 	"k8s.io/apimachinery/pkg/util/sets"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
-
+	staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
 	testutil "k8s.io/kubernetes/cmd/kubeadm/test"
 )
 
@@ -121,7 +124,7 @@ func TestCreateStaticPodFilesAndWrappers(t *testing.T) {
 
 			// Execute createStaticPodFunction
 			manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName)
-			err := CreateStaticPodFiles(manifestPath, cfg, &kubeadmapi.APIEndpoint{}, test.components...)
+			err := CreateStaticPodFiles(manifestPath, "", cfg, &kubeadmapi.APIEndpoint{}, test.components...)
 			if err != nil {
 				t.Errorf("Error executing createStaticPodFunction: %v", err)
 				return
@@ -137,6 +140,56 @@ func TestCreateStaticPodFilesAndWrappers(t *testing.T) {
 	}
 }
 
+func TestCreateStaticPodFilesKustomize(t *testing.T) {
+	// Create temp folder for the test case
+	tmpdir := testutil.SetupTempDir(t)
+	defer os.RemoveAll(tmpdir)
+
+	// Creates a Cluster Configuration
+	cfg := &kubeadmapi.ClusterConfiguration{
+		KubernetesVersion: "v1.9.0",
+	}
+
+	kustomizePath := filepath.Join(tmpdir, "kustomize")
+	err := os.MkdirAll(kustomizePath, 0777)
+	if err != nil {
+		t.Fatalf("Couldn't create %s", kustomizePath)
+	}
+
+	patchString := dedent.Dedent(`
+    apiVersion: v1
+    kind: Pod
+    metadata:
+        name: kube-apiserver
+        namespace: kube-system
+        annotations:
+            kustomize: patch for kube-apiserver
+    `)
+
+	err = ioutil.WriteFile(filepath.Join(kustomizePath, "patch.yaml"), []byte(patchString), 0644)
+	if err != nil {
+		t.Fatalf("WriteFile returned unexpected error: %v", err)
+	}
+
+	// Execute createStaticPodFunction with kustomizations
+	manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName)
+	err = CreateStaticPodFiles(manifestPath, kustomizePath, cfg, &kubeadmapi.APIEndpoint{}, kubeadmconstants.KubeAPIServer)
+	if err != nil {
+		t.Errorf("Error executing createStaticPodFunction: %v", err)
+		return
+	}
+
+	pod, err := staticpodutil.ReadStaticPodFromDisk(filepath.Join(manifestPath, fmt.Sprintf("%s.yaml", kubeadmconstants.KubeAPIServer)))
+	if err != nil {
+		t.Errorf("Error executing ReadStaticPodFromDisk: %v", err)
+		return
+	}
+
+	if _, ok := pod.ObjectMeta.Annotations["kustomize"]; !ok {
+		t.Error("Kustomize did not apply patches corresponding to the resource")
+	}
+}
+
 func TestGetAPIServerCommand(t *testing.T) {
 	var tests = []struct {
 		name     string

cmd/kubeadm/app/phases/etcd/BUILD

@@ -14,7 +14,9 @@ go_test(
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/util/etcd:go_default_library",
+        "//cmd/kubeadm/app/util/staticpod:go_default_library",
         "//cmd/kubeadm/test:go_default_library",
+        "//vendor/github.com/lithammer/dedent:go_default_library",
     ],
 )
 

cmd/kubeadm/app/phases/etcd/local.go

@@ -25,7 +25,7 @@ import (
 	"github.com/pkg/errors"
 	"k8s.io/klog"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
@@ -45,13 +45,22 @@ const (
 // CreateLocalEtcdStaticPodManifestFile will write local etcd static pod manifest file.
 // This function is used by init - when the etcd cluster is empty - or by kubeadm
 // upgrade - when the etcd cluster is already up and running (and the --initial-cluster flag have no impact)
-func CreateLocalEtcdStaticPodManifestFile(manifestDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error {
+func CreateLocalEtcdStaticPodManifestFile(manifestDir, kustomizeDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error {
 	if cfg.Etcd.External != nil {
 		return errors.New("etcd static pod manifest cannot be generated for cluster using external etcd")
 	}
 	// gets etcd StaticPodSpec
 	spec := GetEtcdPodSpec(cfg, endpoint, nodeName, []etcdutil.Member{})
 
+	// if kustomizeDir is defined, customize the static pod manifest
+	if kustomizeDir != "" {
+		kustomizedSpec, err := staticpodutil.KustomizeStaticPod(&spec, kustomizeDir)
+		if err != nil {
+			return errors.Wrapf(err, "failed to kustomize static pod manifest file for %q", kubeadmconstants.Etcd)
+		}
+		spec = *kustomizedSpec
+	}
+
 	// writes etcd StaticPod to disk
 	if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
 		return err

cmd/kubeadm/app/phases/etcd/local_test.go

@@ -18,15 +18,19 @@ package etcd
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"reflect"
 	"sort"
 	"testing"
 
+	"github.com/lithammer/dedent"
+
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
+	staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
 	testutil "k8s.io/kubernetes/cmd/kubeadm/test"
 )
 
@@ -92,7 +96,7 @@ func TestCreateLocalEtcdStaticPodManifestFile(t *testing.T) {
 	for _, test := range tests {
 		// Execute createStaticPodFunction
 		manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName)
-		err := CreateLocalEtcdStaticPodManifestFile(manifestPath, "", test.cfg, &kubeadmapi.APIEndpoint{})
+		err := CreateLocalEtcdStaticPodManifestFile(manifestPath, "", "", test.cfg, &kubeadmapi.APIEndpoint{})
 
 		if !test.expectedError {
 			if err != nil {
@@ -107,6 +111,61 @@ func TestCreateLocalEtcdStaticPodManifestFile(t *testing.T) {
 	}
 }
 
+func TestCreateLocalEtcdStaticPodManifestFileKustomize(t *testing.T) {
+	// Create temp folder for the test case
+	tmpdir := testutil.SetupTempDir(t)
+	defer os.RemoveAll(tmpdir)
+
+	// Creates a Cluster Configuration
+	cfg := &kubeadmapi.ClusterConfiguration{
+		KubernetesVersion: "v1.7.0",
+		Etcd: kubeadmapi.Etcd{
+			Local: &kubeadmapi.LocalEtcd{
+				DataDir: tmpdir + "/etcd",
+			},
+		},
+	}
+
+	kustomizePath := filepath.Join(tmpdir, "kustomize")
+	err := os.MkdirAll(kustomizePath, 0777)
+	if err != nil {
+		t.Fatalf("Couldn't create %s", kustomizePath)
+	}
+
+	patchString := dedent.Dedent(`
+    apiVersion: v1
+    kind: Pod
+    metadata:
+        name: etcd
+        namespace: kube-system
+        annotations:
+            kustomize: patch for etcd
+    `)
+
+	err = ioutil.WriteFile(filepath.Join(kustomizePath, "patch.yaml"), []byte(patchString), 0644)
+	if err != nil {
+		t.Fatalf("WriteFile returned unexpected error: %v", err)
+	}
+
+	// Execute createStaticPodFunction with kustomizations
+	manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName)
+	err = CreateLocalEtcdStaticPodManifestFile(manifestPath, kustomizePath, "", cfg, &kubeadmapi.APIEndpoint{})
+	if err != nil {
+		t.Errorf("Error executing createStaticPodFunction: %v", err)
+		return
+	}
+
+	pod, err := staticpodutil.ReadStaticPodFromDisk(filepath.Join(manifestPath, fmt.Sprintf("%s.yaml", kubeadmconstants.Etcd)))
+	if err != nil {
+		t.Errorf("Error executing ReadStaticPodFromDisk: %v", err)
+		return
+	}
+
+	if _, ok := pod.ObjectMeta.Annotations["kustomize"]; !ok {
+		t.Error("Kustomize did not apply patches corresponding to the resource")
+	}
+}
+
 func TestGetEtcdCommand(t *testing.T) {
 	var tests = []struct {
 		name             string

cmd/kubeadm/app/phases/upgrade/staticpods.go

@@ -314,7 +314,11 @@ func performEtcdStaticPodUpgrade(certsRenewMgr *renewal.Manager, client clientse
 
 	// Write the updated etcd static Pod manifest into the temporary directory, at this point no etcd change
 	// has occurred in any aspects.
-	if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.TempManifestDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil {
+
+	// TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580
+	kustomizeDir := ""
+
+	if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.TempManifestDir(), kustomizeDir, cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil {
 		return true, errors.Wrap(err, "error creating local etcd static pod manifest file")
 	}
 
@@ -460,7 +464,11 @@ func StaticPodControlPlane(client clientset.Interface, waiter apiclient.Waiter,
 
 	// Write the updated static Pod manifests into the temporary directory
 	fmt.Printf("[upgrade/staticpods] Writing new Static Pod manifests to %q\n", pathMgr.TempManifestDir())
-	err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.TempManifestDir(), cfg)
+
+	// TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580
+	kustomizeDir := ""
+
+	err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.TempManifestDir(), kustomizeDir, cfg)
 	if err != nil {
 		return errors.Wrap(err, "error creating init static pod manifest files")
 	}
@@ -612,7 +620,10 @@ func DryRunStaticPodUpgrade(internalcfg *kubeadmapi.InitConfiguration) error {
 	}
 	defer os.RemoveAll(dryRunManifestDir)
 
-	if err := controlplane.CreateInitStaticPodManifestFiles(dryRunManifestDir, internalcfg); err != nil {
+	// TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580
+	kustomizeDir := ""
+
+	if err := controlplane.CreateInitStaticPodManifestFiles(dryRunManifestDir, kustomizeDir, internalcfg); err != nil {
 		return err
 	}
 

cmd/kubeadm/app/phases/upgrade/staticpods_test.go

@@ -512,11 +512,11 @@ func TestStaticPodControlPlane(t *testing.T) {
 			}
 
 			// Initialize the directory with v1.7 manifests; should then be upgraded to v1.8 using the method
-			err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), oldcfg)
+			err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), "", oldcfg)
 			if err != nil {
 				t.Fatalf("couldn't run CreateInitStaticPodManifestFiles: %v", err)
 			}
-			err = etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.RealManifestDir(), oldcfg.NodeRegistration.Name, &oldcfg.ClusterConfiguration, &oldcfg.LocalAPIEndpoint)
+			err = etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.RealManifestDir(), "", oldcfg.NodeRegistration.Name, &oldcfg.ClusterConfiguration, &oldcfg.LocalAPIEndpoint)
 			if err != nil {
 				t.Fatalf("couldn't run CreateLocalEtcdStaticPodManifestFile: %v", err)
 			}

cmd/kubeadm/app/util/BUILD

@@ -83,6 +83,7 @@ filegroup(
         "//cmd/kubeadm/app/util/etcd:all-srcs",
         "//cmd/kubeadm/app/util/initsystem:all-srcs",
         "//cmd/kubeadm/app/util/kubeconfig:all-srcs",
+        "//cmd/kubeadm/app/util/kustomize:all-srcs",
         "//cmd/kubeadm/app/util/pkiutil:all-srcs",
         "//cmd/kubeadm/app/util/pubkeypin:all-srcs",
         "//cmd/kubeadm/app/util/runtime:all-srcs",

cmd/kubeadm/app/util/kustomize/BUILD

@@ -0,0 +1,49 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "kustomize.go",
+        "unstructured.go",
+    ],
+    importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/kustomize",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
+        "//staging/src/k8s.io/cli-runtime/pkg/kustomize:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/sigs.k8s.io/kustomize/pkg/fs:go_default_library",
+        "//vendor/sigs.k8s.io/kustomize/pkg/ifc:go_default_library",
+        "//vendor/sigs.k8s.io/kustomize/pkg/loader:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "kustomize_test.go",
+        "unstructured_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = [
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/github.com/lithammer/dedent:go_default_library",
+    ],
+)

cmd/kubeadm/app/util/kustomize/kustomize.go

@@ -0,0 +1,181 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package kustomize contains helpers for working with embedded kustomize commands
+package kustomize
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"runtime"
+	"sync"
+
+	"k8s.io/cli-runtime/pkg/kustomize"
+	"sigs.k8s.io/kustomize/pkg/fs"
+	"sigs.k8s.io/kustomize/pkg/loader"
+)
+
+// Manager define a manager that allow access to kustomize capabilities
+type Manager struct {
+	kustomizeDir string
+	us           UnstructuredSlice
+}
+
+var (
+	lock      = &sync.Mutex{}
+	instances = map[string]*Manager{}
+)
+
+// GetManager return the KustomizeManager singleton instance
+func GetManager(kustomizeDir string) (*Manager, error) {
+	lock.Lock()
+	defer lock.Unlock()
+
+	// if the instance does not exists, create it
+	if _, ok := instances[kustomizeDir]; !ok {
+		km := &Manager{
+			kustomizeDir: kustomizeDir,
+		}
+
+		// loads the UnstructuredSlice with all the patches into the Manager
+		// NB. this is done at singleton instance level because kubeadm has a unique pool
+		// of patches that are applied to different content, at different time
+		if err := km.getUnstructuredSlice(); err != nil {
+			return nil, err
+		}
+
+		instances[kustomizeDir] = km
+	}
+
+	return instances[kustomizeDir], nil
+}
+
+// getUnstructuredSlice returns a UnstructuredSlice with all the patches.
+func (km *Manager) getUnstructuredSlice() error {
+	// kubeadm does not require a kustomization.yaml file listing all the resources/patches, so it is necessary
+	// to rebuild the list of patches manually
+	// TODO: make this git friendly - currently this works only for patches in local folders -
+	files, err := ioutil.ReadDir(km.kustomizeDir)
+	if err != nil {
+		return err
+	}
+
+	var paths = []string{}
+	for _, file := range files {
+		if file.IsDir() {
+			continue
+		}
+		paths = append(paths, file.Name())
+	}
+
+	// Create a loader that mimics the behavior of kubectl kustomize, including support for reading from
+	// a local git repository like git@github.com:someOrg/someRepo.git or https://github.com/someOrg/someRepo?ref=someHash
+	fSys := fs.MakeRealFS()
+	ldr, err := loader.NewLoader(km.kustomizeDir, fSys)
+	if err != nil {
+		return err
+	}
+	defer ldr.Cleanup()
+
+	// read all the kustomizations and build the UnstructuredSlice
+	us, err := NewUnstructuredSliceFromFiles(ldr, paths)
+	if err != nil {
+		return err
+	}
+
+	km.us = us
+	return nil
+}
+
+// Kustomize apply a set of patches to a resource.
+// Portions of the kustomize logic in this function are taken from the kubernetes-sigs/kind project
+func (km *Manager) Kustomize(res []byte) ([]byte, error) {
+	// create a loader that mimics the behavior of kubectl kustomize
+	// and converts the resource into a UnstructuredSlice
+	// Nb. in kubeadm we are controlling resource generation, and so we
+	// we are expecting 1 object into each resource, eg. the static pod.
+	// Nevertheless, this code is ready for more than one object per resource
+	resList, err := NewUnstructuredSliceFromBytes(res)
+	if err != nil {
+		return nil, err
+	}
+
+	// create a list of resource and corresponding patches
+	var resources, patches UnstructuredSlice
+	for _, r := range resList {
+		resources = append(resources, r)
+
+		resourcePatches := km.us.FilterResource(r.GroupVersionKind(), r.GetNamespace(), r.GetName())
+		patches = append(patches, resourcePatches...)
+	}
+
+	fmt.Printf("[kustomize] Applying %d patches\n", len(patches))
+
+	// if there are no patches, for the target resources, exit
+	if len(patches) == 0 {
+		return res, nil
+	}
+
+	// create an in memory fs to use for the kustomization
+	memFS := fs.MakeFakeFS()
+
+	var kustomization bytes.Buffer
+	fakeDir := "/"
+	// for Windows we need this to be a drive because kustomize uses filepath.Abs()
+	// which will add a drive letter if there is none. which drive letter is
+	// unimportant as the path is on the fake filesystem anyhow
+	if runtime.GOOS == "windows" {
+		fakeDir = `C:\`
+	}
+
+	// write resources and patches to the in memory fs, generate the kustomization.yaml
+	// that ties everything together
+	kustomization.WriteString("resources:\n")
+	for i, r := range resources {
+		b, err := r.MarshalJSON()
+		if err != nil {
+			return nil, err
+		}
+
+		name := fmt.Sprintf("resource-%d.json", i)
+		_ = memFS.WriteFile(filepath.Join(fakeDir, name), b)
+		fmt.Fprintf(&kustomization, " - %s\n", name)
+	}
+
+	kustomization.WriteString("patches:\n")
+	for i, p := range patches {
+		b, err := p.MarshalJSON()
+		if err != nil {
+			return nil, err
+		}
+
+		name := fmt.Sprintf("patch-%d.json", i)
+		_ = memFS.WriteFile(filepath.Join(fakeDir, name), b)
+		fmt.Fprintf(&kustomization, " - %s\n", name)
+	}
+
+	memFS.WriteFile(filepath.Join(fakeDir, "kustomization.yaml"), kustomization.Bytes())
+
+	// Finally customize the target resource
+	var out bytes.Buffer
+	if err := kustomize.RunKustomizeBuild(&out, memFS, fakeDir); err != nil {
+		return nil, err
+	}
+
+	return out.Bytes(), nil
+}

cmd/kubeadm/app/util/kustomize/kustomize_test.go

@@ -0,0 +1,88 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kustomize
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/lithammer/dedent"
+)
+
+func TestKustomize(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal("Couldn't create tmpdir")
+	}
+	defer os.RemoveAll(tmpdir)
+
+	resourceString := dedent.Dedent(`
+    apiVersion: v1
+    kind: Pod
+    metadata:
+        name: kube-apiserver
+    `)
+
+	patch1String := dedent.Dedent(`
+    apiVersion: v1
+    kind: Pod
+    metadata:
+        name: kube-apiserver
+        annotations:
+            kustomize: patch for kube-apiserver
+    `)
+
+	err = ioutil.WriteFile(filepath.Join(tmpdir, "patch-1.yaml"), []byte(patch1String), 0644)
+	if err != nil {
+		t.Fatalf("WriteFile returned unexpected error: %v", err)
+	}
+
+	patch2String := dedent.Dedent(`
+    apiVersion: v1
+    kind: Pod
+    metadata:
+        name: kube-scheduler
+        annotations:
+            kustomize: patch for kube-scheduler
+    `)
+
+	err = ioutil.WriteFile(filepath.Join(tmpdir, "patch-2.yaml"), []byte(patch2String), 0644)
+	if err != nil {
+		t.Fatalf("WriteFile returned unexpected error: %v", err)
+	}
+
+	km, err := GetManager(tmpdir)
+	if err != nil {
+		t.Errorf("GetManager returned unexpected error: %v", err)
+	}
+
+	kustomized, err := km.Kustomize([]byte(resourceString))
+	if err != nil {
+		t.Errorf("Kustomize returned unexpected error: %v", err)
+	}
+
+	if !strings.Contains(string(kustomized), "kustomize: patch for kube-apiserver") {
+		t.Error("Kustomize did not apply patches corresponding to the resource")
+	}
+
+	if strings.Contains(string(kustomized), "kustomize: patch for kube-scheduler") {
+		t.Error("Kustomize did apply patches not corresponding to the resource")
+	}
+}

cmd/kubeadm/app/util/kustomize/unstructured.go

@@ -0,0 +1,148 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kustomize
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/yaml"
+	"sigs.k8s.io/kustomize/pkg/ifc"
+)
+
+// UnstructuredSlice is a slice of Unstructured objects.
+// Unstructured objects are used to represent both resources and patches of any group/version/kind.
+type UnstructuredSlice []*unstructured.Unstructured
+
+// NewUnstructuredSliceFromFiles returns a ResMap given a resource path slice.
+// This func use a Loader to mimic the behavior of kubectl kustomize, and most specifically support for reading from
+// a local git repository like git@github.com:someOrg/someRepo.git or https://github.com/someOrg/someRepo?ref=someHash
+func NewUnstructuredSliceFromFiles(loader ifc.Loader, paths []string) (UnstructuredSlice, error) {
+	var result UnstructuredSlice
+	for _, path := range paths {
+		content, err := loader.Load(path)
+		if err != nil {
+			return nil, errors.Wrapf(err, "load from path %q failed", path)
+		}
+		res, err := NewUnstructuredSliceFromBytes(content)
+		if err != nil {
+			return nil, errors.Wrapf(err, "convert %q to Unstructured failed", path)
+		}
+
+		result = append(result, res...)
+	}
+	return result, nil
+}
+
+// NewUnstructuredSliceFromBytes returns a slice of Unstructured.
+// This functions handles all the nuances of Kubernetes yaml (e.g. many yaml
+// documents in one file, List of objects)
+func NewUnstructuredSliceFromBytes(in []byte) (UnstructuredSlice, error) {
+	decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(in), 1024)
+	var result UnstructuredSlice
+	var err error
+	// Parse all the yaml documents in the file
+	for err == nil || isEmptyYamlError(err) {
+		var u unstructured.Unstructured
+		err = decoder.Decode(&u)
+		// if the yaml document is a valid unstructured object
+		if err == nil {
+			// it the unstructured object is empty, move to the next
+			if len(u.Object) == 0 {
+				continue
+			}
+
+			// validate the object has kind, metadata.name as required by Kustomize
+			if err := validate(u); err != nil {
+				return nil, err
+			}
+
+			// if the document is a list of objects
+			if strings.HasSuffix(u.GetKind(), "List") {
+				// for each item in the list of objects
+				if err := u.EachListItem(func(item runtime.Object) error {
+					// Marshal the object
+					itemJSON, err := json.Marshal(item)
+					if err != nil {
+						return err
+					}
+
+					// Get the UnstructuredSlice for the item
+					itemU, err := NewUnstructuredSliceFromBytes(itemJSON)
+					if err != nil {
+						return err
+					}
+
+					// append the UnstructuredSlice for the item to the UnstructuredSlice
+					result = append(result, itemU...)
+
+					return nil
+				}); err != nil {
+					return nil, err
+				}
+
+				continue
+			}
+
+			// append the object to the UnstructuredSlice
+			result = append(result, &u)
+		}
+	}
+	if err != io.EOF {
+		return nil, err
+	}
+	return result, nil
+}
+
+// FilterResource returns all the Unstructured items in the UnstructuredSlice corresponding to a given resource
+func (rs *UnstructuredSlice) FilterResource(gvk schema.GroupVersionKind, namespace, name string) UnstructuredSlice {
+	var result UnstructuredSlice
+	for _, r := range *rs {
+		if r.GroupVersionKind() == gvk &&
+			r.GetNamespace() == namespace &&
+			r.GetName() == name {
+			result = append(result, r)
+		}
+	}
+	return result
+}
+
+// validate validates that u has kind and name
+// except for kind `List`, which doesn't require a name
+func validate(u unstructured.Unstructured) error {
+	kind := u.GetKind()
+	if kind == "" {
+		return errors.New("missing kind in object")
+	} else if strings.HasSuffix(kind, "List") {
+		return nil
+	}
+	if u.GetName() == "" {
+		return errors.New("missing metadata.name in object")
+	}
+	return nil
+}
+
+func isEmptyYamlError(err error) bool {
+	return strings.Contains(err.Error(), "is missing in 'null'")
+}

cmd/kubeadm/app/util/kustomize/unstructured_test.go

@@ -0,0 +1,222 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kustomize
+
+import (
+	"testing"
+
+	"github.com/lithammer/dedent"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+func TestNewUnstructuredSliceFromBytes(t *testing.T) {
+	var useCases = []struct {
+		name                 string
+		in                   string
+		expectedUnctructured int
+		expectedError        bool
+	}{
+		{
+			name:                 "empty",
+			in:                   "",
+			expectedUnctructured: 0,
+		},
+		{
+			name: "single patch",
+			in: dedent.Dedent(`
+            apiVersion: v1
+            kind: Pod
+            metadata:
+              name: kube-apiserver
+            `),
+			expectedUnctructured: 1,
+		},
+		{
+			name: "two patches as separated yaml documents",
+			in: dedent.Dedent(`
+            apiVersion: v1
+            kind: Pod
+            metadata:
+              name: kube-apiserver
+            ---
+            apiVersion: v1
+            kind: Pod
+            metadata:
+              name: kube-apiserver
+            `),
+			expectedUnctructured: 2,
+		},
+		{
+			name: "two patches as a k8s list",
+			in: dedent.Dedent(`
+            apiVersion: v1
+            kind: List
+            items:
+            - apiVersion: v1
+              kind: Pod
+              metadata:
+                name: kube-apiserver
+            - apiVersion: v1
+              kind: Pod
+              metadata:
+                name: kube-apiserver
+            `),
+			expectedUnctructured: 2,
+		},
+		{
+			name: "nested k8s lists",
+			in: dedent.Dedent(`
+            apiVersion: v1
+            kind: List
+            items:
+            - apiVersion: v1
+              kind: Pod
+              metadata:
+                name: kube-apiserver
+            - apiVersion: v1
+              kind: List
+              items:
+              - apiVersion: v1
+                kind: Pod
+                metadata:
+                  name: kube-apiserver
+            `),
+			expectedUnctructured: 2,
+		},
+		{
+			name:          "invalid yaml",
+			in:            "$$$",
+			expectedError: true,
+		},
+		{
+			name: "invalid patch (missing kind)",
+			in: dedent.Dedent(`
+            apiVersion: v1
+            #kind: Pod
+            metadata:
+              name: kube-apiserver
+            `),
+			expectedError: true,
+		},
+		{
+			name: "invalid patch (missing name)",
+			in: dedent.Dedent(`
+            apiVersion: v1
+            kind: Pod
+            metadata:
+              #name: kube-apiserver
+            `),
+			expectedError: true,
+		},
+	}
+	for _, rt := range useCases {
+		t.Run(rt.name, func(t *testing.T) {
+			r, err := NewUnstructuredSliceFromBytes([]byte(rt.in))
+			if err != nil {
+				if !rt.expectedError {
+					t.Errorf("NewUnstructuredSliceFromBytes returned unexpected error: %v", err)
+				}
+				return
+			}
+			if err == nil && rt.expectedError {
+				t.Error("NewUnstructuredSliceFromBytes does not returned expected error")
+			}
+			if len(r) != rt.expectedUnctructured {
+				t.Errorf("Expected %d Unstructured items in the slice, actual %d", rt.expectedUnctructured, len(r))
+			}
+		})
+	}
+}
+
+func TestFilterResource(t *testing.T) {
+	in := dedent.Dedent(`
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: kube-apiserver
+      namespace: kube-system
+    ---
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: kube-scheduler
+      namespace: kube-system
+    ---
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: kube-scheduler
+      namespace: kube-system
+    `)
+	u, err := NewUnstructuredSliceFromBytes([]byte(in))
+	if err != nil {
+		t.Fatalf("NewUnstructuredSliceFromBytes returned unexpected error: %v", err)
+	}
+
+	var useCases = []struct {
+		name                 string
+		rgvk                 schema.GroupVersionKind
+		rnamespace           string
+		rname                string
+		expectedUnctructured int
+	}{
+		{
+			name:                 "match 1",
+			rgvk:                 schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"},
+			rnamespace:           "kube-system",
+			rname:                "kube-apiserver",
+			expectedUnctructured: 1,
+		},
+		{
+			name:                 "match 2",
+			rgvk:                 schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"},
+			rnamespace:           "kube-system",
+			rname:                "kube-scheduler",
+			expectedUnctructured: 2,
+		},
+		{
+			name:                 "match 0 (wrong gvk)",
+			rgvk:                 schema.GroupVersionKind{Group: "something", Version: "v1", Kind: "Pod"},
+			rnamespace:           "kube-system",
+			rname:                "kube-scheduler",
+			expectedUnctructured: 0,
+		},
+		{
+			name:                 "match 0 (wrong namespace)",
+			rgvk:                 schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"},
+			rnamespace:           "kube-something",
+			rname:                "kube-scheduler",
+			expectedUnctructured: 0,
+		},
+		{
+			name:                 "match 0 (wrong namr)",
+			rgvk:                 schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"},
+			rnamespace:           "kube-system",
+			rname:                "kube-something",
+			expectedUnctructured: 0,
+		},
+	}
+	for _, rt := range useCases {
+		t.Run(rt.name, func(t *testing.T) {
+			r := u.FilterResource(rt.rgvk, rt.rnamespace, rt.rname)
+
+			if len(r) != rt.expectedUnctructured {
+				t.Errorf("Expected %d Unstructured items in the slice, actual %d", rt.expectedUnctructured, len(r))
+			}
+		})
+	}
+}

cmd/kubeadm/app/util/staticpod/BUILD

@@ -15,6 +15,7 @@ go_test(
         "//cmd/kubeadm/test:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/github.com/lithammer/dedent:go_default_library",
     ],
 )
 
@@ -26,6 +27,7 @@ go_library(
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",
+        "//cmd/kubeadm/app/util/kustomize:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

cmd/kubeadm/app/util/staticpod/utils.go

@@ -28,12 +28,13 @@ import (
 
 	"github.com/pkg/errors"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/kustomize"
 )
 
 const (
@@ -165,6 +166,38 @@ func GetExtraParameters(overrides map[string]string, defaults map[string]string)
 	return command
 }
 
+// KustomizeStaticPod applies patches defined in kustomizeDir to a static Pod manifest
+func KustomizeStaticPod(pod *v1.Pod, kustomizeDir string) (*v1.Pod, error) {
+	// marshal the pod manifest into yaml
+	serialized, err := util.MarshalToYaml(pod, v1.SchemeGroupVersion)
+	if err != nil {
+		return pod, errors.Wrapf(err, "failed to marshal manifest to YAML")
+	}
+
+	km, err := kustomize.GetManager(kustomizeDir)
+	if err != nil {
+		return pod, errors.Wrapf(err, "failed to GetPatches from %q", kustomizeDir)
+	}
+
+	kustomized, err := km.Kustomize(serialized)
+	if err != nil {
+		return pod, errors.Wrap(err, "failed to kustomize static Pod manifest")
+	}
+
+	// unmarshal kustomized yaml back into a pod manifest
+	obj, err := util.UnmarshalFromYaml(kustomized, v1.SchemeGroupVersion)
+	if err != nil {
+		return pod, errors.Wrap(err, "failed to unmarshal kustomize manifest from YAML")
+	}
+
+	pod2, ok := obj.(*v1.Pod)
+	if !ok {
+		return pod, errors.Wrap(err, "kustomized manifest is not a valid Pod object")
+	}
+
+	return pod2, nil
+}
+
 // WriteStaticPodToDisk writes a static pod file to disk
 func WriteStaticPodToDisk(componentName, manifestDir string, pod v1.Pod) error {
 

cmd/kubeadm/app/util/staticpod/utils_test.go

@@ -25,7 +25,9 @@ import (
 	"strconv"
 	"testing"
 
-	"k8s.io/api/core/v1"
+	"github.com/lithammer/dedent"
+
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	testutil "k8s.io/kubernetes/cmd/kubeadm/test"
@@ -604,3 +606,44 @@ func TestManifestFilesAreEqual(t *testing.T) {
 		})
 	}
 }
+
+func TestKustomizeStaticPod(t *testing.T) {
+	// Create temp folder for the test case
+	tmpdir := testutil.SetupTempDir(t)
+	defer os.RemoveAll(tmpdir)
+
+	patchString := dedent.Dedent(`
+    apiVersion: v1
+    kind: Pod
+    metadata:
+        name: kube-apiserver
+        namespace: kube-system
+        annotations:
+            kustomize: patch for kube-apiserver
+    `)
+
+	err := ioutil.WriteFile(filepath.Join(tmpdir, "patch.yaml"), []byte(patchString), 0644)
+	if err != nil {
+		t.Fatalf("WriteFile returned unexpected error: %v", err)
+	}
+
+	pod := &v1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "Pod",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "kube-apiserver",
+			Namespace: "kube-system",
+		},
+	}
+
+	kpod, err := KustomizeStaticPod(pod, tmpdir)
+	if err != nil {
+		t.Errorf("KustomizeStaticPod returned unexpected error: %v", err)
+	}
+
+	if _, ok := kpod.ObjectMeta.Annotations["kustomize"]; !ok {
+		t.Error("Kustomize did not apply patches corresponding to the resource")
+	}
+}

go.mod

@@ -168,6 +168,7 @@ require (
 	k8s.io/repo-infra v0.0.0-20181204233714-00fe14e3d1a3
 	k8s.io/sample-apiserver v0.0.0
 	k8s.io/utils v0.0.0-20190801114015-581e00157fb1
+	sigs.k8s.io/kustomize v2.0.3+incompatible
 	sigs.k8s.io/yaml v1.1.0
 	vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc
 )