Merge pull request #109995 from wangyysde/fix_issue_2681

add print-manifest flag to print addon manifests to STDOUT
This commit is contained in:
Kubernetes Prow Robot 2022-06-08 09:00:22 -07:00 committed by GitHub
commit 0985c476e3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 209 additions and 167 deletions

View File

@ -145,4 +145,7 @@ const (
// Patches flag sets the folder where kubeadm component patches are stored
Patches = "patches"
// Print the addon manifests to STDOUT instead of installing them.
PrintManifest = "print-manifest"
)

View File

@ -17,7 +17,10 @@ limitations under the License.
package phases
import (
"io"
"github.com/pkg/errors"
"github.com/spf13/pflag"
clientset "k8s.io/client-go/kubernetes"
@ -38,10 +41,18 @@ var (
kubeProxyAddonLongDesc = cmdutil.LongDesc(`
Install the kube-proxy addon components via the API server.
`)
printManifest bool = false
)
// NewAddonPhase returns the addon Cobra command
func NewAddonPhase() workflow.Phase {
dnsLocalFlags := pflag.NewFlagSet(options.PrintManifest, pflag.ContinueOnError)
dnsLocalFlags.BoolVar(&printManifest, options.PrintManifest, printManifest, "Print the addon manifests to STDOUT instead of installing them")
proxyLocalFlags := pflag.NewFlagSet(options.PrintManifest, pflag.ContinueOnError)
proxyLocalFlags.BoolVar(&printManifest, options.PrintManifest, printManifest, "Print the addon manifests to STDOUT instead of installing them")
return workflow.Phase{
Name: "addon",
Short: "Install required addons for passing conformance tests",
@ -59,6 +70,7 @@ func NewAddonPhase() workflow.Phase {
Long: coreDNSAddonLongDesc,
InheritFlags: getAddonPhaseFlags("coredns"),
Run: runCoreDNSAddon,
LocalFlags: dnsLocalFlags,
},
{
Name: "kube-proxy",
@ -66,40 +78,47 @@ func NewAddonPhase() workflow.Phase {
Long: kubeProxyAddonLongDesc,
InheritFlags: getAddonPhaseFlags("kube-proxy"),
Run: runKubeProxyAddon,
LocalFlags: proxyLocalFlags,
},
},
}
}
func getInitData(c workflow.RunData) (*kubeadmapi.InitConfiguration, clientset.Interface, error) {
func getInitData(c workflow.RunData) (*kubeadmapi.InitConfiguration, clientset.Interface, io.Writer, error) {
data, ok := c.(InitData)
if !ok {
return nil, nil, errors.New("addon phase invoked with an invalid data struct")
return nil, nil, nil, errors.New("addon phase invoked with an invalid data struct")
}
cfg := data.Cfg()
client, err := data.Client()
if err != nil {
return nil, nil, err
var client clientset.Interface
var err error
if !printManifest {
client, err = data.Client()
if err != nil {
return nil, nil, nil, err
}
}
return cfg, client, err
out := data.OutputWriter()
return cfg, client, out, err
}
// runCoreDNSAddon installs CoreDNS addon to a Kubernetes cluster
func runCoreDNSAddon(c workflow.RunData) error {
cfg, client, err := getInitData(c)
cfg, client, out, err := getInitData(c)
if err != nil {
return err
}
return dnsaddon.EnsureDNSAddon(&cfg.ClusterConfiguration, client)
return dnsaddon.EnsureDNSAddon(&cfg.ClusterConfiguration, client, out, printManifest)
}
// runKubeProxyAddon installs KubeProxy addon to a Kubernetes cluster
func runKubeProxyAddon(c workflow.RunData) error {
cfg, client, err := getInitData(c)
cfg, client, out, err := getInitData(c)
if err != nil {
return err
}
return proxyaddon.EnsureProxyAddon(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, client)
return proxyaddon.EnsureProxyAddon(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, client, out, printManifest)
}
func getAddonPhaseFlags(name string) []string {

View File

@ -19,6 +19,7 @@ package dns
import (
"context"
"fmt"
"io"
"strings"
"github.com/coredns/corefile-migration/migration"
@ -86,15 +87,22 @@ func deployedDNSReplicas(client clientset.Interface, replicas int32) (*int32, er
}
// EnsureDNSAddon creates the CoreDNS addon
func EnsureDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface) error {
replicas, err := deployedDNSReplicas(client, coreDNSReplicas)
if err != nil {
return err
func EnsureDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, out io.Writer, printManifest bool) error {
var replicas *int32
var err error
if !printManifest {
replicas, err = deployedDNSReplicas(client, coreDNSReplicas)
if err != nil {
return err
}
} else {
var defaultReplicas int32 = coreDNSReplicas
replicas = &defaultReplicas
}
return coreDNSAddon(cfg, client, replicas)
return coreDNSAddon(cfg, client, replicas, out, printManifest)
}
func coreDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, replicas *int32) error {
func coreDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, replicas *int32, out io.Writer, printManifest bool) error {
// Get the YAML manifest
coreDNSDeploymentBytes, err := kubeadmutil.ParseTemplate(CoreDNSDeployment, struct {
DeploymentName, Image, OldControlPlaneTaintKey, ControlPlaneTaintKey string
@ -132,10 +140,26 @@ func coreDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interfa
return errors.Wrap(err, "error when parsing CoreDNS service template")
}
if printManifest {
fmt.Fprint(out, "---")
fmt.Fprintf(out, "%s", coreDNSDeploymentBytes)
fmt.Fprint(out, "---")
fmt.Fprintf(out, "%s", coreDNSConfigMapBytes)
fmt.Fprint(out, "---")
fmt.Fprintf(out, "%s", coreDNSServiceBytes)
fmt.Fprint(out, "---")
fmt.Fprintf(out, "%s", []byte(CoreDNSClusterRole))
fmt.Fprint(out, "---")
fmt.Fprintf(out, "%s", []byte(CoreDNSClusterRoleBinding))
fmt.Fprint(out, "---")
fmt.Fprintf(out, "%s", []byte(CoreDNSServiceAccount))
return nil
}
if err := createCoreDNSAddon(coreDNSDeploymentBytes, coreDNSServiceBytes, coreDNSConfigMapBytes, client); err != nil {
return err
}
fmt.Println("[addons] Applied essential addon: CoreDNS")
fmt.Fprintln(out, "[addons] Applied essential addon: CoreDNS")
return nil
}

View File

@ -19,6 +19,7 @@ package proxy
import (
"bytes"
"fmt"
"io"
"github.com/pkg/errors"
@ -27,6 +28,7 @@ import (
rbac "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kuberuntime "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
clientset "k8s.io/client-go/kubernetes"
clientsetscheme "k8s.io/client-go/kubernetes/scheme"
@ -47,58 +49,160 @@ const (
)
// EnsureProxyAddon creates the kube-proxy addons
func EnsureProxyAddon(cfg *kubeadmapi.ClusterConfiguration, localEndpoint *kubeadmapi.APIEndpoint, client clientset.Interface) error {
if err := CreateServiceAccount(client); err != nil {
return errors.Wrap(err, "error when creating kube-proxy service account")
}
if err := createKubeProxyConfigMap(cfg, localEndpoint, client); err != nil {
func EnsureProxyAddon(cfg *kubeadmapi.ClusterConfiguration, localEndpoint *kubeadmapi.APIEndpoint, client clientset.Interface, out io.Writer, printManifest bool) error {
cmByte, err := createKubeProxyConfigMap(cfg, localEndpoint, client, printManifest)
if err != nil {
return err
}
if err := createKubeProxyAddon(cfg, client); err != nil {
dsByte, err := createKubeProxyAddon(cfg, client, printManifest)
if err != nil {
return err
}
if err := CreateRBACRules(client); err != nil {
return errors.Wrap(err, "error when creating kube-proxy RBAC rules")
if err := printOrCreateKubeProxyObjects(cmByte, dsByte, client, out, printManifest); err != nil {
return err
}
fmt.Println("[addons] Applied essential addon: kube-proxy")
return nil
}
// CreateServiceAccount creates the necessary serviceaccounts that kubeadm uses/might use, if they don't already exist.
func CreateServiceAccount(client clientset.Interface) error {
// Create SA, RBACRules or print manifests of them to out if printManifest is true
func printOrCreateKubeProxyObjects(cmByte []byte, dsByte []byte, client clientset.Interface, out io.Writer, printManifest bool) error {
var saBytes, crbBytes, roleBytes, roleBindingBytes []byte
var err error
return apiclient.CreateOrUpdateServiceAccount(client, &v1.ServiceAccount{
sa := &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: KubeProxyServiceAccountName,
Namespace: metav1.NamespaceSystem,
},
})
}
crb := &rbac.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: constants.KubeProxyClusterRoleBindingName,
},
RoleRef: rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "ClusterRole",
Name: constants.KubeProxyClusterRoleName,
},
Subjects: []rbac.Subject{
{
Kind: rbac.ServiceAccountKind,
Name: KubeProxyServiceAccountName,
Namespace: metav1.NamespaceSystem,
},
},
}
role := &rbac.Role{
ObjectMeta: metav1.ObjectMeta{
Name: KubeProxyConfigMapRoleName,
Namespace: metav1.NamespaceSystem,
},
Rules: []rbac.PolicyRule{
{
Verbs: []string{"get"},
APIGroups: []string{""},
Resources: []string{"configmaps"},
ResourceNames: []string{constants.KubeProxyConfigMap},
},
},
}
rb := &rbac.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: KubeProxyConfigMapRoleName,
Namespace: metav1.NamespaceSystem,
},
RoleRef: rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "Role",
Name: KubeProxyConfigMapRoleName,
},
Subjects: []rbac.Subject{
{
Kind: rbac.GroupKind,
Name: constants.NodeBootstrapTokenAuthGroup,
},
},
}
// Create the objects if printManifest is false
if !printManifest {
if err := apiclient.CreateOrUpdateServiceAccount(client, sa); err != nil {
return errors.Wrap(err, "error when creating kube-proxy service account")
}
if err := apiclient.CreateOrUpdateClusterRoleBinding(client, crb); err != nil {
return err
}
if err := apiclient.CreateOrUpdateRole(client, role); err != nil {
return err
}
if err := apiclient.CreateOrUpdateRoleBinding(client, rb); err != nil {
return err
}
fmt.Fprintln(out, "[addons] Applied essential addon: kube-proxy")
return nil
}
gv := schema.GroupVersion{Group: "", Version: "v1"}
if saBytes, err = kubeadmutil.MarshalToYaml(sa, gv); err != nil {
return err
}
gv = schema.GroupVersion{Group: "rbac.authorization.k8s.io", Version: "v1"}
if crbBytes, err = kubeadmutil.MarshalToYaml(crb, gv); err != nil {
return err
}
if roleBytes, err = kubeadmutil.MarshalToYaml(role, gv); err != nil {
return err
}
if roleBindingBytes, err = kubeadmutil.MarshalToYaml(rb, gv); err != nil {
return err
}
fmt.Fprintln(out, "---")
fmt.Fprintf(out, "%s", saBytes)
fmt.Fprintln(out, "---")
fmt.Fprintf(out, "%s", crbBytes)
fmt.Fprintln(out, "---")
fmt.Fprintf(out, "%s", roleBytes)
fmt.Fprintln(out, "---")
fmt.Fprintf(out, "%s", roleBindingBytes)
fmt.Fprint(out, "---")
fmt.Fprintf(out, "%s", cmByte)
fmt.Fprint(out, "---")
fmt.Fprintf(out, "%s", dsByte)
return nil
}
// CreateRBACRules creates the essential RBAC rules for a minimally set-up cluster
func CreateRBACRules(client clientset.Interface) error {
return createClusterRoleBindings(client)
}
func createKubeProxyConfigMap(cfg *kubeadmapi.ClusterConfiguration, localEndpoint *kubeadmapi.APIEndpoint, client clientset.Interface) error {
func createKubeProxyConfigMap(cfg *kubeadmapi.ClusterConfiguration, localEndpoint *kubeadmapi.APIEndpoint, client clientset.Interface, printManifest bool) ([]byte, error) {
// Generate ControlPlane Enpoint kubeconfig file
controlPlaneEndpoint, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, localEndpoint)
if err != nil {
return err
return []byte(""), err
}
kubeProxyCfg, ok := cfg.ComponentConfigs[componentconfigs.KubeProxyGroup]
if !ok {
return errors.New("no kube-proxy component config found in the active component config set")
return []byte(""), errors.New("no kube-proxy component config found in the active component config set")
}
proxyBytes, err := kubeProxyCfg.Marshal()
if err != nil {
return errors.Wrap(err, "error when marshaling")
return []byte(""), errors.Wrap(err, "error when marshaling")
}
var prefixBytes bytes.Buffer
apiclient.PrintBytesWithLinePrefix(&prefixBytes, proxyBytes, " ")
@ -115,12 +219,16 @@ func createKubeProxyConfigMap(cfg *kubeadmapi.ClusterConfiguration, localEndpoin
ProxyConfigMapKey: constants.KubeProxyConfigMapKey,
})
if err != nil {
return errors.Wrap(err, "error when parsing kube-proxy configmap template")
return []byte(""), errors.Wrap(err, "error when parsing kube-proxy configmap template")
}
if printManifest {
return configMapBytes, nil
}
kubeproxyConfigMap := &v1.ConfigMap{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), configMapBytes, kubeproxyConfigMap); err != nil {
return errors.Wrap(err, "unable to decode kube-proxy configmap")
return []byte(""), errors.Wrap(err, "unable to decode kube-proxy configmap")
}
if !kubeProxyCfg.IsUserSupplied() {
@ -128,86 +236,31 @@ func createKubeProxyConfigMap(cfg *kubeadmapi.ClusterConfiguration, localEndpoin
}
// Create the ConfigMap for kube-proxy or update it in case it already exists
return apiclient.CreateOrUpdateConfigMap(client, kubeproxyConfigMap)
return []byte(""), apiclient.CreateOrUpdateConfigMap(client, kubeproxyConfigMap)
}
func createKubeProxyAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface) error {
func createKubeProxyAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, printManifest bool) ([]byte, error) {
daemonSetbytes, err := kubeadmutil.ParseTemplate(KubeProxyDaemonSet19, struct{ Image, ProxyConfigMap, ProxyConfigMapKey string }{
Image: images.GetKubernetesImage(constants.KubeProxy, cfg),
ProxyConfigMap: constants.KubeProxyConfigMap,
ProxyConfigMapKey: constants.KubeProxyConfigMapKey,
})
if err != nil {
return errors.Wrap(err, "error when parsing kube-proxy daemonset template")
return []byte(""), errors.Wrap(err, "error when parsing kube-proxy daemonset template")
}
if printManifest {
return daemonSetbytes, nil
}
kubeproxyDaemonSet := &apps.DaemonSet{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), daemonSetbytes, kubeproxyDaemonSet); err != nil {
return errors.Wrap(err, "unable to decode kube-proxy daemonset")
return []byte(""), errors.Wrap(err, "unable to decode kube-proxy daemonset")
}
// Propagate the http/https proxy host environment variables to the container
env := &kubeproxyDaemonSet.Spec.Template.Spec.Containers[0].Env
*env = append(*env, kubeadmutil.GetProxyEnvVars()...)
// Create the DaemonSet for kube-proxy or update it in case it already exists
return apiclient.CreateOrUpdateDaemonSet(client, kubeproxyDaemonSet)
}
func createClusterRoleBindings(client clientset.Interface) error {
if err := apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: constants.KubeProxyClusterRoleBindingName,
},
RoleRef: rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "ClusterRole",
Name: constants.KubeProxyClusterRoleName,
},
Subjects: []rbac.Subject{
{
Kind: rbac.ServiceAccountKind,
Name: KubeProxyServiceAccountName,
Namespace: metav1.NamespaceSystem,
},
},
}); err != nil {
return err
}
// Create a role for granting read only access to the kube-proxy component config ConfigMap
if err := apiclient.CreateOrUpdateRole(client, &rbac.Role{
ObjectMeta: metav1.ObjectMeta{
Name: KubeProxyConfigMapRoleName,
Namespace: metav1.NamespaceSystem,
},
Rules: []rbac.PolicyRule{
{
Verbs: []string{"get"},
APIGroups: []string{""},
Resources: []string{"configmaps"},
ResourceNames: []string{constants.KubeProxyConfigMap},
},
},
}); err != nil {
return err
}
// Bind the role to bootstrap tokens for allowing fetchConfiguration during join
return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: KubeProxyConfigMapRoleName,
Namespace: metav1.NamespaceSystem,
},
RoleRef: rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "Role",
Name: KubeProxyConfigMapRoleName,
},
Subjects: []rbac.Subject{
{
Kind: rbac.GroupKind,
Name: constants.NodeBootstrapTokenAuthGroup,
},
},
})
return []byte(""), apiclient.CreateOrUpdateDaemonSet(client, kubeproxyDaemonSet)
}

View File

@ -17,13 +17,13 @@ limitations under the License.
package proxy
import (
"os"
"strings"
"testing"
apps "k8s.io/api/apps/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
clientsetfake "k8s.io/client-go/kubernetes/fake"
clientsetscheme "k8s.io/client-go/kubernetes/scheme"
core "k8s.io/client-go/testing"
@ -33,63 +33,6 @@ import (
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
)
func TestCreateServiceAccount(t *testing.T) {
tests := []struct {
name string
createErr error
expectErr bool
}{
{
"error-free case",
nil,
false,
},
{
"duplication errors should be ignored",
apierrors.NewAlreadyExists(schema.GroupResource{}, ""),
false,
},
{
"unexpected errors should be returned",
apierrors.NewUnauthorized(""),
true,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
client := clientsetfake.NewSimpleClientset()
if tc.createErr != nil {
client.PrependReactor("create", "serviceaccounts", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, tc.createErr
})
}
err := CreateServiceAccount(client)
if tc.expectErr {
if err == nil {
t.Errorf("CreateServiceAccounts(%s) wanted err, got nil", tc.name)
}
return
} else if !tc.expectErr && err != nil {
t.Errorf("CreateServiceAccounts(%s) returned unexpected err: %v", tc.name, err)
}
wantResourcesCreated := 1
if len(client.Actions()) != wantResourcesCreated {
t.Errorf("CreateServiceAccounts(%s) should have made %d actions, but made %d", tc.name, wantResourcesCreated, len(client.Actions()))
}
for _, action := range client.Actions() {
if action.GetVerb() != "create" || action.GetResource().Resource != "serviceaccounts" {
t.Errorf("CreateServiceAccounts(%s) called [%v %v], but wanted [create serviceaccounts]",
tc.name, action.GetVerb(), action.GetResource().Resource)
}
}
})
}
}
func TestCompileManifests(t *testing.T) {
var tests = []struct {
name string
@ -198,7 +141,7 @@ func TestEnsureProxyAddon(t *testing.T) {
initConfiguration.ClusterConfiguration.Networking.PodSubnet = "2001:101::/48"
}
err = EnsureProxyAddon(&initConfiguration.ClusterConfiguration, &initConfiguration.LocalAPIEndpoint, client)
err = EnsureProxyAddon(&initConfiguration.ClusterConfiguration, &initConfiguration.LocalAPIEndpoint, client, os.Stdout, false)
// Compare actual to expected errors
actErr := "No error"

View File

@ -128,7 +128,7 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitCon
metav1.NamespaceSystem)
} else {
// Upgrade CoreDNS
if err := dns.EnsureDNSAddon(&cfg.ClusterConfiguration, client); err != nil {
if err := dns.EnsureDNSAddon(&cfg.ClusterConfiguration, client, out, false); err != nil {
errs = append(errs, err)
}
}
@ -151,7 +151,7 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitCon
metav1.NamespaceSystem)
} else {
// Upgrade kube-proxy
if err := proxy.EnsureProxyAddon(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, client); err != nil {
if err := proxy.EnsureProxyAddon(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, client, out, false); err != nil {
errs = append(errs, err)
}
}