kubelet: support alpha credential provider exec plugins

Signed-off-by: Andrew Sy Kim <kim.andrewsy@gmail.com>

cmd/kubelet/app/server.go

@@ -1131,6 +1131,8 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie
 		kubeServer.CloudProvider,
 		kubeServer.CertDirectory,
 		kubeServer.RootDirectory,
+		kubeServer.ImageCredentialProviderConfigFile,
+		kubeServer.ImageCredentialProviderBinDir,
 		kubeServer.RegisterNode,
 		kubeServer.RegisterWithTaints,
 		kubeServer.AllowedUnsafeSysctls,
@@ -1205,6 +1207,8 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 	cloudProvider string,
 	certDirectory string,
 	rootDirectory string,
+	imageCredentialProviderConfigFile string,
+	imageCredentialProviderBinDir string,
 	registerNode bool,
 	registerWithTaints []api.Taint,
 	allowedUnsafeSysctls []string,
@@ -1236,6 +1240,8 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		cloudProvider,
 		certDirectory,
 		rootDirectory,
+		imageCredentialProviderConfigFile,
+		imageCredentialProviderBinDir,
 		registerNode,
 		registerWithTaints,
 		allowedUnsafeSysctls,

pkg/kubelet/config/flags.go

@@ -65,6 +65,20 @@ type ContainerRuntimeOptions struct {
 	// CNICacheDir is the full path of the directory in which CNI should store
 	// cache files
 	CNICacheDir string
+
+	// Image credential provider plugin options
+
+	// ImageCredentialProviderConfigFile is the path to the credential provider plugin config file.
+	// This config file is a specification for what credential providers are enabled and invokved
+	// by the kubelet. The plugin config should contain information about what plugin binary
+	// to execute and what container images the plugin should be called for.
+	// +optional
+	ImageCredentialProviderConfigFile string
+	// ImageCredentialProviderBinDir is the path to the directory where credential provider plugin
+	// binaries exist. The name of each plugin binary is expected to match the name of the plugin
+	// specified in imageCredentialProviderConfigFile.
+	// +optional
+	ImageCredentialProviderBinDir string
 }
 
 // AddFlags adds flags to the container runtime, according to ContainerRuntimeOptions.
@@ -90,4 +104,8 @@ func (s *ContainerRuntimeOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.CNIBinDir, "cni-bin-dir", s.CNIBinDir, fmt.Sprintf("A comma-separated list of full paths of directories in which to search for CNI plugin binaries. %s", dockerOnlyWarning))
 	fs.StringVar(&s.CNICacheDir, "cni-cache-dir", s.CNICacheDir, fmt.Sprintf("The full path of the directory in which CNI should store cache files. %s", dockerOnlyWarning))
 	fs.Int32Var(&s.NetworkPluginMTU, "network-plugin-mtu", s.NetworkPluginMTU, fmt.Sprintf("The MTU to be passed to the network plugin, to override the default. Set to 0 to use the default 1460 MTU. %s", dockerOnlyWarning))
+
+	// Image credential provider settings.
+	fs.StringVar(&s.ImageCredentialProviderConfigFile, "image-credential-provider-config", s.ImageCredentialProviderConfigFile, "The path to the credential provider plugin config file.")
+	fs.StringVar(&s.ImageCredentialProviderBinDir, "image-credential-provider-bin-dir", s.ImageCredentialProviderBinDir, "The path to the directory where credential provider plugin binaries are located.")
 }

pkg/kubelet/kubelet.go

@@ -338,6 +338,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 	cloudProvider string,
 	certDirectory string,
 	rootDirectory string,
+	imageCredentialProviderConfigFile string,
+	imageCredentialProviderBinDir string,
 	registerNode bool,
 	registerWithTaints []api.Taint,
 	allowedUnsafeSysctls []string,
@@ -600,6 +602,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		kubeCfg.SerializeImagePulls,
 		float32(kubeCfg.RegistryPullQPS),
 		int(kubeCfg.RegistryBurst),
+		imageCredentialProviderConfigFile,
+		imageCredentialProviderBinDir,
 		kubeCfg.CPUCFSQuota,
 		kubeCfg.CPUCFSQuotaPeriod,
 		kubeDeps.RemoteRuntimeService,

pkg/kubelet/kuberuntime/BUILD

@@ -35,6 +35,7 @@ go_library(
     deps = [
         "//pkg/api/legacyscheme:go_default_library",
         "//pkg/credentialprovider:go_default_library",
+        "//pkg/credentialprovider/plugin:go_default_library",
         "//pkg/credentialprovider/secrets:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/kubelet/cm:go_default_library",

pkg/kubelet/kuberuntime/kuberuntime_manager.go

@@ -40,6 +40,7 @@ import (
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/credentialprovider"
+	"k8s.io/kubernetes/pkg/credentialprovider/plugin"
 	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet/cm"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
@@ -166,6 +167,8 @@ func NewKubeGenericRuntimeManager(
 	serializeImagePulls bool,
 	imagePullQPS float32,
 	imagePullBurst int,
+	imageCredentialProviderConfigFile string,
+	imageCredentialProviderBinDir string,
 	cpuCFSQuota bool,
 	cpuCFSQuotaPeriod metav1.Duration,
 	runtimeService internalapi.RuntimeService,
@@ -187,7 +190,6 @@ func NewKubeGenericRuntimeManager(
 		runtimeHelper:       runtimeHelper,
 		runtimeService:      newInstrumentedRuntimeService(runtimeService),
 		imageService:        newInstrumentedImageManagerService(imageService),
-		keyring:             credentialprovider.NewDockerKeyring(),
 		internalLifecycle:   internalLifecycle,
 		legacyLogProvider:   legacyLogProvider,
 		logManager:          logManager,
@@ -225,6 +227,18 @@ func NewKubeGenericRuntimeManager(
 		}
 	}
 
+	if !utilfeature.DefaultFeatureGate.Enabled(features.KubeletCredentialProviders) && (imageCredentialProviderConfigFile != "" || imageCredentialProviderBinDir != "") {
+		klog.Warningf("Flags --image-credential-provider-config or --image-credential-provider-bin-dir were set but the feature gate %s was disabled, these flags will be ignored",
+			features.KubeletCredentialProviders)
+	}
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.KubeletCredentialProviders) && (imageCredentialProviderConfigFile != "" || imageCredentialProviderBinDir != "") {
+		if err := plugin.RegisterCredentialProviderPlugins(imageCredentialProviderConfigFile, imageCredentialProviderBinDir); err != nil {
+			klog.Fatalf("Failed to register CRI auth plugins: %v", err)
+		}
+	}
+	kubeRuntimeManager.keyring = credentialprovider.NewDockerKeyring()
+
 	kubeRuntimeManager.imagePuller = images.NewImageManager(
 		kubecontainer.FilterEventRecorder(recorder),
 		kubeRuntimeManager,