Merge pull request #77613 from mikedanese/fixinclusterconfig

BoundServiceAccountTokenVolume: fix InClusterConfig

staging/src/k8s.io/apiserver/pkg/util/webhook/authentication.go

@@ -171,6 +171,7 @@ func restConfigFromKubeconfig(configAuthInfo *clientcmdapi.AuthInfo) (*rest.Conf
 	// blindly overwrite existing values based on precedence
 	if len(configAuthInfo.Token) > 0 {
 		config.BearerToken = configAuthInfo.Token
+		config.BearerTokenFile = configAuthInfo.TokenFile
 	} else if len(configAuthInfo.TokenFile) > 0 {
 		tokenBytes, err := ioutil.ReadFile(configAuthInfo.TokenFile)
 		if err != nil {

staging/src/k8s.io/client-go/rest/transport.go

@@ -74,9 +74,10 @@ func (c *Config) TransportConfig() (*transport.Config, error) {
 			KeyFile:    c.KeyFile,
 			KeyData:    c.KeyData,
 		},
-		Username:    c.Username,
-		Password:    c.Password,
-		BearerToken: c.BearerToken,
+		Username:        c.Username,
+		Password:        c.Password,
+		BearerToken:     c.BearerToken,
+		BearerTokenFile: c.BearerTokenFile,
 		Impersonate: transport.ImpersonationConfig{
 			UserName: c.Impersonate.UserName,
 			Groups:   c.Impersonate.Groups,

staging/src/k8s.io/client-go/tools/clientcmd/client_config.go

@@ -228,6 +228,7 @@ func (config *DirectClientConfig) getUserIdentificationPartialConfig(configAuthI
 	// blindly overwrite existing values based on precedence
 	if len(configAuthInfo.Token) > 0 {
 		mergedConfig.BearerToken = configAuthInfo.Token
+		mergedConfig.BearerTokenFile = configAuthInfo.TokenFile
 	} else if len(configAuthInfo.TokenFile) > 0 {
 		tokenBytes, err := ioutil.ReadFile(configAuthInfo.TokenFile)
 		if err != nil {
@@ -491,6 +492,7 @@ func (config *inClusterClientConfig) ClientConfig() (*restclient.Config, error)
 		}
 		if token := config.overrides.AuthInfo.Token; len(token) > 0 {
 			icc.BearerToken = token
+			icc.BearerTokenFile = ""
 		}
 		if certificateAuthorityFile := config.overrides.ClusterInfo.CertificateAuthority; len(certificateAuthorityFile) > 0 {
 			icc.TLSClientConfig.CAFile = certificateAuthorityFile

test/e2e/auth/service_accounts.go

@@ -19,6 +19,8 @@ package auth
 import (
 	"fmt"
 	"path"
+	"regexp"
+	"strings"
 	"time"
 
 	authenticationv1 "k8s.io/api/authentication/v1"
@@ -38,6 +40,7 @@ import (
 )
 
 var mountImage = imageutils.GetE2EImage(imageutils.Mounttest)
+var inClusterClientImage = imageutils.GetE2EImage(imageutils.InClusterClient)
 
 var _ = SIGDescribe("ServiceAccounts", func() {
 	f := framework.NewDefaultFramework("svcaccounts")
@@ -410,4 +413,138 @@ var _ = SIGDescribe("ServiceAccounts", func() {
 			}
 		}
 	})
+
+	ginkgo.It("should support InClusterConfig with token rotation [Slow] [Feature:TokenRequestProjection]", func() {
+		cfg, err := framework.LoadConfig()
+		framework.ExpectNoError(err)
+
+		if _, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(&v1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "kube-root-ca.crt",
+			},
+			Data: map[string]string{
+				"ca.crt": string(cfg.TLSClientConfig.CAData),
+			},
+		}); err != nil && !apierrors.IsAlreadyExists(err) {
+			framework.Failf("Unexpected err creating kube-ca-crt: %v", err)
+		}
+
+		tenMin := int64(10 * 60)
+		pod := &v1.Pod{
+			ObjectMeta: metav1.ObjectMeta{Name: "inclusterclient"},
+			Spec: v1.PodSpec{
+				Containers: []v1.Container{{
+					Name:  "inclusterclient",
+					Image: inClusterClientImage,
+					VolumeMounts: []v1.VolumeMount{{
+						MountPath: "/var/run/secrets/kubernetes.io/serviceaccount",
+						Name:      "kube-api-access-e2e",
+						ReadOnly:  true,
+					}},
+				}},
+				RestartPolicy:      v1.RestartPolicyNever,
+				ServiceAccountName: "default",
+				Volumes: []v1.Volume{{
+					Name: "kube-api-access-e2e",
+					VolumeSource: v1.VolumeSource{
+						Projected: &v1.ProjectedVolumeSource{
+							Sources: []v1.VolumeProjection{
+								{
+									ServiceAccountToken: &v1.ServiceAccountTokenProjection{
+										Path:              "token",
+										ExpirationSeconds: &tenMin,
+									},
+								},
+								{
+									ConfigMap: &v1.ConfigMapProjection{
+										LocalObjectReference: v1.LocalObjectReference{
+											Name: "kube-root-ca.crt",
+										},
+										Items: []v1.KeyToPath{
+											{
+												Key:  "ca.crt",
+												Path: "ca.crt",
+											},
+										},
+									},
+								},
+								{
+									DownwardAPI: &v1.DownwardAPIProjection{
+										Items: []v1.DownwardAPIVolumeFile{
+											{
+												Path: "namespace",
+												FieldRef: &v1.ObjectFieldSelector{
+													APIVersion: "v1",
+													FieldPath:  "metadata.namespace",
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				}},
+			},
+		}
+		pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod)
+		framework.ExpectNoError(err)
+
+		framework.Logf("created pod")
+		if !framework.CheckPodsRunningReady(f.ClientSet, f.Namespace.Name, []string{pod.Name}, time.Minute) {
+			framework.Failf("pod %q in ns %q never became ready", pod.Name, f.Namespace.Name)
+		}
+
+		framework.Logf("pod is ready")
+
+		var logs string
+		if err := wait.Poll(1*time.Minute, 20*time.Minute, func() (done bool, err error) {
+			framework.Logf("polling logs")
+			logs, err = framework.GetPodLogs(f.ClientSet, f.Namespace.Name, "inclusterclient", "inclusterclient")
+			if err != nil {
+				framework.Logf("Error pulling logs: %v", err)
+				return false, nil
+			}
+			tokenCount, err := parseInClusterClientLogs(logs)
+			if err != nil {
+				return false, fmt.Errorf("inclusterclient reported an error: %v", err)
+			}
+			if tokenCount < 2 {
+				framework.Logf("Retrying. Still waiting to see more unique tokens: got=%d, want=2", tokenCount)
+				return false, nil
+			}
+			return true, nil
+		}); err != nil {
+			framework.Failf("Unexpected error: %v\n%s", err, logs)
+		}
+	})
 })
+
+var reportLogsParser = regexp.MustCompile("([a-zA-Z0-9-_]*)=([a-zA-Z0-9-_]*)$")
+
+func parseInClusterClientLogs(logs string) (int, error) {
+	seenTokens := map[string]struct{}{}
+
+	lines := strings.Split(logs, "\n")
+	for _, line := range lines {
+		parts := reportLogsParser.FindStringSubmatch(line)
+		if len(parts) != 3 {
+			continue
+		}
+
+		key, value := parts[1], parts[2]
+		switch key {
+		case "authz_header":
+			if value == "<empty>" {
+				return 0, fmt.Errorf("saw empty Authorization header")
+			}
+			seenTokens[value] = struct{}{}
+		case "status":
+			if value == "failed" {
+				return 0, fmt.Errorf("saw status=failed")
+			}
+		}
+	}
+
+	return len(seenTokens), nil
+}

test/images/BUILD

@@ -18,6 +18,7 @@ filegroup(
         "//test/images/echoserver:all-srcs",
         "//test/images/entrypoint-tester:all-srcs",
         "//test/images/fakegitserver:all-srcs",
+        "//test/images/inclusterclient:all-srcs",
         "//test/images/liveness:all-srcs",
         "//test/images/logs-generator:all-srcs",
         "//test/images/metadata-concealment:all-srcs",

test/images/inclusterclient/BUILD

@@ -0,0 +1,34 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "k8s.io/kubernetes/test/images/inclusterclient",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/client-go/rest:go_default_library",
+        "//staging/src/k8s.io/component-base/logs:go_default_library",
+        "//vendor/k8s.io/klog:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "inclusterconfig",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

test/images/inclusterclient/Dockerfile

@@ -0,0 +1,18 @@
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcr.io/distroless/static:latest
+
+ADD inclusterclient /inclusterclient
+ENTRYPOINT ["/inclusterclient"]

test/images/inclusterclient/Makefile

@@ -0,0 +1,25 @@
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SRCS = inclusterclient
+ARCH ?= amd64
+TARGET ?= $(CURDIR)
+GOLANG_VERSION ?= latest
+SRC_DIR = $(notdir $(shell pwd))
+export
+
+bin:
+	../image-util.sh bin $(SRCS)
+
+.PHONY: bin

test/images/inclusterclient/VERSION

@@ -0,0 +1 @@
+1.0

test/images/inclusterclient/main.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"time"
+
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/component-base/logs"
+	"k8s.io/klog"
+)
+
+func main() {
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	pollInterval := flag.Int("poll-interval", 30, "poll interval of call to /healhtz in seconds")
+	flag.Set("logtostderr", "true")
+	flag.Parse()
+
+	klog.Infof("started")
+
+	cfg, err := rest.InClusterConfig()
+	if err != nil {
+		log.Fatalf("err: %v", err)
+	}
+
+	cfg.Wrap(func(rt http.RoundTripper) http.RoundTripper {
+		return &debugRt{
+			rt: rt,
+		}
+	})
+
+	c := kubernetes.NewForConfigOrDie(cfg).RESTClient()
+
+	t := time.Tick(time.Duration(*pollInterval) * time.Second)
+	for {
+		<-t
+		klog.Infof("calling /healthz")
+		b, err := c.Get().AbsPath("/healthz").Do().Raw()
+		if err != nil {
+			klog.Errorf("status=failed")
+			klog.Errorf("error checking /healthz: %v\n%s\n", err, string(b))
+		}
+	}
+}
+
+type debugRt struct {
+	rt http.RoundTripper
+}
+
+func (rt *debugRt) RoundTrip(req *http.Request) (*http.Response, error) {
+	authHeader := req.Header.Get("Authorization")
+	if len(authHeader) != 0 {
+		authHash := sha256.Sum256([]byte(fmt.Sprintf("%s|%s", "salt", authHeader)))
+		klog.Infof("authz_header=%s", base64.RawURLEncoding.EncodeToString(authHash[:]))
+	} else {
+		klog.Errorf("authz_header=<empty>")
+	}
+	return rt.rt.RoundTrip(req)
+}
+
+func (rt *debugRt) WrappedRoundTripper() http.RoundTripper { return rt.rt }

test/utils/image/manifest.go

@@ -130,6 +130,8 @@ const (
 	GBRedisSlave
 	// Hostexec image
 	Hostexec
+	// InClusterClient image
+	InClusterClient
 	// IpcUtils image
 	IpcUtils
 	// Iperf image
@@ -211,6 +213,7 @@ func initImageConfigs() map[int]Config {
 	configs[GBFrontend] = Config{sampleRegistry, "gb-frontend", "v6"}
 	configs[GBRedisSlave] = Config{sampleRegistry, "gb-redisslave", "v3"}
 	configs[Hostexec] = Config{e2eRegistry, "hostexec", "1.1"}
+	configs[InClusterClient] = Config{e2eRegistry, "inclusterclient", "1.0"}
 	configs[IpcUtils] = Config{e2eRegistry, "ipc-utils", "1.0"}
 	configs[Iperf] = Config{e2eRegistry, "iperf", "1.0"}
 	configs[JessieDnsutils] = Config{e2eRegistry, "jessie-dnsutils", "1.0"}