Merge pull request #42075 from deads2k/agg-22-auth-auto

Automatic merge from submit-queue (batch tested with PRs 41597, 42185, 42075, 42178, 41705)

auto discovery CA for extension API servers

This is what the smaller pulls were leading to.  Only the last commit is unique and I expect I'll still tweak some pod definitions, but this is where I was going.

@sttts @liggitt

hack/local-up-cluster.sh

@@ -466,7 +466,7 @@ function start_apiserver {
     # this uses the API port because if you don't have any authenticator, you can't seem to use the secure port at all.
     # this matches what happened with the combination in 1.4.
     # TODO change this conditionally based on whether API_PORT is on or off
-    kube::util::wait_for_url "http://${API_HOST_IP}:${API_PORT}/version" "apiserver: " 1 ${WAIT_FOR_URL_API_SERVER} \
+    kube::util::wait_for_url "http://${API_HOST_IP}:${API_SECURE_PORT}/healthz" "apiserver: " 1 ${WAIT_FOR_URL_API_SERVER} \
         || { echo "check apiserver logs: ${APISERVER_LOG}" ; exit 1 ; }
 
     # Create kubeconfigs for all components, using client certs

staging/src/k8s.io/apiserver/pkg/server/options/authentication.go

@@ -17,14 +17,19 @@ limitations under the License.
 package options
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"time"
 
+	"github.com/golang/glog"
 	"github.com/spf13/pflag"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
 	"k8s.io/apiserver/pkg/server"
 	authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1beta1"
+	coreclient "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 )
@@ -98,6 +103,8 @@ type DelegatingAuthenticationOptions struct {
 
 	ClientCert    ClientCertAuthenticationOptions
 	RequestHeader RequestHeaderAuthenticationOptions
+
+	SkipInClusterLookup bool
 }
 
 func NewDelegatingAuthenticationOptions() *DelegatingAuthenticationOptions {
@@ -128,15 +135,28 @@ func (s *DelegatingAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
 
 	s.ClientCert.AddFlags(fs)
 	s.RequestHeader.AddFlags(fs)
+
+	fs.BoolVar(&s.SkipInClusterLookup, "authentication-skip-lookup", s.SkipInClusterLookup, ""+
+		"If false, the authentication-kubeconfig will be used to lookup missing authentication "+
+		"configuration from the cluster.")
+
 }
 
 func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.Config) error {
-	var err error
-	c, err = c.ApplyClientCert(s.ClientCert.ClientCA)
+	clientCA, err := s.getClientCA()
+	if err != nil {
+		return err
+	}
+	c, err = c.ApplyClientCert(clientCA.ClientCA)
 	if err != nil {
 		return fmt.Errorf("unable to load client CA file: %v", err)
 	}
-	c, err = c.ApplyClientCert(s.RequestHeader.ClientCAFile)
+
+	requestHeader, err := s.getRequestHeader()
+	if err != nil {
+		return err
+	}
+	c, err = c.ApplyClientCert(requestHeader.ClientCAFile)
 	if err != nil {
 		return fmt.Errorf("unable to load client CA file: %v", err)
 	}
@@ -165,17 +185,162 @@ func (s *DelegatingAuthenticationOptions) ToAuthenticationConfig() (authenticato
 		return authenticatorfactory.DelegatingAuthenticatorConfig{}, err
 	}
 
+	clientCA, err := s.getClientCA()
+	if err != nil {
+		return authenticatorfactory.DelegatingAuthenticatorConfig{}, err
+	}
+	requestHeader, err := s.getRequestHeader()
+	if err != nil {
+		return authenticatorfactory.DelegatingAuthenticatorConfig{}, err
+	}
+
 	ret := authenticatorfactory.DelegatingAuthenticatorConfig{
 		Anonymous:               true,
 		TokenAccessReviewClient: tokenClient,
 		CacheTTL:                s.CacheTTL,
-		ClientCAFile:            s.ClientCert.ClientCA,
-		RequestHeaderConfig:     s.RequestHeader.ToAuthenticationRequestHeaderConfig(),
+		ClientCAFile:            clientCA.ClientCA,
+		RequestHeaderConfig:     requestHeader.ToAuthenticationRequestHeaderConfig(),
 	}
 	return ret, nil
 }
 
-func (s *DelegatingAuthenticationOptions) newTokenAccessReview() (authenticationclient.TokenReviewInterface, error) {
+const (
+	authenticationConfigMapNamespace = metav1.NamespaceSystem
+	authenticationConfigMapName      = "extension-apiserver-authentication"
+	authenticationRoleName           = "extension-apiserver-authentication-reader"
+)
+
+func (s *DelegatingAuthenticationOptions) getClientCA() (*ClientCertAuthenticationOptions, error) {
+	if len(s.ClientCert.ClientCA) > 0 || s.SkipInClusterLookup {
+		return &s.ClientCert, nil
+	}
+
+	incluster, err := s.lookupInClusterClientCA()
+	if err != nil {
+		glog.Warningf("Unable to get configmap/%s in %s.  Usually fixed by "+
+			"'kubectl create rolebinding -n %s ROLE_NAME --role=%s --serviceaccount=YOUR_NS:YOUR_SA'",
+			authenticationConfigMapName, authenticationConfigMapNamespace, authenticationConfigMapNamespace, authenticationRoleName)
+		return nil, err
+	}
+	if incluster == nil {
+		return nil, fmt.Errorf("cluster doesn't provide client-ca-file")
+	}
+	return incluster, nil
+}
+
+func (s *DelegatingAuthenticationOptions) getRequestHeader() (*RequestHeaderAuthenticationOptions, error) {
+	if len(s.RequestHeader.ClientCAFile) > 0 || s.SkipInClusterLookup {
+		return &s.RequestHeader, nil
+	}
+
+	incluster, err := s.lookupInClusterRequestHeader()
+	if err != nil {
+		glog.Warningf("Unable to get configmap/%s in %s.  Usually fixed by "+
+			"'kubectl create rolebinding -n %s ROLE_NAME --role=%s --serviceaccount=YOUR_NS:YOUR_SA'",
+			authenticationConfigMapName, authenticationConfigMapNamespace, authenticationConfigMapNamespace, authenticationRoleName)
+		return nil, err
+	}
+	if incluster == nil {
+		return nil, fmt.Errorf("cluster doesn't provide requestheader-client-ca-file")
+	}
+	return incluster, nil
+}
+
+func (s *DelegatingAuthenticationOptions) lookupInClusterClientCA() (*ClientCertAuthenticationOptions, error) {
+	clientConfig, err := s.getClientConfig()
+	if err != nil {
+		return nil, err
+	}
+	client, err := coreclient.NewForConfig(clientConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	authConfigMap, err := client.ConfigMaps(authenticationConfigMapNamespace).Get(authenticationConfigMapName, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	clientCA, ok := authConfigMap.Data["client-ca-file"]
+	if !ok {
+		return nil, nil
+	}
+
+	f, err := ioutil.TempFile("", "client-ca-file")
+	if err != nil {
+		return nil, err
+	}
+	if err := ioutil.WriteFile(f.Name(), []byte(clientCA), 0600); err != nil {
+		return nil, err
+	}
+	return &ClientCertAuthenticationOptions{ClientCA: f.Name()}, nil
+}
+
+func (s *DelegatingAuthenticationOptions) lookupInClusterRequestHeader() (*RequestHeaderAuthenticationOptions, error) {
+	clientConfig, err := s.getClientConfig()
+	if err != nil {
+		return nil, err
+	}
+	client, err := coreclient.NewForConfig(clientConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	authConfigMap, err := client.ConfigMaps(authenticationConfigMapNamespace).Get(authenticationConfigMapName, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	requestHeaderCA, ok := authConfigMap.Data["requestheader-client-ca-file"]
+	if !ok {
+		return nil, nil
+	}
+
+	f, err := ioutil.TempFile("", "requestheader-client-ca-file")
+	if err != nil {
+		return nil, err
+	}
+	if err := ioutil.WriteFile(f.Name(), []byte(requestHeaderCA), 0600); err != nil {
+		return nil, err
+	}
+	usernameHeaders, err := deserializeStrings(authConfigMap.Data["requestheader-username-headers"])
+	if err != nil {
+		return nil, err
+	}
+	groupHeaders, err := deserializeStrings(authConfigMap.Data["requestheader-group-headers"])
+	if err != nil {
+		return nil, err
+	}
+	extraHeaderPrefixes, err := deserializeStrings(authConfigMap.Data["requestheader-extra-headers-prefix"])
+	if err != nil {
+		return nil, err
+	}
+	allowedNames, err := deserializeStrings(authConfigMap.Data["requestheader-allowed-names"])
+	if err != nil {
+		return nil, err
+	}
+
+	return &RequestHeaderAuthenticationOptions{
+		UsernameHeaders:     usernameHeaders,
+		GroupHeaders:        groupHeaders,
+		ExtraHeaderPrefixes: extraHeaderPrefixes,
+		ClientCAFile:        f.Name(),
+		AllowedNames:        allowedNames,
+	}, nil
+}
+
+func deserializeStrings(in string) ([]string, error) {
+	if len(in) == 0 {
+		return nil, nil
+	}
+	var ret []string
+	if err := json.Unmarshal([]byte(in), &ret); err != nil {
+		return nil, err
+	}
+	return ret, nil
+}
+
+func (s *DelegatingAuthenticationOptions) getClientConfig() (*rest.Config, error) {
 	var clientConfig *rest.Config
 	var err error
 	if len(s.RemoteKubeConfigFile) > 0 {
@@ -197,6 +362,14 @@ func (s *DelegatingAuthenticationOptions) newTokenAccessReview() (authentication
 	clientConfig.QPS = 200
 	clientConfig.Burst = 400
 
+	return clientConfig, nil
+}
+
+func (s *DelegatingAuthenticationOptions) newTokenAccessReview() (authenticationclient.TokenReviewInterface, error) {
+	clientConfig, err := s.getClientConfig()
+	if err != nil {
+		return nil, err
+	}
 	client, err := authenticationclient.NewForConfig(clientConfig)
 	if err != nil {
 		return nil, err

staging/src/k8s.io/kube-aggregator/artifacts/self-contained/kubernetes-discover-pod.yaml

@@ -41,11 +41,6 @@ spec:
         - "--tls-cert-file=/var/run/serving-cert/tls.crt"
         - "--tls-private-key-file=/var/run/serving-cert/tls.key"
         - "--tls-ca-file=/var/run/serving-ca/ca.crt"
-        - "--client-ca-file=/var/run/client-ca/ca.crt"
-        - "--requestheader-username-headers=X-Remote-User"
-        - "--requestheader-group-headers=X-Remote-Group"
-        - "--requestheader-extra-headers-prefix=X-Remote-Extra-"
-        - "--requestheader-client-ca-file=/var/run/request-header-ca/ca.crt"
         - "--etcd-servers=https://etcd.kube-public.svc:4001"
         - "--etcd-certfile=/var/run/etcd-client-cert/tls.crt"
         - "--etcd-keyfile=/var/run/etcd-client-cert/tls.key"

staging/src/k8s.io/kube-aggregator/hack/local-up-kube-aggregator.sh

@@ -60,16 +60,18 @@ function start_kube-aggregator {
 	kubectl delete clusterrolebinding kube-aggregator:system:kube-aggregator > /dev/null 2>&1 || true
 	kubectl create clusterrolebinding kube-aggregator:system:auth-delegator --clusterrole=system:auth-delegator --serviceaccount=kube-public:kube-aggregator
 	kubectl create clusterrolebinding kube-aggregator:system:kube-aggregator --clusterrole=system:kube-aggregator --serviceaccount=kube-public:kube-aggregator
+	kubectl delete rolebinding kube-aggregator:authentication-reader > /dev/null 2>&1 || true
+	kubectl create rolebinding -n kube-system kube-aggregator:authentication-reader --role=extension-apiserver-authentication-reader --serviceaccount=kube-public:kube-aggregator
 
 	# make sure the resources we're about to create don't exist
 	kubectl -n kube-public delete secret auth-proxy-client serving-etcd serving-kube-aggregator kube-aggregator-etcd > /dev/null 2>&1 || true
 	kubectl -n kube-public delete configmap etcd-ca kube-aggregator-ca client-ca request-header-ca > /dev/null 2>&1 || true
 	kubectl -n kube-public delete -f "${AGG_ROOT}/artifacts/self-contained" > /dev/null 2>&1 || true
 
-	kubectl -n kube-public create secret tls auth-proxy-client --cert="${FRONT_PROXY_CLIENT_CERT}" --key="${FRONT_PROXY_CLIENT_KEY}"
-	kubectl -n kube-public create secret tls serving-etcd --cert="${AGGREGATOR_CERT_DIR}/serving-etcd.crt" --key="${AGGREGATOR_CERT_DIR}/serving-etcd.key"
-	kubectl -n kube-public create secret tls serving-kube-aggregator --cert="${SERVING_CERT}" --key="${SERVING_KEY}"
-	kubectl -n kube-public create secret tls kube-aggregator-etcd --cert="${AGGREGATOR_CERT_DIR}/client-kube-aggregator-etcd.crt" --key="${AGGREGATOR_CERT_DIR}/client-kube-aggregator-etcd.key"
+	${sudo} $(which kubectl) -n kube-public create secret tls auth-proxy-client --cert="${FRONT_PROXY_CLIENT_CERT}" --key="${FRONT_PROXY_CLIENT_KEY}"
+	${sudo} $(which kubectl)  -n kube-public create secret tls serving-etcd --cert="${AGGREGATOR_CERT_DIR}/serving-etcd.crt" --key="${AGGREGATOR_CERT_DIR}/serving-etcd.key"
+	${sudo} $(which kubectl)  -n kube-public create secret tls serving-kube-aggregator --cert="${SERVING_CERT}" --key="${SERVING_KEY}"
+	${sudo} $(which kubectl)  -n kube-public create secret tls kube-aggregator-etcd --cert="${AGGREGATOR_CERT_DIR}/client-kube-aggregator-etcd.crt" --key="${AGGREGATOR_CERT_DIR}/client-kube-aggregator-etcd.key"
 	kubectl -n kube-public create configmap etcd-ca --from-file="ca.crt=${AGGREGATOR_CERT_DIR}/etcd-ca.crt" || true
 	kubectl -n kube-public create configmap kube-aggregator-ca --from-file="ca.crt=${SERVING_CERT_CA_CERT}" || true
 	kubectl -n kube-public create configmap client-ca --from-file="ca.crt=${CLIENT_CERT_CA_CERT}" || true

staging/src/k8s.io/sample-apiserver/artifacts/example/apiservice.yaml

@@ -0,0 +1,12 @@
+apiVersion: apiregistration.k8s.io/v1alpha1
+kind: APIService
+metadata:
+  name: v1alpha1.wardle.k8s.io
+spec:
+  insecureSkipTLSVerify: true
+  group: wardle.k8s.io
+  priority: 200
+  service:
+    name: api
+    namespace: wardle
+  version: v1alpha1

staging/src/k8s.io/sample-apiserver/artifacts/example/auth-delegator.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1alpha1
+kind: ClusterRoleBinding
+metadata:
+  name: wardle:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- apiVersion: v1
+  kind: ServiceAccount
+  name: apiserver
+  namespace: wardle

staging/src/k8s.io/sample-apiserver/artifacts/example/auth-reader.yaml

@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1alpha1
+kind: RoleBinding
+metadata:
+  name: wardle-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- apiVersion: v1
+  kind: ServiceAccount
+  name: apiserver
+  namespace: wardle

staging/src/k8s.io/sample-apiserver/artifacts/example/example.yaml

@@ -1,7 +0,0 @@
-apiVersion: serviceinjection.k8s.io/v1alpha1
-kind: ServiceInjection
-metadata:
-  name: injector
-  namespace: kube-system
-  labels:
-    sample-label: foo

staging/src/k8s.io/sample-apiserver/artifacts/example/rc.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: wardle-server
+  namespace: wardle
+  labels:
+    apiserver: "true"
+spec:
+  replicas: 1
+  selector:
+    apiserver: "true"
+  template:
+    metadata:
+      labels:
+        apiserver: "true"
+    spec:
+      serviceAccountName: apiserver
+      containers:
+      - name: wardle-server
+        image: kube-sample-apiserver:latest
+        imagePullPolicy: Never
+        args:
+        - "--etcd-servers=http://localhost:2379"
+      - name: etcd
+        image: quay.io/coreos/etcd:v3.0.17

staging/src/k8s.io/sample-apiserver/artifacts/example/sa.yaml

@@ -0,0 +1,5 @@
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: apiserver
+  namespace: wardle

staging/src/k8s.io/sample-apiserver/artifacts/example/service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: api
+  namespace: wardle
+spec:
+  ports:
+  - port: 443
+    protocol: TCP
+    targetPort: 443
+  selector:
+    apiserver: "true"

staging/src/k8s.io/sample-apiserver/artifacts/flunders/01-flunder.yaml

@@ -0,0 +1,6 @@
+apiVersion: wardle.k8s.io/v1alpha1
+kind: Flunder
+metadata:
+  name: my-first-flunder
+  labels:
+    sample-label: "true"

staging/src/k8s.io/sample-apiserver/artifacts/simple-image/Dockerfile

@@ -13,5 +13,5 @@
 # limitations under the License.
 
 FROM fedora
-ADD kube-service-injection /
-ENTRYPOINT ["/kube-service-injection"]
+ADD kube-sample-apiserver /
+ENTRYPOINT ["/kube-sample-apiserver"]

staging/src/k8s.io/sample-apiserver/artifacts/static/static-pod.yaml

@@ -1,33 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: kube-service-injection
-  namespace: kube-system
-spec:
-  hostNetwork: true
-  containers:
-  - name: kube-service-injection
-    image: kube-service-injection
-    imagePullPolicy: Never
-    args:
-    - "--secure-port=9443"
-    - "--authentication-kubeconfig=/all-certs/admin.kubeconfig"
-    - "--authorization-kubeconfig=/all-certs/admin.kubeconfig"
-    - "--tls-ca-file=/all-certs/apiserver.crt"
-    - "--client-ca-file=/all-certs/client-ca.crt"
-    - "--requestheader-username-headers=X-Remote-User"
-    - "--requestheader-group-headers=X-Remote-Group"
-    - "--requestheader-extra-headers-prefix=X-Remote-Extra-"
-    - "--requestheader-client-ca-file=/all-certs/request-header-ca.crt"
-    - "--etcd-servers=http://127.0.0.1:2379"
-    ports:
-    - containerPort: 9443
-      hostPort: 9443
-    volumeMounts:
-    - name: all-certs
-      mountPath: /all-certs
-      readOnly: true
-  volumes:
-  - name: all-certs
-    hostPath:
-      path: /var/run/kubernetes/

staging/src/k8s.io/sample-apiserver/hack/build-image.sh

@@ -15,14 +15,16 @@
 # limitations under the License.
 
 
-KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../../..
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../../../../..
 source "${KUBE_ROOT}/hack/lib/util.sh"
 
 # Register function to be called on EXIT to remove generated binary.
 function cleanup {
-  rm "${KUBE_ROOT}/cmd/kube-sample-apiserver/artifacts/simple-image/kube-sample-apiserver"
+  rm "${KUBE_ROOT}/vendor/k8s.io/sample-apiserver/artifacts/simple-image/kube-sample-apiserver"
 }
 trap cleanup EXIT
 
-cp -v ${KUBE_ROOT}/_output/local/bin/linux/amd64/kube-sample-apiserver "${KUBE_ROOT}/cmd/kube-sample-apiserver/artifacts/simple-image/kube-sample-apiserver"
-docker build -t kube-sample-apiserver:latest ${KUBE_ROOT}/cmd/kube-sample-apiserver/artifacts/simple-image
+pushd "${KUBE_ROOT}/vendor/k8s.io/sample-apiserver"
+cp -v ../../../../_output/local/bin/linux/amd64/sample-apiserver ./artifacts/simple-image/kube-sample-apiserver
+docker build -t kube-sample-apiserver:latest ./artifacts/simple-image
+popd

test/integration/examples/apiserver_test.go

@@ -81,7 +81,6 @@ func TestAggregatedAPIServer(t *testing.T) {
 	defer os.RemoveAll(certDir)
 	_, defaultServiceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24")
 	proxySigningKey, err := cert.NewPrivateKey()
-
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -93,6 +92,18 @@ func TestAggregatedAPIServer(t *testing.T) {
 	if err := ioutil.WriteFile(proxyCACertFile.Name(), cert.EncodeCertPEM(proxySigningCert), 0644); err != nil {
 		t.Fatal(err)
 	}
+	clientSigningKey, err := cert.NewPrivateKey()
+	if err != nil {
+		t.Fatal(err)
+	}
+	clientSigningCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "client-ca"}, clientSigningKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	clientCACertFile, _ := ioutil.TempFile(certDir, "client-ca.crt")
+	if err := ioutil.WriteFile(clientCACertFile.Name(), cert.EncodeCertPEM(clientSigningCert), 0644); err != nil {
+		t.Fatal(err)
+	}
 
 	kubeAPIServerOptions := options.NewServerRunOptions()
 	kubeAPIServerOptions.SecureServing.ServingOptions.BindAddress = net.ParseIP("127.0.0.1")
@@ -106,6 +117,7 @@ func TestAggregatedAPIServer(t *testing.T) {
 	kubeAPIServerOptions.Authentication.RequestHeader.ExtraHeaderPrefixes = []string{"X-Remote-Extra-"}
 	kubeAPIServerOptions.Authentication.RequestHeader.AllowedNames = []string{"kube-aggregator"}
 	kubeAPIServerOptions.Authentication.RequestHeader.ClientCAFile = proxyCACertFile.Name()
+	kubeAPIServerOptions.Authentication.ClientCert.ClientCA = clientCACertFile.Name()
 	kubeAPIServerOptions.Authorization.Mode = "RBAC"
 
 	config, sharedInformers, err := app.BuildMasterConfig(kubeAPIServerOptions)

vendor/BUILD

@@ -10573,6 +10573,7 @@ go_library(
         "//vendor:k8s.io/apiserver/pkg/util/flag",
         "//vendor:k8s.io/client-go/kubernetes/typed/authentication/v1beta1",
         "//vendor:k8s.io/client-go/kubernetes/typed/authorization/v1beta1",
+        "//vendor:k8s.io/client-go/kubernetes/typed/core/v1",
         "//vendor:k8s.io/client-go/rest",
         "//vendor:k8s.io/client-go/tools/clientcmd",
         "//vendor:k8s.io/client-go/util/cert",