Merge pull request #40928 from deads2k/agg-01-local-up

Automatic merge from submit-queue

make kube-aggregator run as static pod for local-up-cluster

Runs the kube-aggregator as a static pod for local-up-cluster.  Looks like someone broke kubectl negotiation again, so I'll fix that up separately.

@kubernetes/sig-api-machinery-misc 
@lavalamp you're probably looking to run kube-aggregator as a static pod, here's an example.
@jwforres I'll make a secure variant for wiring up to openshift.

hack/local-up-cluster.sh

@@ -28,7 +28,7 @@ PSP_ADMISSION=${PSP_ADMISSION:-""}
 RUNTIME_CONFIG=${RUNTIME_CONFIG:-""}
 KUBELET_AUTHORIZATION_WEBHOOK=${KUBELET_AUTHORIZATION_WEBHOOK:-""}
 KUBELET_AUTHENTICATION_WEBHOOK=${KUBELET_AUTHENTICATION_WEBHOOK:-""}
-POD_MANIFEST_PATH=${POD_MANIFEST_PATH:-""}
+POD_MANIFEST_PATH=${POD_MANIFEST_PATH:-"/var/run/kubernetes/static-pods"}
 # Name of the network plugin, eg: "kubenet"
 NET_PLUGIN=${NET_PLUGIN:-""}
 # Place the binaries required by NET_PLUGIN in this directory, eg: "/home/kubernetes/bin".
@@ -192,7 +192,7 @@ ENABLE_CONTROLLER_ATTACH_DETACH=${ENABLE_CONTROLLER_ATTACH_DETACH:-"true"} # cur
 # This is the default dir and filename where the apiserver will generate a self-signed cert
 # which should be able to be used as the CA to verify itself
 CERT_DIR=${CERT_DIR:-"/var/run/kubernetes"}
-ROOT_CA_FILE=$CERT_DIR/apiserver.crt
+ROOT_CA_FILE=${CERT_DIR}/server-ca.crt
 EXPERIMENTAL_CRI=${EXPERIMENTAL_CRI:-"false"}
 
 # name of the cgroup driver, i.e. cgroupfs or systemd
@@ -398,8 +398,14 @@ function start_apiserver {
         advertise_address="--advertise_address=${API_HOST_IP}"
     fi
 
-    # Create client ca
+    # Create CA signers
+    kube::util::create_signing_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" server '"server auth"'
     kube::util::create_signing_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" client '"client auth"'
+    # Create auth proxy client ca
+    kube::util::create_signing_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" request-header '"client auth"'
+
+    # serving cert for kube-apiserver
+    kube::util::create_serving_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" "server-ca" kube-apiserver kubernetes.default.svc "localhost" ${API_HOST_IP} ${API_HOST}
 
     # Create client certs signed with client-ca, given id, given CN and a number of groups
     kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" 'client-ca' kubelet system:node:${HOSTNAME_OVERRIDE} system:nodes
@@ -408,9 +414,13 @@ function start_apiserver {
     kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" 'client-ca' scheduler  system:kube-scheduler
     kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" 'client-ca' admin system:admin system:masters
 
-    # Create auth proxy client ca
-    kube::util::create_signing_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" request-header '"client auth"'
+    # Create matching certificates for kube-aggregator
+    kube::util::create_serving_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" "server-ca" kube-aggregator api.kube-public.svc "localhost" ${API_HOST_IP}
     kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" request-header-ca auth-proxy system:auth-proxy
+    # TODO remove masters and add rolebinding
+    kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" 'client-ca' kube-aggregator system:kube-aggregator system:masters
+    kube::util::write_client_kubeconfig "${CONTROLPLANE_SUDO}" "${CERT_DIR}" "${ROOT_CA_FILE}" "${API_HOST}" "${API_SECURE_PORT}" kube-aggregator
+
 
     APISERVER_LOG=/tmp/kube-apiserver.log
     ${CONTROLPLANE_SUDO} "${GO_OUT}/hyperkube" apiserver ${anytoken_arg} ${authorizer_arg} ${priv_arg} ${runtime_config}\
@@ -423,7 +433,9 @@ function start_apiserver {
       --admission-control="${ADMISSION_CONTROL}" \
       --bind-address="${API_BIND_ADDR}" \
       --secure-port="${API_SECURE_PORT}" \
-      --tls-ca-file="${ROOT_CA_FILE}" \
+      --tls-cert-file="${CERT_DIR}/serving-kube-apiserver.crt" \
+      --tls-private-key-file="${CERT_DIR}/serving-kube-apiserver.key" \
+      --tls-ca-file="${CERT_DIR}/server-ca.crt" \
       --insecure-bind-address="${API_HOST_IP}" \
       --insecure-port="${API_PORT}" \
       --etcd-servers="http://${ETCD_HOST}:${ETCD_PORT}" \
@@ -468,6 +480,14 @@ function start_apiserver {
             AUTH_ARGS="--client-key=${CERT_DIR}/client-admin.key --client-certificate=${CERT_DIR}/client-admin.crt"
         fi
     fi
+
+    # create the kube-public namespace for the aggregator
+    ${KUBECTL} --kubeconfig="${CERT_DIR}/admin.kubeconfig" create namespace kube-public
+    ${CONTROLPLANE_SUDO} cp "${CERT_DIR}/admin.kubeconfig" "${CERT_DIR}/admin-kube-aggregator.kubeconfig"
+    ${CONTROLPLANE_SUDO} chown $(whoami) "${CERT_DIR}/admin-kube-aggregator.kubeconfig"
+    ${KUBECTL} config set-cluster local-up-cluster --kubeconfig="${CERT_DIR}/admin-kube-aggregator.kubeconfig" --server="https://${API_HOST_IP}:9443"
+    echo "use 'kubectl --kubeconfig=${CERT_DIR}/admin-kube-aggregator.kubeconfig' to use the aggregated API server"
+
 }
 
 function start_controller_manager {
@@ -495,6 +515,8 @@ function start_controller_manager {
 
 function start_kubelet {
     KUBELET_LOG=/tmp/kubelet.log
+    mkdir -p ${POD_MANIFEST_PATH} || true
+    cp ${KUBE_ROOT}/vendor/k8s.io/kube-aggregator/artifacts/hostpath-pods/insecure-etcd-pod.yaml ${POD_MANIFEST_PATH}/kube-aggregator.yaml
 
     priv_arg=""
     if [[ -n "${ALLOW_PRIVILEGED}" ]]; then

hack/verify-flags/known-flags.txt

@@ -117,6 +117,7 @@ contain-pod-resources
 contention-profiling
 controllermanager-arg-overrides
 controller-start-interval
+core-kubeconfig
 cors-allowed-origins
 cpu-cfs-quota
 cpu-percent

staging/src/k8s.io/kube-aggregator/artifacts/hostpath-pods/insecure-etcd-pod.yaml

@@ -0,0 +1,81 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-aggregator
+  namespace: kube-public
+spec:
+  hostNetwork: true
+  containers:
+  - name: kube-aggregator
+    image: kube-aggregator
+    imagePullPolicy: IfNotPresent
+    args:
+    - "/usr/local/bin/kube-aggregator"
+    - "--secure-port=9443"
+    - "--core-kubeconfig=/var/run/auth-client/kube-aggregator.kubeconfig"
+    - "--authentication-kubeconfig=/var/run/auth-client/kube-aggregator.kubeconfig"
+    - "--authorization-kubeconfig=/var/run/auth-client/kube-aggregator.kubeconfig"
+    - "--proxy-client-cert-file=/var/run/auth-proxy-client/client-auth-proxy.crt"
+    - "--proxy-client-key-file=/var/run/auth-proxy-client/client-auth-proxy.key"
+    - "--tls-cert-file=/var/run/serving-cert/serving-kube-aggregator.crt"
+    - "--tls-private-key-file=/var/run/serving-cert/serving-kube-aggregator.key"
+    - "--tls-ca-file=/var/run/serving-ca/server-ca.crt"
+    - "--client-ca-file=/var/run/client-ca/client-ca.crt"
+    - "--requestheader-username-headers=X-Remote-User"
+    - "--requestheader-group-headers=X-Remote-Group"
+    - "--requestheader-extra-headers-prefix=X-Remote-Extra-"
+    - "--requestheader-client-ca-file=/var/run/request-header-ca/request-header-ca.crt"
+    - "--etcd-servers=http://127.0.0.1:2379"
+    ports:
+    - containerPort: 9443
+      hostPort: 9443
+    volumeMounts:
+    - mountPath: /var/run/request-header-ca
+      name: volume-request-header-ca
+      readOnly: true
+    - mountPath: /var/run/client-ca
+      name: volume-client-ca
+      readOnly: true
+    - mountPath: /var/run/auth-proxy-client
+      name: volume-auth-proxy-client
+      readOnly: true
+    - mountPath: /var/run/etcd-client-cert
+      name: volume-etcd-client-cert
+      readOnly: true
+    - mountPath: /var/run/serving-ca
+      name: volume-serving-ca
+      readOnly: true
+    - mountPath: /var/run/serving-cert
+      name: volume-serving-cert
+      readOnly: true
+    - mountPath: /var/run/etcd-ca
+      name: volume-etcd-ca
+      readOnly: true
+    - mountPath: /var/run/auth-client
+      name: volume-auth-client
+      readOnly: true
+  volumes:
+  - name: volume-request-header-ca
+    hostPath:
+      path: /var/run/kubernetes/
+  - name: volume-client-ca
+    hostPath:
+      path: /var/run/kubernetes/
+  - name: volume-auth-proxy-client
+    hostPath:
+      path: /var/run/kubernetes/
+  - name: volume-etcd-client-cert
+    hostPath:
+      path: /var/run/kubernetes/
+  - name: volume-serving-cert
+    hostPath:
+      path: /var/run/kubernetes/
+  - name: volume-serving-ca
+    hostPath:
+      path: /var/run/kubernetes/
+  - name: volume-etcd-ca
+    hostPath:
+      path: /var/run/kubernetes/
+  - name: volume-auth-client
+    hostPath:
+      path: /var/run/kubernetes/

staging/src/k8s.io/kube-aggregator/artifacts/self-contained/kubernetes-discover-pod.yaml

@@ -84,14 +84,14 @@ spec:
       - name: volume-etcd-client-cert
         secret:
           defaultMode: 420
-          secretName: kube-aggregator-etcd
+          secretName: discovery-etcd
       - name: volume-serving-cert
         secret:
           defaultMode: 420
-          secretName: serving-kube-aggregator
+          secretName: serving-discovery
       - configMap:
           defaultMode: 420
-          name: kube-aggregator-ca
+          name: discovery-ca
         name: volume-serving-ca
       - configMap:
           defaultMode: 420

staging/src/k8s.io/kube-aggregator/hack/build-image.sh

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 
-KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../../..
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../../../../..
 source "${KUBE_ROOT}/hack/lib/util.sh"
 
 # Register function to be called on EXIT to remove generated binary.
@@ -24,5 +24,7 @@ function cleanup {
 }
 trap cleanup EXIT
 
-cp -v ${KUBE_ROOT}/_output/local/bin/linux/amd64/kube-aggregator "${KUBE_ROOT}/vendor/k8s.io/kube-aggregator/artifacts/simple-image/kube-aggregator"
-docker build -t kube-aggregator:latest ${KUBE_ROOT}/vendor/k8s.io/kube-aggregator/artifacts/simple-image
+pushd "${KUBE_ROOT}/vendor/k8s.io/kube-aggregator"
+cp -v ../../../../_output/local/bin/linux/amd64/kube-aggregator ./artifacts/simple-image/kube-aggregator
+docker build -t kube-aggregator:latest ./artifacts/simple-image
+popd

staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go

@@ -30,7 +30,8 @@ import (
 	genericoptions "k8s.io/apiserver/pkg/server/options"
 	kubeclientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/pkg/api"
-	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/kube-aggregator/pkg/apiserver"
 
 	"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1alpha1"
@@ -46,6 +47,10 @@ type AggregatorOptions struct {
 	ProxyClientCertFile string
 	ProxyClientKeyFile  string
 
+	// CoreAPIKubeconfig is a filename for a kubeconfig file to contact the core API server wtih
+	// If it is not set, the in cluster config is used
+	CoreAPIKubeconfig string
+
 	StdOut io.Writer
 	StdErr io.Writer
 }
@@ -81,7 +86,9 @@ func NewCommandStartAggregator(out, err io.Writer) *cobra.Command {
 	o.RecommendedOptions.AddFlags(flags)
 	flags.StringVar(&o.ProxyClientCertFile, "proxy-client-cert-file", o.ProxyClientCertFile, "client certificate used identify the proxy to the API server")
 	flags.StringVar(&o.ProxyClientKeyFile, "proxy-client-key-file", o.ProxyClientKeyFile, "client certificate key used identify the proxy to the API server")
-
+	flags.StringVar(&o.CoreAPIKubeconfig, "core-kubeconfig", o.CoreAPIKubeconfig, ""+
+		"kubeconfig file pointing at the 'core' kubernetes server with enough rights to get,list,watch "+
+		" services,endpoints.  If not set, the in-cluster config is used")
 	return cmd
 }
 
@@ -110,10 +117,21 @@ func (o AggregatorOptions) RunAggregator() error {
 		sets.NewString("attach", "exec", "proxy", "log", "portforward"),
 	)
 
-	kubeconfig, err := restclient.InClusterConfig()
+	var kubeconfig *rest.Config
+	var err error
+	if len(o.CoreAPIKubeconfig) > 0 {
+		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: o.CoreAPIKubeconfig}
+		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})
+
+		kubeconfig, err = loader.ClientConfig()
+
+	} else {
+		kubeconfig, err = rest.InClusterConfig()
+	}
 	if err != nil {
 		return err
 	}
+
 	coreAPIServerClient, err := kubeclientset.NewForConfig(kubeconfig)
 	if err != nil {
 		return err

vendor/BUILD

@@ -16798,6 +16798,7 @@ go_library(
         "//vendor:k8s.io/client-go/kubernetes",
         "//vendor:k8s.io/client-go/pkg/api",
         "//vendor:k8s.io/client-go/rest",
+        "//vendor:k8s.io/client-go/tools/clientcmd",
         "//vendor:k8s.io/kube-aggregator/pkg/apis/apiregistration/v1alpha1",
         "//vendor:k8s.io/kube-aggregator/pkg/apiserver",
     ],