diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index d39997bf418..3aedfe93988 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -28,7 +28,7 @@ PSP_ADMISSION=${PSP_ADMISSION:-""} RUNTIME_CONFIG=${RUNTIME_CONFIG:-""} KUBELET_AUTHORIZATION_WEBHOOK=${KUBELET_AUTHORIZATION_WEBHOOK:-""} KUBELET_AUTHENTICATION_WEBHOOK=${KUBELET_AUTHENTICATION_WEBHOOK:-""} -POD_MANIFEST_PATH=${POD_MANIFEST_PATH:-""} +POD_MANIFEST_PATH=${POD_MANIFEST_PATH:-"/var/run/kubernetes/static-pods"} # Name of the network plugin, eg: "kubenet" NET_PLUGIN=${NET_PLUGIN:-""} # Place the binaries required by NET_PLUGIN in this directory, eg: "/home/kubernetes/bin". @@ -192,7 +192,7 @@ ENABLE_CONTROLLER_ATTACH_DETACH=${ENABLE_CONTROLLER_ATTACH_DETACH:-"true"} # cur # This is the default dir and filename where the apiserver will generate a self-signed cert # which should be able to be used as the CA to verify itself CERT_DIR=${CERT_DIR:-"/var/run/kubernetes"} -ROOT_CA_FILE=$CERT_DIR/apiserver.crt +ROOT_CA_FILE=${CERT_DIR}/server-ca.crt EXPERIMENTAL_CRI=${EXPERIMENTAL_CRI:-"false"} # name of the cgroup driver, i.e. cgroupfs or systemd @@ -398,8 +398,14 @@ function start_apiserver { advertise_address="--advertise_address=${API_HOST_IP}" fi - # Create client ca + # Create CA signers + kube::util::create_signing_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" server '"server auth"' kube::util::create_signing_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" client '"client auth"' + # Create auth proxy client ca + kube::util::create_signing_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" request-header '"client auth"' + + # serving cert for kube-apiserver + kube::util::create_serving_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" "server-ca" kube-apiserver kubernetes.default.svc "localhost" ${API_HOST_IP} ${API_HOST} # Create client certs signed with client-ca, given id, given CN and a number of groups kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" 'client-ca' kubelet system:node:${HOSTNAME_OVERRIDE} system:nodes @@ -408,9 +414,13 @@ function start_apiserver { kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" 'client-ca' scheduler system:kube-scheduler kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" 'client-ca' admin system:admin system:masters - # Create auth proxy client ca - kube::util::create_signing_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" request-header '"client auth"' + # Create matching certificates for kube-aggregator + kube::util::create_serving_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" "server-ca" kube-aggregator api.kube-public.svc "localhost" ${API_HOST_IP} kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" request-header-ca auth-proxy system:auth-proxy + # TODO remove masters and add rolebinding + kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" 'client-ca' kube-aggregator system:kube-aggregator system:masters + kube::util::write_client_kubeconfig "${CONTROLPLANE_SUDO}" "${CERT_DIR}" "${ROOT_CA_FILE}" "${API_HOST}" "${API_SECURE_PORT}" kube-aggregator + APISERVER_LOG=/tmp/kube-apiserver.log ${CONTROLPLANE_SUDO} "${GO_OUT}/hyperkube" apiserver ${anytoken_arg} ${authorizer_arg} ${priv_arg} ${runtime_config}\ @@ -423,7 +433,9 @@ function start_apiserver { --admission-control="${ADMISSION_CONTROL}" \ --bind-address="${API_BIND_ADDR}" \ --secure-port="${API_SECURE_PORT}" \ - --tls-ca-file="${ROOT_CA_FILE}" \ + --tls-cert-file="${CERT_DIR}/serving-kube-apiserver.crt" \ + --tls-private-key-file="${CERT_DIR}/serving-kube-apiserver.key" \ + --tls-ca-file="${CERT_DIR}/server-ca.crt" \ --insecure-bind-address="${API_HOST_IP}" \ --insecure-port="${API_PORT}" \ --etcd-servers="http://${ETCD_HOST}:${ETCD_PORT}" \ @@ -468,6 +480,14 @@ function start_apiserver { AUTH_ARGS="--client-key=${CERT_DIR}/client-admin.key --client-certificate=${CERT_DIR}/client-admin.crt" fi fi + + # create the kube-public namespace for the aggregator + ${KUBECTL} --kubeconfig="${CERT_DIR}/admin.kubeconfig" create namespace kube-public + ${CONTROLPLANE_SUDO} cp "${CERT_DIR}/admin.kubeconfig" "${CERT_DIR}/admin-kube-aggregator.kubeconfig" + ${CONTROLPLANE_SUDO} chown $(whoami) "${CERT_DIR}/admin-kube-aggregator.kubeconfig" + ${KUBECTL} config set-cluster local-up-cluster --kubeconfig="${CERT_DIR}/admin-kube-aggregator.kubeconfig" --server="https://${API_HOST_IP}:9443" + echo "use 'kubectl --kubeconfig=${CERT_DIR}/admin-kube-aggregator.kubeconfig' to use the aggregated API server" + } function start_controller_manager { @@ -495,6 +515,8 @@ function start_controller_manager { function start_kubelet { KUBELET_LOG=/tmp/kubelet.log + mkdir -p ${POD_MANIFEST_PATH} || true + cp ${KUBE_ROOT}/vendor/k8s.io/kube-aggregator/artifacts/hostpath-pods/insecure-etcd-pod.yaml ${POD_MANIFEST_PATH}/kube-aggregator.yaml priv_arg="" if [[ -n "${ALLOW_PRIVILEGED}" ]]; then diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 575fad0db84..b63b648d18a 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -117,6 +117,7 @@ contain-pod-resources contention-profiling controllermanager-arg-overrides controller-start-interval +core-kubeconfig cors-allowed-origins cpu-cfs-quota cpu-percent diff --git a/staging/src/k8s.io/kube-aggregator/artifacts/hostpath-pods/insecure-etcd-pod.yaml b/staging/src/k8s.io/kube-aggregator/artifacts/hostpath-pods/insecure-etcd-pod.yaml new file mode 100644 index 00000000000..af7f2969ea3 --- /dev/null +++ b/staging/src/k8s.io/kube-aggregator/artifacts/hostpath-pods/insecure-etcd-pod.yaml @@ -0,0 +1,81 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kube-aggregator + namespace: kube-public +spec: + hostNetwork: true + containers: + - name: kube-aggregator + image: kube-aggregator + imagePullPolicy: IfNotPresent + args: + - "/usr/local/bin/kube-aggregator" + - "--secure-port=9443" + - "--core-kubeconfig=/var/run/auth-client/kube-aggregator.kubeconfig" + - "--authentication-kubeconfig=/var/run/auth-client/kube-aggregator.kubeconfig" + - "--authorization-kubeconfig=/var/run/auth-client/kube-aggregator.kubeconfig" + - "--proxy-client-cert-file=/var/run/auth-proxy-client/client-auth-proxy.crt" + - "--proxy-client-key-file=/var/run/auth-proxy-client/client-auth-proxy.key" + - "--tls-cert-file=/var/run/serving-cert/serving-kube-aggregator.crt" + - "--tls-private-key-file=/var/run/serving-cert/serving-kube-aggregator.key" + - "--tls-ca-file=/var/run/serving-ca/server-ca.crt" + - "--client-ca-file=/var/run/client-ca/client-ca.crt" + - "--requestheader-username-headers=X-Remote-User" + - "--requestheader-group-headers=X-Remote-Group" + - "--requestheader-extra-headers-prefix=X-Remote-Extra-" + - "--requestheader-client-ca-file=/var/run/request-header-ca/request-header-ca.crt" + - "--etcd-servers=http://127.0.0.1:2379" + ports: + - containerPort: 9443 + hostPort: 9443 + volumeMounts: + - mountPath: /var/run/request-header-ca + name: volume-request-header-ca + readOnly: true + - mountPath: /var/run/client-ca + name: volume-client-ca + readOnly: true + - mountPath: /var/run/auth-proxy-client + name: volume-auth-proxy-client + readOnly: true + - mountPath: /var/run/etcd-client-cert + name: volume-etcd-client-cert + readOnly: true + - mountPath: /var/run/serving-ca + name: volume-serving-ca + readOnly: true + - mountPath: /var/run/serving-cert + name: volume-serving-cert + readOnly: true + - mountPath: /var/run/etcd-ca + name: volume-etcd-ca + readOnly: true + - mountPath: /var/run/auth-client + name: volume-auth-client + readOnly: true + volumes: + - name: volume-request-header-ca + hostPath: + path: /var/run/kubernetes/ + - name: volume-client-ca + hostPath: + path: /var/run/kubernetes/ + - name: volume-auth-proxy-client + hostPath: + path: /var/run/kubernetes/ + - name: volume-etcd-client-cert + hostPath: + path: /var/run/kubernetes/ + - name: volume-serving-cert + hostPath: + path: /var/run/kubernetes/ + - name: volume-serving-ca + hostPath: + path: /var/run/kubernetes/ + - name: volume-etcd-ca + hostPath: + path: /var/run/kubernetes/ + - name: volume-auth-client + hostPath: + path: /var/run/kubernetes/ diff --git a/staging/src/k8s.io/kube-aggregator/artifacts/local-cluster-up/etcd-pod.yaml b/staging/src/k8s.io/kube-aggregator/artifacts/self-contained/etcd-pod.yaml similarity index 100% rename from staging/src/k8s.io/kube-aggregator/artifacts/local-cluster-up/etcd-pod.yaml rename to staging/src/k8s.io/kube-aggregator/artifacts/self-contained/etcd-pod.yaml diff --git a/staging/src/k8s.io/kube-aggregator/artifacts/local-cluster-up/etcd-svc.yaml b/staging/src/k8s.io/kube-aggregator/artifacts/self-contained/etcd-svc.yaml similarity index 100% rename from staging/src/k8s.io/kube-aggregator/artifacts/local-cluster-up/etcd-svc.yaml rename to staging/src/k8s.io/kube-aggregator/artifacts/self-contained/etcd-svc.yaml diff --git a/staging/src/k8s.io/kube-aggregator/artifacts/local-cluster-up/kube-aggregator-pod.yaml b/staging/src/k8s.io/kube-aggregator/artifacts/self-contained/kubernetes-discover-pod.yaml similarity index 96% rename from staging/src/k8s.io/kube-aggregator/artifacts/local-cluster-up/kube-aggregator-pod.yaml rename to staging/src/k8s.io/kube-aggregator/artifacts/self-contained/kubernetes-discover-pod.yaml index 7b21ac210fd..a67f06b8995 100644 --- a/staging/src/k8s.io/kube-aggregator/artifacts/local-cluster-up/kube-aggregator-pod.yaml +++ b/staging/src/k8s.io/kube-aggregator/artifacts/self-contained/kubernetes-discover-pod.yaml @@ -84,14 +84,14 @@ spec: - name: volume-etcd-client-cert secret: defaultMode: 420 - secretName: kube-aggregator-etcd + secretName: discovery-etcd - name: volume-serving-cert secret: defaultMode: 420 - secretName: serving-kube-aggregator + secretName: serving-discovery - configMap: defaultMode: 420 - name: kube-aggregator-ca + name: discovery-ca name: volume-serving-ca - configMap: defaultMode: 420 diff --git a/staging/src/k8s.io/kube-aggregator/artifacts/local-cluster-up/kube-aggregator-sa.yaml b/staging/src/k8s.io/kube-aggregator/artifacts/self-contained/kubernetes-discovery-sa.yaml similarity index 100% rename from staging/src/k8s.io/kube-aggregator/artifacts/local-cluster-up/kube-aggregator-sa.yaml rename to staging/src/k8s.io/kube-aggregator/artifacts/self-contained/kubernetes-discovery-sa.yaml diff --git a/staging/src/k8s.io/kube-aggregator/artifacts/local-cluster-up/kube-aggregator-svc.yaml b/staging/src/k8s.io/kube-aggregator/artifacts/self-contained/kubernetes-discovery-svc.yaml similarity index 100% rename from staging/src/k8s.io/kube-aggregator/artifacts/local-cluster-up/kube-aggregator-svc.yaml rename to staging/src/k8s.io/kube-aggregator/artifacts/self-contained/kubernetes-discovery-svc.yaml diff --git a/staging/src/k8s.io/kube-aggregator/hack/build-image.sh b/staging/src/k8s.io/kube-aggregator/hack/build-image.sh index 014252f3046..6aae5fb6240 100755 --- a/staging/src/k8s.io/kube-aggregator/hack/build-image.sh +++ b/staging/src/k8s.io/kube-aggregator/hack/build-image.sh @@ -15,7 +15,7 @@ # limitations under the License. -KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../../.. +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../../../../.. source "${KUBE_ROOT}/hack/lib/util.sh" # Register function to be called on EXIT to remove generated binary. @@ -24,5 +24,7 @@ function cleanup { } trap cleanup EXIT -cp -v ${KUBE_ROOT}/_output/local/bin/linux/amd64/kube-aggregator "${KUBE_ROOT}/vendor/k8s.io/kube-aggregator/artifacts/simple-image/kube-aggregator" -docker build -t kube-aggregator:latest ${KUBE_ROOT}/vendor/k8s.io/kube-aggregator/artifacts/simple-image +pushd "${KUBE_ROOT}/vendor/k8s.io/kube-aggregator" +cp -v ../../../../_output/local/bin/linux/amd64/kube-aggregator ./artifacts/simple-image/kube-aggregator +docker build -t kube-aggregator:latest ./artifacts/simple-image +popd \ No newline at end of file diff --git a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go index 0f006d61917..a0a2ded4fc9 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go @@ -30,7 +30,8 @@ import ( genericoptions "k8s.io/apiserver/pkg/server/options" kubeclientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/pkg/api" - restclient "k8s.io/client-go/rest" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" "k8s.io/kube-aggregator/pkg/apiserver" "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1alpha1" @@ -46,6 +47,10 @@ type AggregatorOptions struct { ProxyClientCertFile string ProxyClientKeyFile string + // CoreAPIKubeconfig is a filename for a kubeconfig file to contact the core API server wtih + // If it is not set, the in cluster config is used + CoreAPIKubeconfig string + StdOut io.Writer StdErr io.Writer } @@ -81,7 +86,9 @@ func NewCommandStartAggregator(out, err io.Writer) *cobra.Command { o.RecommendedOptions.AddFlags(flags) flags.StringVar(&o.ProxyClientCertFile, "proxy-client-cert-file", o.ProxyClientCertFile, "client certificate used identify the proxy to the API server") flags.StringVar(&o.ProxyClientKeyFile, "proxy-client-key-file", o.ProxyClientKeyFile, "client certificate key used identify the proxy to the API server") - + flags.StringVar(&o.CoreAPIKubeconfig, "core-kubeconfig", o.CoreAPIKubeconfig, ""+ + "kubeconfig file pointing at the 'core' kubernetes server with enough rights to get,list,watch "+ + " services,endpoints. If not set, the in-cluster config is used") return cmd } @@ -110,10 +117,21 @@ func (o AggregatorOptions) RunAggregator() error { sets.NewString("attach", "exec", "proxy", "log", "portforward"), ) - kubeconfig, err := restclient.InClusterConfig() + var kubeconfig *rest.Config + var err error + if len(o.CoreAPIKubeconfig) > 0 { + loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: o.CoreAPIKubeconfig} + loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{}) + + kubeconfig, err = loader.ClientConfig() + + } else { + kubeconfig, err = rest.InClusterConfig() + } if err != nil { return err } + coreAPIServerClient, err := kubeclientset.NewForConfig(kubeconfig) if err != nil { return err diff --git a/vendor/BUILD b/vendor/BUILD index 14fd079407d..d673d320bfd 100644 --- a/vendor/BUILD +++ b/vendor/BUILD @@ -16798,6 +16798,7 @@ go_library( "//vendor:k8s.io/client-go/kubernetes", "//vendor:k8s.io/client-go/pkg/api", "//vendor:k8s.io/client-go/rest", + "//vendor:k8s.io/client-go/tools/clientcmd", "//vendor:k8s.io/kube-aggregator/pkg/apis/apiregistration/v1alpha1", "//vendor:k8s.io/kube-aggregator/pkg/apiserver", ],