mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 19:56:01 +00:00
Merge pull request #40928 from deads2k/agg-01-local-up
Automatic merge from submit-queue make kube-aggregator run as static pod for local-up-cluster Runs the kube-aggregator as a static pod for local-up-cluster. Looks like someone broke kubectl negotiation again, so I'll fix that up separately. @kubernetes/sig-api-machinery-misc @lavalamp you're probably looking to run kube-aggregator as a static pod, here's an example. @jwforres I'll make a secure variant for wiring up to openshift.
This commit is contained in:
commit
5802799e56
@ -28,7 +28,7 @@ PSP_ADMISSION=${PSP_ADMISSION:-""}
|
|||||||
RUNTIME_CONFIG=${RUNTIME_CONFIG:-""}
|
RUNTIME_CONFIG=${RUNTIME_CONFIG:-""}
|
||||||
KUBELET_AUTHORIZATION_WEBHOOK=${KUBELET_AUTHORIZATION_WEBHOOK:-""}
|
KUBELET_AUTHORIZATION_WEBHOOK=${KUBELET_AUTHORIZATION_WEBHOOK:-""}
|
||||||
KUBELET_AUTHENTICATION_WEBHOOK=${KUBELET_AUTHENTICATION_WEBHOOK:-""}
|
KUBELET_AUTHENTICATION_WEBHOOK=${KUBELET_AUTHENTICATION_WEBHOOK:-""}
|
||||||
POD_MANIFEST_PATH=${POD_MANIFEST_PATH:-""}
|
POD_MANIFEST_PATH=${POD_MANIFEST_PATH:-"/var/run/kubernetes/static-pods"}
|
||||||
# Name of the network plugin, eg: "kubenet"
|
# Name of the network plugin, eg: "kubenet"
|
||||||
NET_PLUGIN=${NET_PLUGIN:-""}
|
NET_PLUGIN=${NET_PLUGIN:-""}
|
||||||
# Place the binaries required by NET_PLUGIN in this directory, eg: "/home/kubernetes/bin".
|
# Place the binaries required by NET_PLUGIN in this directory, eg: "/home/kubernetes/bin".
|
||||||
@ -192,7 +192,7 @@ ENABLE_CONTROLLER_ATTACH_DETACH=${ENABLE_CONTROLLER_ATTACH_DETACH:-"true"} # cur
|
|||||||
# This is the default dir and filename where the apiserver will generate a self-signed cert
|
# This is the default dir and filename where the apiserver will generate a self-signed cert
|
||||||
# which should be able to be used as the CA to verify itself
|
# which should be able to be used as the CA to verify itself
|
||||||
CERT_DIR=${CERT_DIR:-"/var/run/kubernetes"}
|
CERT_DIR=${CERT_DIR:-"/var/run/kubernetes"}
|
||||||
ROOT_CA_FILE=$CERT_DIR/apiserver.crt
|
ROOT_CA_FILE=${CERT_DIR}/server-ca.crt
|
||||||
EXPERIMENTAL_CRI=${EXPERIMENTAL_CRI:-"false"}
|
EXPERIMENTAL_CRI=${EXPERIMENTAL_CRI:-"false"}
|
||||||
|
|
||||||
# name of the cgroup driver, i.e. cgroupfs or systemd
|
# name of the cgroup driver, i.e. cgroupfs or systemd
|
||||||
@ -398,8 +398,14 @@ function start_apiserver {
|
|||||||
advertise_address="--advertise_address=${API_HOST_IP}"
|
advertise_address="--advertise_address=${API_HOST_IP}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Create client ca
|
# Create CA signers
|
||||||
|
kube::util::create_signing_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" server '"server auth"'
|
||||||
kube::util::create_signing_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" client '"client auth"'
|
kube::util::create_signing_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" client '"client auth"'
|
||||||
|
# Create auth proxy client ca
|
||||||
|
kube::util::create_signing_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" request-header '"client auth"'
|
||||||
|
|
||||||
|
# serving cert for kube-apiserver
|
||||||
|
kube::util::create_serving_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" "server-ca" kube-apiserver kubernetes.default.svc "localhost" ${API_HOST_IP} ${API_HOST}
|
||||||
|
|
||||||
# Create client certs signed with client-ca, given id, given CN and a number of groups
|
# Create client certs signed with client-ca, given id, given CN and a number of groups
|
||||||
kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" 'client-ca' kubelet system:node:${HOSTNAME_OVERRIDE} system:nodes
|
kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" 'client-ca' kubelet system:node:${HOSTNAME_OVERRIDE} system:nodes
|
||||||
@ -408,9 +414,13 @@ function start_apiserver {
|
|||||||
kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" 'client-ca' scheduler system:kube-scheduler
|
kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" 'client-ca' scheduler system:kube-scheduler
|
||||||
kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" 'client-ca' admin system:admin system:masters
|
kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" 'client-ca' admin system:admin system:masters
|
||||||
|
|
||||||
# Create auth proxy client ca
|
# Create matching certificates for kube-aggregator
|
||||||
kube::util::create_signing_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" request-header '"client auth"'
|
kube::util::create_serving_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" "server-ca" kube-aggregator api.kube-public.svc "localhost" ${API_HOST_IP}
|
||||||
kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" request-header-ca auth-proxy system:auth-proxy
|
kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" request-header-ca auth-proxy system:auth-proxy
|
||||||
|
# TODO remove masters and add rolebinding
|
||||||
|
kube::util::create_client_certkey "${CONTROLPLANE_SUDO}" "${CERT_DIR}" 'client-ca' kube-aggregator system:kube-aggregator system:masters
|
||||||
|
kube::util::write_client_kubeconfig "${CONTROLPLANE_SUDO}" "${CERT_DIR}" "${ROOT_CA_FILE}" "${API_HOST}" "${API_SECURE_PORT}" kube-aggregator
|
||||||
|
|
||||||
|
|
||||||
APISERVER_LOG=/tmp/kube-apiserver.log
|
APISERVER_LOG=/tmp/kube-apiserver.log
|
||||||
${CONTROLPLANE_SUDO} "${GO_OUT}/hyperkube" apiserver ${anytoken_arg} ${authorizer_arg} ${priv_arg} ${runtime_config}\
|
${CONTROLPLANE_SUDO} "${GO_OUT}/hyperkube" apiserver ${anytoken_arg} ${authorizer_arg} ${priv_arg} ${runtime_config}\
|
||||||
@ -423,7 +433,9 @@ function start_apiserver {
|
|||||||
--admission-control="${ADMISSION_CONTROL}" \
|
--admission-control="${ADMISSION_CONTROL}" \
|
||||||
--bind-address="${API_BIND_ADDR}" \
|
--bind-address="${API_BIND_ADDR}" \
|
||||||
--secure-port="${API_SECURE_PORT}" \
|
--secure-port="${API_SECURE_PORT}" \
|
||||||
--tls-ca-file="${ROOT_CA_FILE}" \
|
--tls-cert-file="${CERT_DIR}/serving-kube-apiserver.crt" \
|
||||||
|
--tls-private-key-file="${CERT_DIR}/serving-kube-apiserver.key" \
|
||||||
|
--tls-ca-file="${CERT_DIR}/server-ca.crt" \
|
||||||
--insecure-bind-address="${API_HOST_IP}" \
|
--insecure-bind-address="${API_HOST_IP}" \
|
||||||
--insecure-port="${API_PORT}" \
|
--insecure-port="${API_PORT}" \
|
||||||
--etcd-servers="http://${ETCD_HOST}:${ETCD_PORT}" \
|
--etcd-servers="http://${ETCD_HOST}:${ETCD_PORT}" \
|
||||||
@ -468,6 +480,14 @@ function start_apiserver {
|
|||||||
AUTH_ARGS="--client-key=${CERT_DIR}/client-admin.key --client-certificate=${CERT_DIR}/client-admin.crt"
|
AUTH_ARGS="--client-key=${CERT_DIR}/client-admin.key --client-certificate=${CERT_DIR}/client-admin.crt"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# create the kube-public namespace for the aggregator
|
||||||
|
${KUBECTL} --kubeconfig="${CERT_DIR}/admin.kubeconfig" create namespace kube-public
|
||||||
|
${CONTROLPLANE_SUDO} cp "${CERT_DIR}/admin.kubeconfig" "${CERT_DIR}/admin-kube-aggregator.kubeconfig"
|
||||||
|
${CONTROLPLANE_SUDO} chown $(whoami) "${CERT_DIR}/admin-kube-aggregator.kubeconfig"
|
||||||
|
${KUBECTL} config set-cluster local-up-cluster --kubeconfig="${CERT_DIR}/admin-kube-aggregator.kubeconfig" --server="https://${API_HOST_IP}:9443"
|
||||||
|
echo "use 'kubectl --kubeconfig=${CERT_DIR}/admin-kube-aggregator.kubeconfig' to use the aggregated API server"
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function start_controller_manager {
|
function start_controller_manager {
|
||||||
@ -495,6 +515,8 @@ function start_controller_manager {
|
|||||||
|
|
||||||
function start_kubelet {
|
function start_kubelet {
|
||||||
KUBELET_LOG=/tmp/kubelet.log
|
KUBELET_LOG=/tmp/kubelet.log
|
||||||
|
mkdir -p ${POD_MANIFEST_PATH} || true
|
||||||
|
cp ${KUBE_ROOT}/vendor/k8s.io/kube-aggregator/artifacts/hostpath-pods/insecure-etcd-pod.yaml ${POD_MANIFEST_PATH}/kube-aggregator.yaml
|
||||||
|
|
||||||
priv_arg=""
|
priv_arg=""
|
||||||
if [[ -n "${ALLOW_PRIVILEGED}" ]]; then
|
if [[ -n "${ALLOW_PRIVILEGED}" ]]; then
|
||||||
|
@ -117,6 +117,7 @@ contain-pod-resources
|
|||||||
contention-profiling
|
contention-profiling
|
||||||
controllermanager-arg-overrides
|
controllermanager-arg-overrides
|
||||||
controller-start-interval
|
controller-start-interval
|
||||||
|
core-kubeconfig
|
||||||
cors-allowed-origins
|
cors-allowed-origins
|
||||||
cpu-cfs-quota
|
cpu-cfs-quota
|
||||||
cpu-percent
|
cpu-percent
|
||||||
|
@ -0,0 +1,81 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: kube-aggregator
|
||||||
|
namespace: kube-public
|
||||||
|
spec:
|
||||||
|
hostNetwork: true
|
||||||
|
containers:
|
||||||
|
- name: kube-aggregator
|
||||||
|
image: kube-aggregator
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
args:
|
||||||
|
- "/usr/local/bin/kube-aggregator"
|
||||||
|
- "--secure-port=9443"
|
||||||
|
- "--core-kubeconfig=/var/run/auth-client/kube-aggregator.kubeconfig"
|
||||||
|
- "--authentication-kubeconfig=/var/run/auth-client/kube-aggregator.kubeconfig"
|
||||||
|
- "--authorization-kubeconfig=/var/run/auth-client/kube-aggregator.kubeconfig"
|
||||||
|
- "--proxy-client-cert-file=/var/run/auth-proxy-client/client-auth-proxy.crt"
|
||||||
|
- "--proxy-client-key-file=/var/run/auth-proxy-client/client-auth-proxy.key"
|
||||||
|
- "--tls-cert-file=/var/run/serving-cert/serving-kube-aggregator.crt"
|
||||||
|
- "--tls-private-key-file=/var/run/serving-cert/serving-kube-aggregator.key"
|
||||||
|
- "--tls-ca-file=/var/run/serving-ca/server-ca.crt"
|
||||||
|
- "--client-ca-file=/var/run/client-ca/client-ca.crt"
|
||||||
|
- "--requestheader-username-headers=X-Remote-User"
|
||||||
|
- "--requestheader-group-headers=X-Remote-Group"
|
||||||
|
- "--requestheader-extra-headers-prefix=X-Remote-Extra-"
|
||||||
|
- "--requestheader-client-ca-file=/var/run/request-header-ca/request-header-ca.crt"
|
||||||
|
- "--etcd-servers=http://127.0.0.1:2379"
|
||||||
|
ports:
|
||||||
|
- containerPort: 9443
|
||||||
|
hostPort: 9443
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /var/run/request-header-ca
|
||||||
|
name: volume-request-header-ca
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /var/run/client-ca
|
||||||
|
name: volume-client-ca
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /var/run/auth-proxy-client
|
||||||
|
name: volume-auth-proxy-client
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /var/run/etcd-client-cert
|
||||||
|
name: volume-etcd-client-cert
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /var/run/serving-ca
|
||||||
|
name: volume-serving-ca
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /var/run/serving-cert
|
||||||
|
name: volume-serving-cert
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /var/run/etcd-ca
|
||||||
|
name: volume-etcd-ca
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /var/run/auth-client
|
||||||
|
name: volume-auth-client
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: volume-request-header-ca
|
||||||
|
hostPath:
|
||||||
|
path: /var/run/kubernetes/
|
||||||
|
- name: volume-client-ca
|
||||||
|
hostPath:
|
||||||
|
path: /var/run/kubernetes/
|
||||||
|
- name: volume-auth-proxy-client
|
||||||
|
hostPath:
|
||||||
|
path: /var/run/kubernetes/
|
||||||
|
- name: volume-etcd-client-cert
|
||||||
|
hostPath:
|
||||||
|
path: /var/run/kubernetes/
|
||||||
|
- name: volume-serving-cert
|
||||||
|
hostPath:
|
||||||
|
path: /var/run/kubernetes/
|
||||||
|
- name: volume-serving-ca
|
||||||
|
hostPath:
|
||||||
|
path: /var/run/kubernetes/
|
||||||
|
- name: volume-etcd-ca
|
||||||
|
hostPath:
|
||||||
|
path: /var/run/kubernetes/
|
||||||
|
- name: volume-auth-client
|
||||||
|
hostPath:
|
||||||
|
path: /var/run/kubernetes/
|
@ -84,14 +84,14 @@ spec:
|
|||||||
- name: volume-etcd-client-cert
|
- name: volume-etcd-client-cert
|
||||||
secret:
|
secret:
|
||||||
defaultMode: 420
|
defaultMode: 420
|
||||||
secretName: kube-aggregator-etcd
|
secretName: discovery-etcd
|
||||||
- name: volume-serving-cert
|
- name: volume-serving-cert
|
||||||
secret:
|
secret:
|
||||||
defaultMode: 420
|
defaultMode: 420
|
||||||
secretName: serving-kube-aggregator
|
secretName: serving-discovery
|
||||||
- configMap:
|
- configMap:
|
||||||
defaultMode: 420
|
defaultMode: 420
|
||||||
name: kube-aggregator-ca
|
name: discovery-ca
|
||||||
name: volume-serving-ca
|
name: volume-serving-ca
|
||||||
- configMap:
|
- configMap:
|
||||||
defaultMode: 420
|
defaultMode: 420
|
@ -15,7 +15,7 @@
|
|||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
|
||||||
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../../..
|
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../../../../..
|
||||||
source "${KUBE_ROOT}/hack/lib/util.sh"
|
source "${KUBE_ROOT}/hack/lib/util.sh"
|
||||||
|
|
||||||
# Register function to be called on EXIT to remove generated binary.
|
# Register function to be called on EXIT to remove generated binary.
|
||||||
@ -24,5 +24,7 @@ function cleanup {
|
|||||||
}
|
}
|
||||||
trap cleanup EXIT
|
trap cleanup EXIT
|
||||||
|
|
||||||
cp -v ${KUBE_ROOT}/_output/local/bin/linux/amd64/kube-aggregator "${KUBE_ROOT}/vendor/k8s.io/kube-aggregator/artifacts/simple-image/kube-aggregator"
|
pushd "${KUBE_ROOT}/vendor/k8s.io/kube-aggregator"
|
||||||
docker build -t kube-aggregator:latest ${KUBE_ROOT}/vendor/k8s.io/kube-aggregator/artifacts/simple-image
|
cp -v ../../../../_output/local/bin/linux/amd64/kube-aggregator ./artifacts/simple-image/kube-aggregator
|
||||||
|
docker build -t kube-aggregator:latest ./artifacts/simple-image
|
||||||
|
popd
|
@ -30,7 +30,8 @@ import (
|
|||||||
genericoptions "k8s.io/apiserver/pkg/server/options"
|
genericoptions "k8s.io/apiserver/pkg/server/options"
|
||||||
kubeclientset "k8s.io/client-go/kubernetes"
|
kubeclientset "k8s.io/client-go/kubernetes"
|
||||||
"k8s.io/client-go/pkg/api"
|
"k8s.io/client-go/pkg/api"
|
||||||
restclient "k8s.io/client-go/rest"
|
"k8s.io/client-go/rest"
|
||||||
|
"k8s.io/client-go/tools/clientcmd"
|
||||||
"k8s.io/kube-aggregator/pkg/apiserver"
|
"k8s.io/kube-aggregator/pkg/apiserver"
|
||||||
|
|
||||||
"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1alpha1"
|
"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1alpha1"
|
||||||
@ -46,6 +47,10 @@ type AggregatorOptions struct {
|
|||||||
ProxyClientCertFile string
|
ProxyClientCertFile string
|
||||||
ProxyClientKeyFile string
|
ProxyClientKeyFile string
|
||||||
|
|
||||||
|
// CoreAPIKubeconfig is a filename for a kubeconfig file to contact the core API server wtih
|
||||||
|
// If it is not set, the in cluster config is used
|
||||||
|
CoreAPIKubeconfig string
|
||||||
|
|
||||||
StdOut io.Writer
|
StdOut io.Writer
|
||||||
StdErr io.Writer
|
StdErr io.Writer
|
||||||
}
|
}
|
||||||
@ -81,7 +86,9 @@ func NewCommandStartAggregator(out, err io.Writer) *cobra.Command {
|
|||||||
o.RecommendedOptions.AddFlags(flags)
|
o.RecommendedOptions.AddFlags(flags)
|
||||||
flags.StringVar(&o.ProxyClientCertFile, "proxy-client-cert-file", o.ProxyClientCertFile, "client certificate used identify the proxy to the API server")
|
flags.StringVar(&o.ProxyClientCertFile, "proxy-client-cert-file", o.ProxyClientCertFile, "client certificate used identify the proxy to the API server")
|
||||||
flags.StringVar(&o.ProxyClientKeyFile, "proxy-client-key-file", o.ProxyClientKeyFile, "client certificate key used identify the proxy to the API server")
|
flags.StringVar(&o.ProxyClientKeyFile, "proxy-client-key-file", o.ProxyClientKeyFile, "client certificate key used identify the proxy to the API server")
|
||||||
|
flags.StringVar(&o.CoreAPIKubeconfig, "core-kubeconfig", o.CoreAPIKubeconfig, ""+
|
||||||
|
"kubeconfig file pointing at the 'core' kubernetes server with enough rights to get,list,watch "+
|
||||||
|
" services,endpoints. If not set, the in-cluster config is used")
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -110,10 +117,21 @@ func (o AggregatorOptions) RunAggregator() error {
|
|||||||
sets.NewString("attach", "exec", "proxy", "log", "portforward"),
|
sets.NewString("attach", "exec", "proxy", "log", "portforward"),
|
||||||
)
|
)
|
||||||
|
|
||||||
kubeconfig, err := restclient.InClusterConfig()
|
var kubeconfig *rest.Config
|
||||||
|
var err error
|
||||||
|
if len(o.CoreAPIKubeconfig) > 0 {
|
||||||
|
loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: o.CoreAPIKubeconfig}
|
||||||
|
loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})
|
||||||
|
|
||||||
|
kubeconfig, err = loader.ClientConfig()
|
||||||
|
|
||||||
|
} else {
|
||||||
|
kubeconfig, err = rest.InClusterConfig()
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
coreAPIServerClient, err := kubeclientset.NewForConfig(kubeconfig)
|
coreAPIServerClient, err := kubeclientset.NewForConfig(kubeconfig)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
1
vendor/BUILD
vendored
1
vendor/BUILD
vendored
@ -16798,6 +16798,7 @@ go_library(
|
|||||||
"//vendor:k8s.io/client-go/kubernetes",
|
"//vendor:k8s.io/client-go/kubernetes",
|
||||||
"//vendor:k8s.io/client-go/pkg/api",
|
"//vendor:k8s.io/client-go/pkg/api",
|
||||||
"//vendor:k8s.io/client-go/rest",
|
"//vendor:k8s.io/client-go/rest",
|
||||||
|
"//vendor:k8s.io/client-go/tools/clientcmd",
|
||||||
"//vendor:k8s.io/kube-aggregator/pkg/apis/apiregistration/v1alpha1",
|
"//vendor:k8s.io/kube-aggregator/pkg/apis/apiregistration/v1alpha1",
|
||||||
"//vendor:k8s.io/kube-aggregator/pkg/apiserver",
|
"//vendor:k8s.io/kube-aggregator/pkg/apiserver",
|
||||||
],
|
],
|
||||||
|
Loading…
Reference in New Issue
Block a user