From ebb65c5f4c51340f42c260cf00bde8629ed68d74 Mon Sep 17 00:00:00 2001 From: Walter Fender Date: Mon, 20 May 2019 12:44:51 -0700 Subject: [PATCH] Get network-proxy working with GCE. Got the proxy-server coming up in the master. Added certs and have it comiung up with those certs. Added a daemonset to run the network-agent. Adding support for agent running as a sameon set on every node. Added quick hack to test that proxy server/agent were correctly tunneling traffic to the kubelet. Added more WIP for reading network proxy configuration. Get flags set correctly and fix connection services. Adding missing ApplyTo Added ConnectivityService. Fixed build directives. Added connectivity service configuration. Fixed log levels. Fixed minor issues for feature turned off. Fixed boilerplate and format. Moved log dialer initialization earlier as per Liggits suggestion. Fixed a few minor issues in the configuration for GCE. Fixed scheme allocation Adding unit test. Added test for direct connectivity service. Switching to injecting the Lookup method rather than using a Singleton. First round of mikedaneses feedback. Fixed deployment to use yaml and other changes suggested by MikeDanese. Switched network proxy server/agent which are kebab-case not camelCase. Picked up DIAL_RSP fix. Factored in deads2k feedback. Feedback from mikedanese Factored in second round of feedback from David. Fix path in verify. Factored in anfernee's feedback. First part of lavalamps feedback. Factored in more changes from lavalamp and mikedanese. Renamed network-proxy to konnectivity-server and konnectivity-agent. Fixed tolerations and config file checking. Added missing strptr Finished lavalamps requested rename. Disambiguating konnectivity service by renaming it egress selector. Switched feature flag to KUBE_ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE --- build/lib/release.sh | 1 + .../addons/konnectivity-agent/daemonset.yaml | 66 ++++++ cluster/gce/config-default.sh | 3 + cluster/gce/gci/configure-helper.sh | 141 ++++++++++++ cluster/gce/manifests/BUILD | 1 + .../gce/manifests/konnectivity-server.yaml | 58 +++++ cluster/gce/manifests/kube-apiserver.manifest | 2 + cluster/gce/util.sh | 155 ++++++++++++- cmd/kube-apiserver/app/options/options.go | 3 + .../app/options/options_test.go | 4 + cmd/kube-apiserver/app/server.go | 7 + pkg/kubelet/client/BUILD | 1 + pkg/kubelet/client/kubelet_client.go | 20 +- .../src/k8s.io/apiextensions-apiserver/go.sum | 1 + staging/src/k8s.io/apiserver/go.sum | 1 + .../apiserver/pkg/apis/apiserver/register.go | 1 + .../apiserver/pkg/apis/apiserver/types.go | 49 ++++ .../pkg/apis/apiserver/v1alpha1/register.go | 1 + .../pkg/apis/apiserver/v1alpha1/types.go | 60 +++++ .../v1alpha1/zz_generated.conversion.go | 134 +++++++++++ .../v1alpha1/zz_generated.deepcopy.go | 86 +++++++ .../apis/apiserver/zz_generated.deepcopy.go | 86 +++++++ .../pkg/registry/generic/rest/streamer.go | 1 + staging/src/k8s.io/apiserver/pkg/server/BUILD | 7 + .../src/k8s.io/apiserver/pkg/server/config.go | 7 +- .../apiserver/pkg/server/egress_selector.go | 192 ++++++++++++++++ .../pkg/server/egress_selector_test.go | 183 +++++++++++++++ .../apiserver/pkg/server/egressselector/BUILD | 42 ++++ .../pkg/server/egressselector/config.go | 179 +++++++++++++++ .../pkg/server/egressselector/config_test.go | 214 ++++++++++++++++++ .../k8s.io/apiserver/pkg/server/options/BUILD | 3 + .../pkg/server/options/egress_selector.go | 92 ++++++++ .../pkg/server/options/recommended.go | 8 + .../apis/example/v1/zz_generated.defaults.go | 32 +++ staging/src/k8s.io/kube-aggregator/go.sum | 1 + staging/src/k8s.io/sample-apiserver/go.sum | 1 + vendor/modules.txt | 1 + 37 files changed, 1839 insertions(+), 5 deletions(-) create mode 100644 cluster/gce/addons/konnectivity-agent/daemonset.yaml create mode 100644 cluster/gce/manifests/konnectivity-server.yaml create mode 100644 staging/src/k8s.io/apiserver/pkg/server/egress_selector.go create mode 100644 staging/src/k8s.io/apiserver/pkg/server/egress_selector_test.go create mode 100644 staging/src/k8s.io/apiserver/pkg/server/egressselector/BUILD create mode 100644 staging/src/k8s.io/apiserver/pkg/server/egressselector/config.go create mode 100644 staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go create mode 100644 staging/src/k8s.io/apiserver/pkg/server/options/egress_selector.go create mode 100644 staging/src/k8s.io/code-generator/_examples/HyphenGroup/apis/example/v1/zz_generated.defaults.go diff --git a/build/lib/release.sh b/build/lib/release.sh index 4c5e0962be3..73c0bcca62d 100644 --- a/build/lib/release.sh +++ b/build/lib/release.sh @@ -432,6 +432,7 @@ function kube::release::package_kube_manifests_tarball() { cp "${src_dir}/etcd.manifest" "${dst_dir}" cp "${src_dir}/kube-scheduler.manifest" "${dst_dir}" cp "${src_dir}/kube-apiserver.manifest" "${dst_dir}" + cp "${src_dir}/konnectivity-server.yaml" "${dst_dir}" cp "${src_dir}/abac-authz-policy.jsonl" "${dst_dir}" cp "${src_dir}/kube-controller-manager.manifest" "${dst_dir}" cp "${src_dir}/kube-addon-manager.yaml" "${dst_dir}" diff --git a/cluster/gce/addons/konnectivity-agent/daemonset.yaml b/cluster/gce/addons/konnectivity-agent/daemonset.yaml new file mode 100644 index 00000000000..1edd52c511a --- /dev/null +++ b/cluster/gce/addons/konnectivity-agent/daemonset.yaml @@ -0,0 +1,66 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + addonmanager.kubernetes.io/mode: Reconcile + k8s-app: konnectivity-agent + namespace: kube-system + name: konnectivity-agent +spec: + selector: + matchLabels: + k8s-app: konnectivity-agent + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: konnectivity-agent + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + priorityClassName: system-cluster-critical + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + hostNetwork: true + volumes: + - name: pki + hostPath: + path: /etc/srv/kubernetes/pki/konnectivity-agent + containers: + - image: gcr.io/google-containers/proxy-agent:v0.0.3 + name: konnectivity-agent + command: ["/proxy-agent"] + args: [ + "--logtostderr=true", + "--ca-cert=/etc/srv/kubernetes/pki/konnectivity-agent/ca.crt", + "--agent-cert=/etc/srv/kubernetes/pki/konnectivity-agent/client.crt", + "--agent-key=/etc/srv/kubernetes/pki/konnectivity-agent/client.key", + "--proxy-server-host=__APISERVER_IP__", + "--proxy-server-port=8132" + ] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + limits: + cpu: 50m + memory: 30Mi + livenessProbe: + httpGet: + host: 127.0.0.1 + port: 8093 + path: /healthz + initialDelaySeconds: 15 + timeoutSeconds: 15 + volumeMounts: + - name: pki + mountPath: /etc/srv/kubernetes/pki/konnectivity-agent diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index fe68c3816ab..fbea33f15ed 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -480,3 +480,6 @@ WINDOWS_NODE_TAINTS="${WINDOWS_NODE_TAINTS:-node.kubernetes.io/os=win1809:NoSche # Whether to set up a private GCE cluster, i.e. a cluster where nodes have only private IPs. GCE_PRIVATE_CLUSTER="${KUBE_GCE_PRIVATE_CLUSTER:-false}" + +# Optional: Create apiserver konnectivity server and agent. +ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE="${KUBE_ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index c9ebe210c70..53dc09f076d 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -495,6 +495,16 @@ function create-node-pki { KUBELET_KEY_PATH="${pki_dir}/kubelet.key" write-pki-data "${KUBELET_KEY}" "${KUBELET_KEY_PATH}" fi + + if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + mkdir -p "${pki_dir}/konnectivity-agent" + KONNECTIVITY_AGENT_CA_CERT_PATH="${pki_dir}/konnectivity-agent/ca.crt" + KONNECTIVITY_AGENT_CLIENT_KEY_PATH="${pki_dir}/konnectivity-agent/client.key" + KONNECTIVITY_AGENT_CLIENT_CERT_PATH="${pki_dir}/konnectivity-agent/client.crt" + write-pki-data "${KONNECTIVITY_AGENT_CA_CERT}" "${KONNECTIVITY_AGENT_CA_CERT_PATH}" + write-pki-data "${KONNECTIVITY_AGENT_CLIENT_KEY}" "${KONNECTIVITY_AGENT_CLIENT_KEY_PATH}" + write-pki-data "${KONNECTIVITY_AGENT_CLIENT_CERT}" "${KONNECTIVITY_AGENT_CLIENT_CERT_PATH}" + fi } function create-master-pki { @@ -558,6 +568,42 @@ function create-master-pki { PROXY_CLIENT_CERT_PATH="${pki_dir}/proxy_client.crt" write-pki-data "${PROXY_CLIENT_CERT}" "${PROXY_CLIENT_CERT_PATH}" fi + + if [[ ! -z "${KONNECTIVITY_SERVER_CA_CERT:-}" ]]; then + mkdir -p "${pki_dir}"/konnectivity-server + #KONNECTIVITY_SERVER_CA_KEY_PATH="${pki_dir}/konnectivity-server/ca.key" + #write-pki-data "${KONNECTIVITY_SERVER_CA_KEY}" "${KONNECTIVITY_SERVER_CA_KEY_PATH}" + + KONNECTIVITY_SERVER_CA_CERT_PATH="${pki_dir}/konnectivity-server/ca.crt" + write-pki-data "${KONNECTIVITY_SERVER_CA_CERT}" "${KONNECTIVITY_SERVER_CA_CERT_PATH}" + + KONNECTIVITY_SERVER_KEY_PATH="${pki_dir}/konnectivity-server/server.key" + write-pki-data "${KONNECTIVITY_SERVER_KEY}" "${KONNECTIVITY_SERVER_KEY_PATH}" + + KONNECTIVITY_SERVER_CERT_PATH="${pki_dir}/konnectivity-server/server.crt" + write-pki-data "${KONNECTIVITY_SERVER_CERT}" "${KONNECTIVITY_SERVER_CERT_PATH}" + + KONNECTIVITY_SERVER_CLIENT_KEY_PATH="${pki_dir}/konnectivity-server/client.key" + write-pki-data "${KONNECTIVITY_SERVER_CLIENT_KEY}" "${KONNECTIVITY_SERVER_CLIENT_KEY_PATH}" + + KONNECTIVITY_SERVER_CLIENT_CERT_PATH="${pki_dir}/konnectivity-server/client.crt" + write-pki-data "${KONNECTIVITY_SERVER_CLIENT_CERT}" "${KONNECTIVITY_SERVER_CLIENT_CERT_PATH}" + fi + + if [[ ! -z "${KONNECTIVITY_AGENT_CA_CERT:-}" ]]; then + mkdir -p "${pki_dir}"/konnectivity-agent + KONNECTIVITY_AGENT_CA_KEY_PATH="${pki_dir}/konnectivity-agent/ca.key" + write-pki-data "${KONNECTIVITY_AGENT_CA_KEY}" "${KONNECTIVITY_AGENT_CA_KEY_PATH}" + + KONNECTIVITY_AGENT_CA_CERT_PATH="${pki_dir}/konnectivity-agent/ca.crt" + write-pki-data "${KONNECTIVITY_AGENT_CA_CERT}" "${KONNECTIVITY_AGENT_CA_CERT_PATH}" + + KONNECTIVITY_AGENT_KEY_PATH="${pki_dir}/konnectivity-agent/server.key" + write-pki-data "${KONNECTIVITY_AGENT_KEY}" "${KONNECTIVITY_AGENT_KEY_PATH}" + + KONNECTIVITY_AGENT_CERT_PATH="${pki_dir}/konnectivity-agent/server.crt" + write-pki-data "${KONNECTIVITY_AGENT_CERT}" "${KONNECTIVITY_AGENT_CERT_PATH}" + fi } # After the first boot and on upgrade, these files exist on the master-pd @@ -757,6 +803,27 @@ contexts: name: webhook EOF fi + if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + cat </etc/srv/kubernetes/egress_selector_configuration.yaml +apiVersion: apiserver.k8s.io/v1alpha1 +kind: EgressSelectorConfiguration +egressSelections: +- name: cluster + connection: + type: http-connect + httpConnect: + url: https://127.0.0.1:8131 + caBundle: /etc/srv/kubernetes/pki/konnectivity-server/ca.crt + clientKey: /etc/srv/kubernetes/pki/konnectivity-server/client.key + clientCert: /etc/srv/kubernetes/pki/konnectivity-server/client.crt +- name: master + connection: + type: direct +- name: etcd + connection: + type: direct +EOF + fi if [[ -n "${WEBHOOK_GKE_EXEC_AUTH:-}" ]]; then if [[ -z "${EXEC_AUTH_PLUGIN_URL:-}" ]]; then @@ -1214,6 +1281,10 @@ function create-master-etcd-apiserver-auth { fi } +function create-master-konnectivity-server-apiserver-auth { + echo TODO: implement create-master-konnectivity-server-apiserver-auth +} + function assemble-docker-flags { echo "Assemble docker command line flags" local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false" @@ -1562,6 +1633,49 @@ function start-etcd-servers { prepare-etcd-manifest "-events" "4002" "2381" "100m" "etcd-events.manifest" } +# Replaces the variables in the konnectivity-server manifest file with the real values, and then +# copy the file to the manifest dir +# $1: value for variable "server_port" +# $2: value for variable "agent_port" +# $3: value for bariable "admin_port" +function prepare-konnectivity-server-manifest { + local -r temp_file="/tmp/konnectivity-server.yaml" + params=() + cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/konnectivity-server.yaml" "${temp_file}" + params+=("--log-file=/var/log/konnectivity-server.log") + params+=("--logtostderr=false") + params+=("--log-file-max-size=0") + params+=("--server-ca-cert=${KONNECTIVITY_SERVER_CA_CERT_PATH}") + params+=("--server-cert=${KONNECTIVITY_SERVER_CERT_PATH}") + params+=("--server-key=${KONNECTIVITY_SERVER_KEY_PATH}") + params+=("--cluster-ca-cert=${KONNECTIVITY_AGENT_CA_CERT_PATH}") + params+=("--cluster-cert=${KONNECTIVITY_AGENT_CERT_PATH}") + params+=("--cluster-key=${KONNECTIVITY_AGENT_KEY_PATH}") + params+=("--mode=http-connect") + params+=("--server-port=$1") + params+=("--agent-port=$2") + params+=("--admin-port=$3") + konnectivity_args="" + for param in "${params[@]}"; do + konnectivity_args+=", \"${param}\"" + done + sed -i -e "s@{{ *konnectivity_args *}}@${konnectivity_args}@g" "${temp_file}" + sed -i -e "s@{{ *server_port *}}@$1@g" "${temp_file}" + sed -i -e "s@{{ *agent_port *}}@$2@g" "${temp_file}" + sed -i -e "s@{{ *admin_port *}}@$3@g" "${temp_file}" + sed -i -e "s@{{ *liveness_probe_initial_delay *}}@30@g" "${temp_file}" + mv "${temp_file}" /etc/kubernetes/manifests +} + +# Starts konnectivity server pod. +# More specifically, it prepares dirs and files, sets the variable value +# in the manifests, and copies them to /etc/kubernetes/manifests. +function start-konnectivity-server { + echo "Start konnectivity server pods" + prepare-log-file /var/log/konnectivity-server.log + prepare-konnectivity-server-manifest "8131" "8132" "8133" +} + # Calculates the following variables based on env variables, which will be used # by the manifests of several kube-master components. # CLOUD_CONFIG_OPT @@ -1905,6 +2019,15 @@ function start-kube-apiserver { authorization_mode="Node,${authorization_mode}" params+=" --authorization-mode=${authorization_mode}" + local csc_config_mount="" + local csc_config_volume="" + if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + # Create the EgressSelectorConfiguration yaml file to control the Egress Selector. + csc_config_mount="{\"name\": \"cscconfigmount\",\"mountPath\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"readOnly\": false}," + csc_config_volume="{\"name\": \"cscconfigmount\",\"hostPath\": {\"path\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"type\": \"FileOrCreate\"}}," + params+=" --egress-selector-config-file=/etc/srv/kubernetes/egress_selector_configuration.yaml" + fi + local container_env="" if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then container_env+="{\"name\": \"KUBE_CACHE_MUTATION_DETECTOR\", \"value\": \"${ENABLE_CACHE_MUTATION_DETECTOR}\"}" @@ -1943,6 +2066,8 @@ function start-kube-apiserver { sed -i -e "s@{{webhook_authn_config_volume}}@${webhook_authn_config_volume}@g" "${src_file}" sed -i -e "s@{{webhook_config_mount}}@${webhook_config_mount}@g" "${src_file}" sed -i -e "s@{{webhook_config_volume}}@${webhook_config_volume}@g" "${src_file}" + sed -i -e "s@{{csc_config_mount}}@${csc_config_mount}@g" "${src_file}" + sed -i -e "s@{{csc_config_volume}}@${csc_config_volume}@g" "${src_file}" sed -i -e "s@{{audit_policy_config_mount}}@${audit_policy_config_mount}@g" "${src_file}" sed -i -e "s@{{audit_policy_config_volume}}@${audit_policy_config_volume}@g" "${src_file}" sed -i -e "s@{{audit_webhook_config_mount}}@${audit_webhook_config_mount}@g" "${src_file}" @@ -2675,6 +2800,11 @@ EOF setup-addon-manifests "addons" "node-termination-handler" setup-node-termination-handler-manifest fi + # Setting up the konnectivity-agent daemonset + if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + setup-addon-manifests "addons" "konnectivity-agent" + setup-konnectivity-agent-manifest + fi if [[ "${ENABLE_CLUSTER_DNS:-}" == "true" ]]; then # Create a new directory for the DNS addon and prepend a "0" on the name. # Prepending "0" to the directory ensures that add-on manager @@ -2773,6 +2903,11 @@ function setup-node-termination-handler-manifest { fi } +function setup-konnectivity-agent-manifest { + local -r manifest="/etc/kubernetes/addons/konnectivity-agent/daemonset.yaml" + sed -i "s|__APISERVER_IP__|${KUBERNETES_MASTER_NAME}|g" "${manifest}" +} + # Setups manifests for ingress controller and gce-specific policies for service controller. function start-lb-controller { setup-addon-manifests "addons" "loadbalancing" @@ -3055,6 +3190,9 @@ function main() { create-master-pki create-master-auth ensure-master-bootstrap-kubectl-auth + if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + create-master-konnectivity-server-apiserver-auth + fi create-master-kubelet-auth create-master-etcd-auth create-master-etcd-apiserver-auth @@ -3088,6 +3226,9 @@ function main() { start-etcd-empty-dir-cleanup-pod fi start-kube-apiserver + if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + start-konnectivity-server + fi start-kube-controller-manager start-kube-scheduler wait-till-apiserver-ready diff --git a/cluster/gce/manifests/BUILD b/cluster/gce/manifests/BUILD index 2af9c9b46d8..9c7379aa26b 100644 --- a/cluster/gce/manifests/BUILD +++ b/cluster/gce/manifests/BUILD @@ -18,6 +18,7 @@ filegroup( "etcd.manifest", "etcd-empty-dir-cleanup.yaml", "glbc.manifest", + "konnectivity-server.yaml", "kube-addon-manager.yaml", "kube-apiserver.manifest", "kube-controller-manager.manifest", diff --git a/cluster/gce/manifests/konnectivity-server.yaml b/cluster/gce/manifests/konnectivity-server.yaml new file mode 100644 index 00000000000..59236a579b5 --- /dev/null +++ b/cluster/gce/manifests/konnectivity-server.yaml @@ -0,0 +1,58 @@ +apiVersion: v1 +kind: Pod +metadata: + name: konnectivity-server + namespace: kube-system + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + seccomp.security.alpha.kubernetes.io/pod: 'docker/default' + component: konnectivity-server +spec: + hostNetwork: true + containers: + - name: konnectivity-server-container + image: gcr.io/google-containers/proxy-server:v0.0.3 + resources: + requests: + cpu: 40m + command: [ "/proxy-server"{{ konnectivity_args }} ] + livenessProbe: + httpGet: + scheme: HTTP + host: 127.0.0.1 + port: {{ admin_port }} + path: /healthz + initialDelaySeconds: {{ liveness_probe_initial_delay }} + timeoutSeconds: 60 + ports: + - name: serverport + containerPort: {{ server_port }} + hostPort: {{ server_port }} + - name: agentport + containerPort: {{ agent_port }} + hostPort: {{ agent_port }} + - name: adminport + containerPort: {{ admin_port }} + hostPort: {{ admin_port }} + volumeMounts: + - name: varlogkonnectivityserver + mountPath: /var/log/konnectivity-server.log + readOnly: false + - name: pkiserver + mountPath: /etc/srv/kubernetes/pki/konnectivity-server + readOnly: true + - name: pkiagent + mountPath: /etc/srv/kubernetes/pki/konnectivity-agent + readOnly: true + volumes: + - name: varlogkonnectivityserver + hostPath: + path: /var/log/konnectivity-server.log + type: FileOrCreate + - name: pkiserver + hostPath: + path: /etc/srv/kubernetes/pki/konnectivity-server + - name: pkiagent + hostPath: + path: /etc/srv/kubernetes/pki/konnectivity-agent + diff --git a/cluster/gce/manifests/kube-apiserver.manifest b/cluster/gce/manifests/kube-apiserver.manifest index 4e882da5533..02c3021282f 100644 --- a/cluster/gce/manifests/kube-apiserver.manifest +++ b/cluster/gce/manifests/kube-apiserver.manifest @@ -63,6 +63,7 @@ {{additional_cloud_config_mount}} {{webhook_config_mount}} {{webhook_authn_config_mount}} + {{csc_config_mount}} {{audit_policy_config_mount}} {{audit_webhook_config_mount}} {{webhook_exec_auth_plugin_mount}} @@ -103,6 +104,7 @@ {{additional_cloud_config_volume}} {{webhook_config_volume}} {{webhook_authn_config_volume}} + {{csc_config_volume}} {{audit_policy_config_volume}} {{audit_webhook_config_volume}} {{webhook_exec_auth_plugin_volume}} diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 01140aeecd2..1246b193eb5 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -1066,6 +1066,18 @@ ETCD_APISERVER_SERVER_KEY: $(yaml-quote ${ETCD_APISERVER_SERVER_KEY_BASE64:-}) ETCD_APISERVER_SERVER_CERT: $(yaml-quote ${ETCD_APISERVER_SERVER_CERT_BASE64:-}) ETCD_APISERVER_CLIENT_KEY: $(yaml-quote ${ETCD_APISERVER_CLIENT_KEY_BASE64:-}) ETCD_APISERVER_CLIENT_CERT: $(yaml-quote ${ETCD_APISERVER_CLIENT_CERT_BASE64:-}) +KONNECTIVITY_SERVER_CA_KEY: $(yaml-quote ${KONNECTIVITY_SERVER_CA_KEY_BASE64:-}) +KONNECTIVITY_SERVER_CA_CERT: $(yaml-quote ${KONNECTIVITY_SERVER_CA_CERT_BASE64:-}) +KONNECTIVITY_SERVER_CERT: $(yaml-quote ${KONNECTIVITY_SERVER_CERT_BASE64:-}) +KONNECTIVITY_SERVER_KEY: $(yaml-quote ${KONNECTIVITY_SERVER_KEY_BASE64:-}) +KONNECTIVITY_SERVER_CLIENT_CERT: $(yaml-quote ${KONNECTIVITY_SERVER_CLIENT_CERT_BASE64:-}) +KONNECTIVITY_SERVER_CLIENT_KEY: $(yaml-quote ${KONNECTIVITY_SERVER_CLIENT_KEY_BASE64:-}) +KONNECTIVITY_AGENT_CA_KEY: $(yaml-quote ${KONNECTIVITY_AGENT_CA_KEY_BASE64:-}) +KONNECTIVITY_AGENT_CA_CERT: $(yaml-quote ${KONNECTIVITY_AGENT_CA_CERT_BASE64:-}) +KONNECTIVITY_AGENT_CERT: $(yaml-quote ${KONNECTIVITY_AGENT_CERT_BASE64:-}) +KONNECTIVITY_AGENT_KEY: $(yaml-quote ${KONNECTIVITY_AGENT_KEY_BASE64:-}) +KONNECTIVITY_AGENT_CLIENT_CERT: $(yaml-quote ${KONNECTIVITY_AGENT_CLIENT_CERT_BASE64:-}) +KONNECTIVITY_AGENT_CLIENT_KEY: $(yaml-quote ${KONNECTIVITY_AGENT_CLIENT_KEY_BASE64:-}) EOF } @@ -1206,6 +1218,13 @@ EOF [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "cos" ]]; then cat >>$file <>$file <>$file <>$file <${cert_create_debug_output} || true + cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/aggregator + mkdir easy-rsa-master/konnectivity-server + cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-server + mkdir easy-rsa-master/konnectivity-agent + cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-agent) &>${cert_create_debug_output} || true CERT_DIR="${KUBE_TEMP}/easy-rsa-master/easyrsa3" AGGREGATOR_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/aggregator" + KONNECTIVITY_SERVER_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-server" + KONNECTIVITY_AGENT_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-agent" if [ ! -x "${CERT_DIR}/easyrsa" -o ! -x "${AGGREGATOR_CERT_DIR}/easyrsa" ]; then # TODO(roberthbailey,porridge): add better error handling here, # see https://github.com/kubernetes/kubernetes/issues/55229 @@ -1726,7 +1773,92 @@ function generate-aggregator-certs { fi } +# Runs the easy RSA commands to generate server side certificate files +# for the konnectivity server. This includes both server side to both +# konnectivity-server and konnectivity-agent. +# The generated files are in ${KONNECTIVITY_SERVER_CERT_DIR} and +# ${KONNECTIVITY_AGENT_CERT_DIR} # +# Assumed vars +# KUBE_TEMP +# KONNECTIVITY_SERVER_CERT_DIR +# KONNECTIVITY_SERVER_PRIMARY_CN: Primary canonical name +# KONNECTIVITY_SERVER_SANS: Subject alternate names +# +function generate-konnectivity-server-certs { + local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX") + # Note: This was heavily cribbed from make-ca-cert.sh + (set -x + # Make the client <-> konnectivity server side certificates. + cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-server" + ./easyrsa init-pki + # this puts the cert into pki/ca.crt and the key into pki/private/ca.key + ./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass + ./easyrsa --subject-alt-name="IP:127.0.0.1,${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass + ./easyrsa build-client-full client nopass + + kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl" + + # make the config for the signer + echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json" + # create the konnectivity server cert with the correct groups + echo '{"CN":"konnectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-server + rm -f "konnectivity-server.csr" + + # Make the agent <-> konnectivity server side certificates. + cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-agent" + ./easyrsa init-pki + # this puts the cert into pki/ca.crt and the key into pki/private/ca.key + ./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass + ./easyrsa --subject-alt-name="${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass + ./easyrsa build-client-full client nopass + + kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl" + + # make the config for the signer + echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","agent auth"]}}}' > "ca-config.json" + # create the konnectivity server cert with the correct groups + echo '{"CN":"koonectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent + rm -f "konnectivity-agent.csr" + + echo `ls ${KONNECTIVITY_SERVER_CERT_DIR}/pki/` + echo `ls ${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/` + echo `ls ${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/` + echo `ls ${KONNECTIVITY_AGENT_CERT_DIR}/pki/` + echo `ls ${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/` + echo `ls ${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/` + echo "completed main certificate section") &>${cert_create_debug_output} || true + + local output_file_missing=0 + local output_file + for output_file in \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/ca.key" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/ca.crt" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/server.crt" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/server.key" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/client.crt" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/client.key" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/ca.key" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/ca.crt" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/server.crt" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key" + do + if [[ ! -s "${output_file}" ]]; then + echo "Expected file ${output_file} not created" >&2 + output_file_missing=1 + fi + done + if (( $output_file_missing )); then + # TODO(roberthbailey,porridge): add better error handling here, + # see https://github.com/kubernetes/kubernetes/issues/55229 + cat "${cert_create_debug_output}" >&2 + echo "=== Failed to generate konnectivity-server certificates: Aborting ===" >&2 + exit 2 + fi +} + # Using provided master env, extracts value from provided key. # # Args: @@ -1766,6 +1898,16 @@ function parse-master-env() { ETCD_APISERVER_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_SERVER_CERT") ETCD_APISERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_KEY") ETCD_APISERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_CERT") + KONNECTIVITY_SERVER_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_KEY") + KONNECTIVITY_SERVER_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_CERT") + KONNECTIVITY_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CERT") + KONNECTIVITY_SERVER_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_KEY") + KONNECTIVITY_SERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_CERT") + KONNECTIVITY_SERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_KEY") + KONNECTIVITY_AGENT_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_KEY") + KONNECTIVITY_AGENT_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_CERT") + KONNECTIVITY_AGENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CERT") + KONNECTIVITY_AGENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_KEY") } # Update or verify required gcloud components are installed @@ -2505,6 +2647,15 @@ function create-master() { --target-tags "${MASTER_TAG}" \ --allow tcp:443 & + echo "Configuring firewall for apiserver konnectivity server" + if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + gcloud compute firewall-rules create "${MASTER_NAME}-konnectivity-server" \ + --project "${NETWORK_PROJECT}" \ + --network "${NETWORK}" \ + --target-tags "${MASTER_TAG}" \ + --allow tcp:8132 & + fi + # We have to make sure the disk is created before creating the master VM, so # run this in the foreground. gcloud compute disks create "${MASTER_NAME}-pd" \ @@ -3197,7 +3348,7 @@ function kube-down() { # If there are no more remaining master replicas, we should delete all remaining network resources. if [[ "${REMAINING_MASTER_COUNT}" -eq 0 ]]; then # Delete firewall rule for the master, etcd servers, and nodes. - delete-firewall-rules "${MASTER_NAME}-https" "${MASTER_NAME}-etcd" "${NODE_TAG}-all" + delete-firewall-rules "${MASTER_NAME}-https" "${MASTER_NAME}-etcd" "${NODE_TAG}-all" "${MASTER_NAME}-konnectivity-server" # Delete the master's reserved IP if gcloud compute addresses describe "${MASTER_NAME}-ip" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then gcloud compute addresses delete \ diff --git a/cmd/kube-apiserver/app/options/options.go b/cmd/kube-apiserver/app/options/options.go index f89910ac46b..5ea40f7cdfc 100644 --- a/cmd/kube-apiserver/app/options/options.go +++ b/cmd/kube-apiserver/app/options/options.go @@ -48,6 +48,7 @@ type ServerRunOptions struct { Authorization *kubeoptions.BuiltInAuthorizationOptions CloudProvider *kubeoptions.CloudProviderOptions APIEnablement *genericoptions.APIEnablementOptions + EgressSelector *genericoptions.EgressSelectorOptions AllowPrivileged bool EnableLogsHandler bool @@ -87,6 +88,7 @@ func NewServerRunOptions() *ServerRunOptions { Authorization: kubeoptions.NewBuiltInAuthorizationOptions(), CloudProvider: kubeoptions.NewCloudProviderOptions(), APIEnablement: genericoptions.NewAPIEnablementOptions(), + EgressSelector: genericoptions.NewEgressSelectorOptions(), EnableLogsHandler: true, EventTTL: 1 * time.Hour, @@ -134,6 +136,7 @@ func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) { s.Authorization.AddFlags(fss.FlagSet("authorization")) s.CloudProvider.AddFlags(fss.FlagSet("cloud provider")) s.APIEnablement.AddFlags(fss.FlagSet("api enablement")) + s.EgressSelector.AddFlags(fss.FlagSet("egress selector")) s.Admission.AddFlags(fss.FlagSet("admission")) // Note: the weird ""+ in below lines seems to be the only way to get gofmt to diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index 917a5661f0f..8cf22957405 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -94,6 +94,7 @@ func TestAddFlags(t *testing.T) { "--cloud-provider=azure", "--cors-allowed-origins=10.10.10.100,10.10.10.200", "--contention-profiling=true", + "--egress-selector-config-file=/var/run/kubernetes/egress-selector/connectivity.yaml", "--enable-aggregator-routing=true", "--enable-logs-handler=false", "--endpoint-reconciler-type=" + string(reconcilers.LeaseEndpointReconcilerType), @@ -293,6 +294,9 @@ func TestAddFlags(t *testing.T) { APIEnablement: &apiserveroptions.APIEnablementOptions{ RuntimeConfig: cliflag.ConfigurationMap{}, }, + EgressSelector: &apiserveroptions.EgressSelectorOptions{ + ConfigFile: "/var/run/kubernetes/egress-selector/connectivity.yaml", + }, EnableLogsHandler: false, EnableAggregatorRouting: true, ProxyClientKeyFile: "/var/run/kubernetes/proxy.key", diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 490dad60c18..43782b4b7c8 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -362,6 +362,10 @@ func CreateKubeAPIServerConfig( // Use the nodeTunneler's dialer to connect to the kubelet config.ExtraConfig.KubeletClientConfig.Dial = nodeTunneler.Dial } + if config.GenericConfig.EgressSelector != nil { + // Use the config.GenericConfig.EgressSelector lookup to find the dialer to connect to the kubelet + config.ExtraConfig.KubeletClientConfig.Lookup = config.GenericConfig.EgressSelector.Lookup + } return } @@ -402,6 +406,9 @@ func buildGenericConfig( if lastErr = s.APIEnablement.ApplyTo(genericConfig, master.DefaultAPIResourceConfigSource(), legacyscheme.Scheme); lastErr != nil { return } + if lastErr = s.EgressSelector.ApplyTo(genericConfig); lastErr != nil { + return + } genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme)) genericConfig.OpenAPIConfig.Info.Title = "Kubernetes" diff --git a/pkg/kubelet/client/BUILD b/pkg/kubelet/client/BUILD index f6791f4f780..36b6a282919 100644 --- a/pkg/kubelet/client/BUILD +++ b/pkg/kubelet/client/BUILD @@ -16,6 +16,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/transport:go_default_library", ], diff --git a/pkg/kubelet/client/kubelet_client.go b/pkg/kubelet/client/kubelet_client.go index bde19b76e66..3a6e7aee8e8 100644 --- a/pkg/kubelet/client/kubelet_client.go +++ b/pkg/kubelet/client/kubelet_client.go @@ -18,6 +18,7 @@ package client import ( "context" + "fmt" "net/http" "strconv" "time" @@ -26,6 +27,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apiserver/pkg/server" restclient "k8s.io/client-go/rest" "k8s.io/client-go/transport" nodeutil "k8s.io/kubernetes/pkg/util/node" @@ -51,6 +53,9 @@ type KubeletClientConfig struct { // Dial is a custom dialer used for the client Dial utilnet.DialFunc + + // Lookup will give us a dialer if the egress selector is configured for it + Lookup server.EgressSelectorLookup } // ConnectionInfo provides the information needed to connect to a kubelet @@ -73,9 +78,20 @@ func MakeTransport(config *KubeletClientConfig) (http.RoundTripper, error) { } rt := http.DefaultTransport - if config.Dial != nil || tlsConfig != nil { + dialer := config.Dial + if dialer == nil && config.Lookup != nil { + // Assuming EgressSelector if SSHTunnel is not turned on. + // We will not get a dialer if egress selector is disabled. + networkContext := server.NetworkContext{EgressSelectionName: server.Cluster} + dialer, err = config.Lookup(networkContext) + if err != nil { + return nil, fmt.Errorf("failed to get context dialer for 'cluster': got %v", err) + } + } + if dialer != nil || tlsConfig != nil { + // If SSH Tunnel is turned on rt = utilnet.SetOldTransportDefaults(&http.Transport{ - DialContext: config.Dial, + DialContext: dialer, TLSClientConfig: tlsConfig, }) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/go.sum b/staging/src/k8s.io/apiextensions-apiserver/go.sum index 0c965371948..9b9702714d3 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/go.sum +++ b/staging/src/k8s.io/apiextensions-apiserver/go.sum @@ -215,6 +215,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/soheilhy/cmux v0.1.3 h1:09wy7WZk4AqO03yH85Ex1X+Uo3vDsil3Fa9AgF8Emss= github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.4 h1:S0tLZ3VOKl2Te0hpq8+ke0eSJPfCnNTPiDlsfwi1/NE= diff --git a/staging/src/k8s.io/apiserver/go.sum b/staging/src/k8s.io/apiserver/go.sum index b15f3d30c15..2744ecbeab0 100644 --- a/staging/src/k8s.io/apiserver/go.sum +++ b/staging/src/k8s.io/apiserver/go.sum @@ -156,6 +156,7 @@ github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/soheilhy/cmux v0.1.3 h1:09wy7WZk4AqO03yH85Ex1X+Uo3vDsil3Fa9AgF8Emss= github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/register.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/register.go index ffe9942a6cc..15519d1c47b 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/register.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/register.go @@ -45,6 +45,7 @@ var ( func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, &AdmissionConfiguration{}, + &EgressSelectorConfiguration{}, ) return nil } diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go index e55da95f95d..d21e7d631d7 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go @@ -48,3 +48,52 @@ type AdmissionPluginConfiguration struct { // +optional Configuration *runtime.Unknown } + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// EgressSelectorConfiguration provides versioned configuration for egress selector clients. +type EgressSelectorConfiguration struct { + metav1.TypeMeta + + // EgressSelections contains a list of egress selection client configurations + EgressSelections []EgressSelection +} + +// EgressSelection provides the configuration for a single egress selection client. +type EgressSelection struct { + // Name is the name of the egress selection. + // Currently supported values are "Master", "Etcd" and "Cluster" + Name string + + // Connection is the exact information used to configure the egress selection + Connection Connection +} + +// Connection provides the configuration for a single egress selection client. +type Connection struct { + // Type is the type of connection used to connect from client to konnectivity server. + // Currently supported values are "http-connect" and "direct". + Type string + + // httpConnect is the config needed to use http-connect to the konnectivity server. + // +optional + HTTPConnect *HTTPConnectConfig +} + +type HTTPConnectConfig struct { + // URL is the location of the konnectivity server to connect to. + // As an example it might be "https://127.0.0.1:8131" + URL string + + // CABundle is the file location of the CA to be used to determine trust with the konnectivity server. + // +optional + CABundle string + + // ClientKey is the file location of the client key to be used in mtls handshakes with the konnectivity server. + // +optional + ClientKey string + + // ClientCert is the file location of the client certificate to be used in mtls handshakes with the konnectivity server. + // +optional + ClientCert string +} diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/register.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/register.go index 466b19ae5bb..758be628a79 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/register.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/register.go @@ -46,6 +46,7 @@ func init() { func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, &AdmissionConfiguration{}, + &EgressSelectorConfiguration{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go index 239b8e20e04..10034d7c340 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go @@ -48,3 +48,63 @@ type AdmissionPluginConfiguration struct { // +optional Configuration *runtime.Unknown `json:"configuration"` } + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// EgressSelectorConfiguration provides versioned configuration for egress selector clients. +type EgressSelectorConfiguration struct { + metav1.TypeMeta `json:",inline"` + + // connectionServices contains a list of egress selection client configurations + EgressSelections []EgressSelection `json:"egressSelections"` +} + +// EgressSelection provides the configuration for a single egress selection client. +type EgressSelection struct { + // name is the name of the egress selection. + // Currently supported values are "Master", "Etcd" and "Cluster" + Name string `json:"name"` + + // connection is the exact information used to configure the egress selection + Connection Connection `json:"connection"` +} + +// Connection provides the configuration for a single egress selection client. +type Connection struct { + // type is the type of connection used to connect from client to network/konnectivity server. + // Currently supported values are "http-connect" and "direct". + Type string `json:"type"` + + // httpConnect is the config needed to use http-connect to the konnectivity server. + // Absence when the type is "http-connect" will cause an error + // Presence when the type is "direct" will also cause an error + // +optional + HTTPConnect *HTTPConnectConfig `json:"httpConnect,omitempty"` +} + +type HTTPConnectConfig struct { + // url is the location of the proxy server to connect to. + // As an example it might be "https://127.0.0.1:8131" + URL string `json:"url"` + + // caBundle is the file location of the CA to be used to determine trust with the konnectivity server. + // Must be absent/empty http-connect using the plain http + // Must be configured for http-connect using the https protocol + // Misconfiguration will cause an error + // +optional + CABundle string `json:"caBundle,omitempty"` + + // clientKey is the file location of the client key to be used in mtls handshakes with the konnectivity server. + // Must be absent/empty http-connect using the plain http + // Must be configured for http-connect using the https protocol + // Misconfiguration will cause an error + // +optional + ClientKey string `json:"clientKey,omitempty"` + + // clientCert is the file location of the client certificate to be used in mtls handshakes with the konnectivity server. + // Must be absent/empty http-connect using the plain http + // Must be configured for http-connect using the https protocol + // Misconfiguration will cause an error + // +optional + ClientCert string `json:"clientCert,omitempty"` +} diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go index 64909b34a8f..80352f02e91 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go @@ -55,6 +55,46 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*Connection)(nil), (*apiserver.Connection)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_Connection_To_apiserver_Connection(a.(*Connection), b.(*apiserver.Connection), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*apiserver.Connection)(nil), (*Connection)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_apiserver_Connection_To_v1alpha1_Connection(a.(*apiserver.Connection), b.(*Connection), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*EgressSelection)(nil), (*apiserver.EgressSelection)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_EgressSelection_To_apiserver_EgressSelection(a.(*EgressSelection), b.(*apiserver.EgressSelection), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*apiserver.EgressSelection)(nil), (*EgressSelection)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_apiserver_EgressSelection_To_v1alpha1_EgressSelection(a.(*apiserver.EgressSelection), b.(*EgressSelection), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*EgressSelectorConfiguration)(nil), (*apiserver.EgressSelectorConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration(a.(*EgressSelectorConfiguration), b.(*apiserver.EgressSelectorConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*apiserver.EgressSelectorConfiguration)(nil), (*EgressSelectorConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration(a.(*apiserver.EgressSelectorConfiguration), b.(*EgressSelectorConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*HTTPConnectConfig)(nil), (*apiserver.HTTPConnectConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(a.(*HTTPConnectConfig), b.(*apiserver.HTTPConnectConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*apiserver.HTTPConnectConfig)(nil), (*HTTPConnectConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(a.(*apiserver.HTTPConnectConfig), b.(*HTTPConnectConfig), scope) + }); err != nil { + return err + } return nil } @@ -101,3 +141,97 @@ func autoConvert_apiserver_AdmissionPluginConfiguration_To_v1alpha1_AdmissionPlu func Convert_apiserver_AdmissionPluginConfiguration_To_v1alpha1_AdmissionPluginConfiguration(in *apiserver.AdmissionPluginConfiguration, out *AdmissionPluginConfiguration, s conversion.Scope) error { return autoConvert_apiserver_AdmissionPluginConfiguration_To_v1alpha1_AdmissionPluginConfiguration(in, out, s) } + +func autoConvert_v1alpha1_Connection_To_apiserver_Connection(in *Connection, out *apiserver.Connection, s conversion.Scope) error { + out.Type = in.Type + out.HTTPConnect = (*apiserver.HTTPConnectConfig)(unsafe.Pointer(in.HTTPConnect)) + return nil +} + +// Convert_v1alpha1_Connection_To_apiserver_Connection is an autogenerated conversion function. +func Convert_v1alpha1_Connection_To_apiserver_Connection(in *Connection, out *apiserver.Connection, s conversion.Scope) error { + return autoConvert_v1alpha1_Connection_To_apiserver_Connection(in, out, s) +} + +func autoConvert_apiserver_Connection_To_v1alpha1_Connection(in *apiserver.Connection, out *Connection, s conversion.Scope) error { + out.Type = in.Type + out.HTTPConnect = (*HTTPConnectConfig)(unsafe.Pointer(in.HTTPConnect)) + return nil +} + +// Convert_apiserver_Connection_To_v1alpha1_Connection is an autogenerated conversion function. +func Convert_apiserver_Connection_To_v1alpha1_Connection(in *apiserver.Connection, out *Connection, s conversion.Scope) error { + return autoConvert_apiserver_Connection_To_v1alpha1_Connection(in, out, s) +} + +func autoConvert_v1alpha1_EgressSelection_To_apiserver_EgressSelection(in *EgressSelection, out *apiserver.EgressSelection, s conversion.Scope) error { + out.Name = in.Name + if err := Convert_v1alpha1_Connection_To_apiserver_Connection(&in.Connection, &out.Connection, s); err != nil { + return err + } + return nil +} + +// Convert_v1alpha1_EgressSelection_To_apiserver_EgressSelection is an autogenerated conversion function. +func Convert_v1alpha1_EgressSelection_To_apiserver_EgressSelection(in *EgressSelection, out *apiserver.EgressSelection, s conversion.Scope) error { + return autoConvert_v1alpha1_EgressSelection_To_apiserver_EgressSelection(in, out, s) +} + +func autoConvert_apiserver_EgressSelection_To_v1alpha1_EgressSelection(in *apiserver.EgressSelection, out *EgressSelection, s conversion.Scope) error { + out.Name = in.Name + if err := Convert_apiserver_Connection_To_v1alpha1_Connection(&in.Connection, &out.Connection, s); err != nil { + return err + } + return nil +} + +// Convert_apiserver_EgressSelection_To_v1alpha1_EgressSelection is an autogenerated conversion function. +func Convert_apiserver_EgressSelection_To_v1alpha1_EgressSelection(in *apiserver.EgressSelection, out *EgressSelection, s conversion.Scope) error { + return autoConvert_apiserver_EgressSelection_To_v1alpha1_EgressSelection(in, out, s) +} + +func autoConvert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration(in *EgressSelectorConfiguration, out *apiserver.EgressSelectorConfiguration, s conversion.Scope) error { + out.EgressSelections = *(*[]apiserver.EgressSelection)(unsafe.Pointer(&in.EgressSelections)) + return nil +} + +// Convert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration is an autogenerated conversion function. +func Convert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration(in *EgressSelectorConfiguration, out *apiserver.EgressSelectorConfiguration, s conversion.Scope) error { + return autoConvert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration(in, out, s) +} + +func autoConvert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration(in *apiserver.EgressSelectorConfiguration, out *EgressSelectorConfiguration, s conversion.Scope) error { + out.EgressSelections = *(*[]EgressSelection)(unsafe.Pointer(&in.EgressSelections)) + return nil +} + +// Convert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration is an autogenerated conversion function. +func Convert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration(in *apiserver.EgressSelectorConfiguration, out *EgressSelectorConfiguration, s conversion.Scope) error { + return autoConvert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration(in, out, s) +} + +func autoConvert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(in *HTTPConnectConfig, out *apiserver.HTTPConnectConfig, s conversion.Scope) error { + out.URL = in.URL + out.CABundle = in.CABundle + out.ClientKey = in.ClientKey + out.ClientCert = in.ClientCert + return nil +} + +// Convert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig is an autogenerated conversion function. +func Convert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(in *HTTPConnectConfig, out *apiserver.HTTPConnectConfig, s conversion.Scope) error { + return autoConvert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(in, out, s) +} + +func autoConvert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(in *apiserver.HTTPConnectConfig, out *HTTPConnectConfig, s conversion.Scope) error { + out.URL = in.URL + out.CABundle = in.CABundle + out.ClientKey = in.ClientKey + out.ClientCert = in.ClientCert + return nil +} + +// Convert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig is an autogenerated conversion function. +func Convert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(in *apiserver.HTTPConnectConfig, out *HTTPConnectConfig, s conversion.Scope) error { + return autoConvert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(in, out, s) +} diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.deepcopy.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.deepcopy.go index 24151bbd22b..e8d6086798b 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.deepcopy.go @@ -76,3 +76,89 @@ func (in *AdmissionPluginConfiguration) DeepCopy() *AdmissionPluginConfiguration in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Connection) DeepCopyInto(out *Connection) { + *out = *in + if in.HTTPConnect != nil { + in, out := &in.HTTPConnect, &out.HTTPConnect + *out = new(HTTPConnectConfig) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Connection. +func (in *Connection) DeepCopy() *Connection { + if in == nil { + return nil + } + out := new(Connection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressSelection) DeepCopyInto(out *EgressSelection) { + *out = *in + in.Connection.DeepCopyInto(&out.Connection) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressSelection. +func (in *EgressSelection) DeepCopy() *EgressSelection { + if in == nil { + return nil + } + out := new(EgressSelection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressSelectorConfiguration) DeepCopyInto(out *EgressSelectorConfiguration) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.EgressSelections != nil { + in, out := &in.EgressSelections, &out.EgressSelections + *out = make([]EgressSelection, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressSelectorConfiguration. +func (in *EgressSelectorConfiguration) DeepCopy() *EgressSelectorConfiguration { + if in == nil { + return nil + } + out := new(EgressSelectorConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EgressSelectorConfiguration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HTTPConnectConfig) DeepCopyInto(out *HTTPConnectConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPConnectConfig. +func (in *HTTPConnectConfig) DeepCopy() *HTTPConnectConfig { + if in == nil { + return nil + } + out := new(HTTPConnectConfig) + in.DeepCopyInto(out) + return out +} diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/zz_generated.deepcopy.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/zz_generated.deepcopy.go index 542ef977b6b..3159f7c1a34 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/zz_generated.deepcopy.go @@ -76,3 +76,89 @@ func (in *AdmissionPluginConfiguration) DeepCopy() *AdmissionPluginConfiguration in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Connection) DeepCopyInto(out *Connection) { + *out = *in + if in.HTTPConnect != nil { + in, out := &in.HTTPConnect, &out.HTTPConnect + *out = new(HTTPConnectConfig) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Connection. +func (in *Connection) DeepCopy() *Connection { + if in == nil { + return nil + } + out := new(Connection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressSelection) DeepCopyInto(out *EgressSelection) { + *out = *in + in.Connection.DeepCopyInto(&out.Connection) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressSelection. +func (in *EgressSelection) DeepCopy() *EgressSelection { + if in == nil { + return nil + } + out := new(EgressSelection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressSelectorConfiguration) DeepCopyInto(out *EgressSelectorConfiguration) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.EgressSelections != nil { + in, out := &in.EgressSelections, &out.EgressSelections + *out = make([]EgressSelection, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressSelectorConfiguration. +func (in *EgressSelectorConfiguration) DeepCopy() *EgressSelectorConfiguration { + if in == nil { + return nil + } + out := new(EgressSelectorConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EgressSelectorConfiguration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HTTPConnectConfig) DeepCopyInto(out *HTTPConnectConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPConnectConfig. +func (in *HTTPConnectConfig) DeepCopy() *HTTPConnectConfig { + if in == nil { + return nil + } + out := new(HTTPConnectConfig) + in.DeepCopyInto(out) + return out +} diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/rest/streamer.go b/staging/src/k8s.io/apiserver/pkg/registry/generic/rest/streamer.go index 562288a3aee..be911fab2f1 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/rest/streamer.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/rest/streamer.go @@ -62,6 +62,7 @@ func (s *LocationStreamer) InputStream(ctx context.Context, apiVersion, acceptHe if transport == nil { transport = http.DefaultTransport } + client := &http.Client{ Transport: transport, CheckRedirect: s.RedirectChecker, diff --git a/staging/src/k8s.io/apiserver/pkg/server/BUILD b/staging/src/k8s.io/apiserver/pkg/server/BUILD index 43bf3731c4a..de85f2dcc42 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/BUILD @@ -11,6 +11,7 @@ go_test( srcs = [ "config_selfclient_test.go", "config_test.go", + "egress_selector_test.go", "genericapiserver_test.go", "healthz_test.go", ], @@ -21,9 +22,11 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/version:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/apiserver:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/example:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/example/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", @@ -49,6 +52,7 @@ go_library( "config_selfclient.go", "deprecated_insecure_serving.go", "doc.go", + "egress_selector.go", "genericapiserver.go", "handler.go", "healthz.go", @@ -70,6 +74,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library", @@ -79,6 +84,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/apiserver:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/apiserver/install:go_default_library", "//staging/src/k8s.io/apiserver/pkg/audit:go_default_library", "//staging/src/k8s.io/apiserver/pkg/audit/policy:go_default_library", @@ -133,6 +139,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//staging/src/k8s.io/apiserver/pkg/server/egressselector:all-srcs", "//staging/src/k8s.io/apiserver/pkg/server/filters:all-srcs", "//staging/src/k8s.io/apiserver/pkg/server/healthz:all-srcs", "//staging/src/k8s.io/apiserver/pkg/server/httplog:all-srcs", diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index d00637e5515..91fac23ddcd 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -29,7 +29,7 @@ import ( "sync/atomic" "time" - jsonpatch "github.com/evanphx/json-patch" + "github.com/evanphx/json-patch" "github.com/go-openapi/spec" "github.com/pborman/uuid" @@ -94,6 +94,11 @@ type Config struct { // This is required for proper functioning of the PostStartHooks on a GenericAPIServer // TODO: move into SecureServing(WithLoopback) as soon as insecure serving is gone LoopbackClientConfig *restclient.Config + + // EgressSelector provides a lookup mechanism for dialing outbound connections. + // It does so based on a EgressSelectorConfiguration which was read at startup. + EgressSelector *EgressSelector + // RuleResolver is required to get the list of rules that apply to a given user // in a given namespace RuleResolver authorizer.RuleResolver diff --git a/staging/src/k8s.io/apiserver/pkg/server/egress_selector.go b/staging/src/k8s.io/apiserver/pkg/server/egress_selector.go new file mode 100644 index 00000000000..58a9dd320ad --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/egress_selector.go @@ -0,0 +1,192 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "bufio" + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apiserver/pkg/apis/apiserver" + "k8s.io/klog" + "net" + "net/http" + "net/url" + "strings" +) + +var directDialer utilnet.DialFunc = http.DefaultTransport.(*http.Transport).DialContext + +type EgressSelector struct { + egressToDialer map[EgressType]utilnet.DialFunc +} + +// EgressType is an indicator of which egress selection should be used for sending traffic. +// See https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190226-network-proxy.md#network-context +type EgressType int + +const ( + // Master is the EgressType for traffic intended to go to the control plane. + Master EgressType = iota + // Etcd is the EgressType for traffic intended to go to Kubernetes persistence store. + Etcd + // Cluster is the EgressType for traffic intended to go to the system being managed by Kubernetes. + Cluster +) + +// NetworkContext is the struct used by Kubernetes API Server to indicate where it intends traffic to be sent. +type NetworkContext struct { + // EgressSelectionName is the unique name of the + // EgressSelectorConfiguration which determines + // the network we route the traffic to. + EgressSelectionName EgressType +} + +// EgressSelectorLookup is the interface to get the dialer function for the network context. +type EgressSelectorLookup func(networkContext NetworkContext) (utilnet.DialFunc, error) + +func (s EgressType) String() string { + switch s { + case Master: + return "master" + case Etcd: + return "etcd" + case Cluster: + return "cluster" + default: + return "invalid" + } +} + +func lookupServiceName(name string) (EgressType, error) { + switch strings.ToLower(name) { + case "master": + return Master, nil + case "etcd": + return Etcd, nil + case "cluster": + return Cluster, nil + } + return -1, fmt.Errorf("unrecognized service name %s", name) +} + +func createConnectDialer(connectConfig *apiserver.HTTPConnectConfig) (utilnet.DialFunc, error) { + clientCert := connectConfig.ClientCert + clientKey := connectConfig.ClientKey + caCert := connectConfig.CABundle + proxyURL, err := url.Parse(connectConfig.URL) + if err != nil { + return nil, fmt.Errorf("invalid proxy server url %q: %v", connectConfig.URL, err) + } + proxyAddress := proxyURL.Host + + clientCerts, err := tls.LoadX509KeyPair(clientCert, clientKey) + if err != nil { + return nil, fmt.Errorf("failed to read key pair %s & %s, got %v", clientCert, clientKey, err) + } + certPool := x509.NewCertPool() + certBytes, err := ioutil.ReadFile(caCert) + if err != nil { + return nil, fmt.Errorf("failed to read cert file %s, got %v", caCert, err) + } + ok := certPool.AppendCertsFromPEM(certBytes) + if !ok { + return nil, fmt.Errorf("failed to append CA cert to the cert pool") + } + contextDialer := func(ctx context.Context, network, addr string) (net.Conn, error) { + klog.V(4).Infof("Sending request to %q.", addr) + proxyConn, err := tls.Dial("tcp", proxyAddress, + &tls.Config{ + Certificates: []tls.Certificate{clientCerts}, + RootCAs: certPool, + }, + ) + if err != nil { + return nil, fmt.Errorf("dialing proxy %q failed: %v", proxyAddress, err) + } + fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", addr, "127.0.0.1") + br := bufio.NewReader(proxyConn) + res, err := http.ReadResponse(br, nil) + if err != nil { + proxyConn.Close() + return nil, fmt.Errorf("reading HTTP response from CONNECT to %s via proxy %s failed: %v", + addr, proxyAddress, err) + } + if res.StatusCode != 200 { + proxyConn.Close() + return nil, fmt.Errorf("proxy error from %s while dialing %s, code %d: %v", + proxyAddress, addr, res.StatusCode, res.Status) + } + + // It's safe to discard the bufio.Reader here and return the + // original TCP conn directly because we only use this for + // TLS, and in TLS the client speaks first, so we know there's + // no unbuffered data. But we can double-check. + if br.Buffered() > 0 { + proxyConn.Close() + return nil, fmt.Errorf("unexpected %d bytes of buffered data from CONNECT proxy %q", + br.Buffered(), proxyAddress) + } + klog.V(4).Infof("About to proxy request to %s over %s.", addr, proxyAddress) + return proxyConn, nil + } + return contextDialer, nil +} + +// NewEgressSelector configures lookup mechanism for Lookup. +// It does so based on a EgressSelectorConfiguration which was read at startup. +func NewEgressSelector(config *apiserver.EgressSelectorConfiguration) (*EgressSelector, error) { + if config == nil || config.EgressSelections == nil { + // No Connection Services configured, leaving the serviceMap empty, will return default dialer. + return nil, nil + } + cs := &EgressSelector{ + egressToDialer: make(map[EgressType]utilnet.DialFunc), + } + for _, service := range config.EgressSelections { + name, err := lookupServiceName(service.Name) + if err != nil { + return nil, err + } + switch service.Connection.Type { + case "http-connect": + contextDialer, err := createConnectDialer(service.Connection.HTTPConnect) + if err != nil { + return nil, fmt.Errorf("failed to create http-connect dialer: %v", err) + } + cs.egressToDialer[name] = contextDialer + case "direct": + cs.egressToDialer[name] = directDialer + default: + return nil, fmt.Errorf("unrecognized service connection type %q", service.Connection.Type) + } + } + return cs, nil +} + +// Lookup gets the dialer function for the network context. +// This is configured for the Kubernetes API Server at startup. +func (cs *EgressSelector) Lookup(networkContext NetworkContext) (utilnet.DialFunc, error) { + if cs.egressToDialer == nil { + // The round trip wrapper will over-ride the dialContext method appropriately + return nil, nil + } + return cs.egressToDialer[networkContext.EgressSelectionName], nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/egress_selector_test.go b/staging/src/k8s.io/apiserver/pkg/server/egress_selector_test.go new file mode 100644 index 00000000000..825e53415d8 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/egress_selector_test.go @@ -0,0 +1,183 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "context" + "net" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apiserver/pkg/apis/apiserver" +) + +type fakeEgressSelection struct { + directDialerCalled bool +} + +func TestEgressSelector(t *testing.T) { + testcases := []struct { + name string + input *apiserver.EgressSelectorConfiguration + services []struct { + egressType EgressType + validateDialer func(dialer utilnet.DialFunc, s *fakeEgressSelection) (bool, error) + lookupError *string + dialerError *string + } + expectedError *string + }{ + { + name: "direct", + input: &apiserver.EgressSelectorConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "", + APIVersion: "", + }, + EgressSelections: []apiserver.EgressSelection{ + { + Name: "cluster", + Connection: apiserver.Connection{ + Type: "direct", + HTTPConnect: &apiserver.HTTPConnectConfig{ + URL: "", + CABundle: "", + ClientKey: "", + ClientCert: "", + }, + }, + }, + { + Name: "master", + Connection: apiserver.Connection{ + Type: "direct", + HTTPConnect: &apiserver.HTTPConnectConfig{ + URL: "", + CABundle: "", + ClientKey: "", + ClientCert: "", + }, + }, + }, + { + Name: "etcd", + Connection: apiserver.Connection{ + Type: "direct", + HTTPConnect: &apiserver.HTTPConnectConfig{ + URL: "", + CABundle: "", + ClientKey: "", + ClientCert: "", + }, + }, + }, + }, + }, + services: []struct { + egressType EgressType + validateDialer func(dialer utilnet.DialFunc, s *fakeEgressSelection) (bool, error) + lookupError *string + dialerError *string + }{ + { + Cluster, + validateDirectDialer, + nil, + nil, + }, + { + Master, + validateDirectDialer, + nil, + nil, + }, + { + Etcd, + validateDirectDialer, + nil, + nil, + }, + }, + expectedError: nil, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + // Setup the various pieces such as the fake dialer prior to initializing the egress selector. + // Go doesn't allow function pointer comparison, nor does its reflect package + // So overriding the default dialer to detect if it is returned. + fake := &fakeEgressSelection{} + directDialer = fake.fakeDirectDialer + cs, err := NewEgressSelector(tc.input) + if err == nil && tc.expectedError != nil { + t.Errorf("calling NewEgressSelector expected error: %s, did not get it", *tc.expectedError) + } + if err != nil && tc.expectedError == nil { + t.Errorf("unexpected error calling NewEgressSelector got: %#v", err) + } + if err != nil && tc.expectedError != nil && err.Error() != *tc.expectedError { + t.Errorf("calling NewEgressSelector expected error: %s, got %#v", *tc.expectedError, err) + } + + for _, service := range tc.services { + networkContext := NetworkContext{EgressSelectionName: service.egressType} + dialer, lookupErr := cs.Lookup(networkContext) + if lookupErr == nil && service.lookupError != nil { + t.Errorf("calling Lookup expected error: %s, did not get it", *service.lookupError) + } + if lookupErr != nil && service.lookupError == nil { + t.Errorf("unexpected error calling Lookup got: %#v", lookupErr) + } + if lookupErr != nil && service.lookupError != nil && lookupErr.Error() != *service.lookupError { + t.Errorf("calling Lookup expected error: %s, got %#v", *service.lookupError, lookupErr) + } + fake.directDialerCalled = false + ok, dialerErr := service.validateDialer(dialer, fake) + if dialerErr == nil && service.dialerError != nil { + t.Errorf("calling Lookup expected error: %s, did not get it", *service.dialerError) + } + if dialerErr != nil && service.dialerError == nil { + t.Errorf("unexpected error calling Lookup got: %#v", dialerErr) + } + if dialerErr != nil && service.dialerError != nil && dialerErr.Error() != *service.dialerError { + t.Errorf("calling Lookup expected error: %s, got %#v", *service.dialerError, dialerErr) + } + if !ok { + t.Errorf("Could not validate dialer for service %q", service.egressType) + } + } + }) + } +} + +func (s *fakeEgressSelection) fakeDirectDialer(ctx context.Context, network, address string) (net.Conn, error) { + s.directDialerCalled = true + return nil, nil +} + +func validateDirectDialer(dialer utilnet.DialFunc, s *fakeEgressSelection) (bool, error) { + conn, err := dialer(context.Background(), "tcp", "127.0.0.1:8080") + if err != nil { + return false, err + } + if conn != nil { + return false, nil + } + return s.directDialerCalled, nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/egressselector/BUILD b/staging/src/k8s.io/apiserver/pkg/server/egressselector/BUILD new file mode 100644 index 00000000000..8df8af8ac63 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/egressselector/BUILD @@ -0,0 +1,42 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["config.go"], + importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/egressselector", + importpath = "k8s.io/apiserver/pkg/server/egressselector", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/apiserver:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/apiserver/install:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1:go_default_library", + "//vendor/k8s.io/utils/path:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = ["config_test.go"], + embed = [":go_default_library"], + deps = [ + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/apiserver:go_default_library", + ], +) diff --git a/staging/src/k8s.io/apiserver/pkg/server/egressselector/config.go b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config.go new file mode 100644 index 00000000000..a14714edaf1 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config.go @@ -0,0 +1,179 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package egressselector + +import ( + "fmt" + "io/ioutil" + "strings" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apiserver/pkg/apis/apiserver" + "k8s.io/apiserver/pkg/apis/apiserver/install" + "k8s.io/apiserver/pkg/apis/apiserver/v1alpha1" + "k8s.io/utils/path" + "sigs.k8s.io/yaml" +) + +var cfgScheme = runtime.NewScheme() + +func init() { + install.Install(cfgScheme) +} + +// ReadEgressSelectorConfiguration reads the egress selector configuration at the specified path. +// It returns the loaded egress selector configuration if the input file aligns with the required syntax. +// If it does not align with the provided syntax, it returns a default configuration which should function as a no-op. +// It does this by returning a nil configuration, which preserves backward compatibility. +// This works because prior to this there was no egress selector configuration. +// It returns an error if the file did not exist. +func ReadEgressSelectorConfiguration(configFilePath string) (*apiserver.EgressSelectorConfiguration, error) { + if configFilePath == "" { + return nil, nil + } + // a file was provided, so we just read it. + data, err := ioutil.ReadFile(configFilePath) + if err != nil { + return nil, fmt.Errorf("unable to read egress selector configuration from %q [%v]", configFilePath, err) + } + var decodedConfig v1alpha1.EgressSelectorConfiguration + err = yaml.Unmarshal(data, &decodedConfig) + if err != nil { + // we got an error where the decode wasn't related to a missing type + return nil, err + } + if decodedConfig.Kind != "EgressSelectorConfiguration" { + return nil, fmt.Errorf("invalid service configuration object %q", decodedConfig.Kind) + } + config, err := cfgScheme.ConvertToVersion(&decodedConfig, apiserver.SchemeGroupVersion) + if err != nil { + // we got an error where the decode wasn't related to a missing type + return nil, err + } + if internalConfig, ok := config.(*apiserver.EgressSelectorConfiguration); ok { + return internalConfig, nil + } + return nil, fmt.Errorf("unable to convert %T to *apiserver.EgressSelectorConfiguration", config) +} + +// ValidateEgressSelectorConfiguration checks the apiserver.EgressSelectorConfiguration for +// common configuration errors. It will return error for problems such as configuring mtls/cert +// settings for protocol which do not support security. It will also try to catch errors such as +// incorrect file paths. It will return nil if it does not find anything wrong. +func ValidateEgressSelectorConfiguration(config *apiserver.EgressSelectorConfiguration) field.ErrorList { + allErrs := field.ErrorList{} + if config == nil { + return allErrs // Treating a nil configuration as valid + } + for _, service := range config.EgressSelections { + base := field.NewPath("service", "connection") + switch service.Connection.Type { + case "direct": + allErrs = append(allErrs, validateDirectConnection(service.Connection, base)...) + case "http-connect": + allErrs = append(allErrs, validateHTTPConnection(service.Connection, base)...) + default: + allErrs = append(allErrs, field.NotSupported( + base.Child("type"), + service.Connection.Type, + []string{"direct", "http-connect"})) + } + } + + return allErrs +} + +func validateDirectConnection(connection apiserver.Connection, fldPath *field.Path) field.ErrorList { + if connection.HTTPConnect != nil { + return field.ErrorList{field.Invalid( + fldPath.Child("httpConnect"), + "direct", + "httpConnect config should be absent for direct connect"), + } + } + return nil +} + +func validateHTTPConnection(connection apiserver.Connection, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if connection.HTTPConnect == nil { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect"), + "nil", + "httpConnect config should be present for http-connect")) + } else if strings.HasPrefix(connection.HTTPConnect.URL, "https://") { + if connection.HTTPConnect.CABundle == "" { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "caBundle"), + "nil", + "http-connect via https requires caBundle")) + } else if exists, err := path.Exists(path.CheckFollowSymlink, connection.HTTPConnect.CABundle); exists == false || err != nil { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "caBundle"), + connection.HTTPConnect.CABundle, + "http-connect ca bundle does not exist")) + } + if connection.HTTPConnect.ClientCert == "" { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "clientCert"), + "nil", + "http-connect via https requires clientCert")) + } else if exists, err := path.Exists(path.CheckFollowSymlink, connection.HTTPConnect.ClientCert); exists == false || err != nil { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "clientCert"), + connection.HTTPConnect.ClientCert, + "http-connect client cert does not exist")) + } + if connection.HTTPConnect.ClientKey == "" { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "clientKey"), + "nil", + "http-connect via https requires clientKey")) + } else if exists, err := path.Exists(path.CheckFollowSymlink, connection.HTTPConnect.ClientKey); exists == false || err != nil { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "clientKey"), + connection.HTTPConnect.ClientKey, + "http-connect client key does not exist")) + } + } else if strings.HasPrefix(connection.HTTPConnect.URL, "http://") { + if connection.HTTPConnect.CABundle != "" { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "caBundle"), + connection.HTTPConnect.CABundle, + "http-connect via http does not support caBundle")) + } + if connection.HTTPConnect.ClientCert != "" { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "clientCert"), + connection.HTTPConnect.ClientCert, + "http-connect via http does not support clientCert")) + } + if connection.HTTPConnect.ClientKey != "" { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "clientKey"), + connection.HTTPConnect.ClientKey, + "http-connect via http does not support clientKey")) + } + } else { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "url"), + connection.HTTPConnect.URL, + "supported connection protocols are http:// and https://")) + } + return allErrs +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go new file mode 100644 index 00000000000..feefe5fe11e --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go @@ -0,0 +1,214 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package egressselector + +import ( + "fmt" + "io/ioutil" + "os" + "reflect" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/apis/apiserver" +) + +func strptr(s string) *string { + return &s +} + +func TestReadEgressSelectorConfiguration(t *testing.T) { + testcases := []struct { + name string + contents string + createFile bool + expectedResult *apiserver.EgressSelectorConfiguration + expectedError *string + }{ + { + name: "empty", + createFile: true, + contents: ``, + expectedResult: nil, + expectedError: strptr("invalid service configuration object \"\""), + }, + { + name: "absent", + createFile: false, + contents: ``, + expectedResult: nil, + expectedError: strptr("unable to read egress selector configuration from \"test-egress-selector-config-absent\" [open test-egress-selector-config-absent: no such file or directory]"), + }, + { + name: "v1alpha1", + createFile: true, + contents: ` +apiVersion: apiserver.k8s.io/v1alpha1 +kind: EgressSelectorConfiguration +egressSelections: +- name: "cluster" + connection: + type: "http-connect" + httpConnect: + url: "https://127.0.0.1:8131" + caBundle: "/etc/srv/kubernetes/pki/konnectivity-server/ca.crt" + clientKey: "/etc/srv/kubernetes/pki/konnectivity-server/client.key" + clientCert: "/etc/srv/kubernetes/pki/konnectivity-server/client.crt" +- name: "master" + connection: + type: "http-connect" + httpConnect: + url: "https://127.0.0.1:8132" + caBundle: "/etc/srv/kubernetes/pki/konnectivity-server-master/ca.crt" + clientKey: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.key" + clientCert: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.crt" +- name: "etcd" + connection: + type: "direct" +`, + expectedResult: &apiserver.EgressSelectorConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "", + APIVersion: "", + }, + EgressSelections: []apiserver.EgressSelection{ + { + Name: "cluster", + Connection: apiserver.Connection{ + Type: "http-connect", + HTTPConnect: &apiserver.HTTPConnectConfig{ + URL: "https://127.0.0.1:8131", + CABundle: "/etc/srv/kubernetes/pki/konnectivity-server/ca.crt", + ClientKey: "/etc/srv/kubernetes/pki/konnectivity-server/client.key", + ClientCert: "/etc/srv/kubernetes/pki/konnectivity-server/client.crt", + }, + }, + }, + { + Name: "master", + Connection: apiserver.Connection{ + Type: "http-connect", + HTTPConnect: &apiserver.HTTPConnectConfig{ + URL: "https://127.0.0.1:8132", + CABundle: "/etc/srv/kubernetes/pki/konnectivity-server-master/ca.crt", + ClientKey: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.key", + ClientCert: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.crt", + }, + }, + }, + { + Name: "etcd", + Connection: apiserver.Connection{ + Type: "direct", + }, + }, + }, + }, + expectedError: nil, + }, + { + name: "wrong_type", + createFile: true, + contents: ` +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + addonmanager.kubernetes.io/mode: Reconcile + k8s-app: konnectivity-agent + namespace: kube-system + name: proxy-agent +spec: + selector: + matchLabels: + k8s-app: konnectivity-agent + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: proxy-agent + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + priorityClassName: system-cluster-critical + # Necessary to reboot node + hostPID: true + volumes: + - name: pki + hostPath: + path: /etc/srv/kubernetes/pki/konnectivity-agent + containers: + - image: gcr.io/google-containers/proxy-agent:v0.0.3 + name: proxy-agent + command: ["/proxy-agent"] + args: ["--caCert=/etc/srv/kubernetes/pki/proxy-agent/ca.crt", "--agentCert=/etc/srv/kubernetes/pki/proxy-agent/client.crt", "--agentKey=/etc/srv/kubernetes/pki/proxy-agent/client.key", "--proxyServerHost=127.0.0.1", "--proxyServerPort=8132"] + securityContext: + capabilities: + add: ["SYS_BOOT"] + env: + - name: wrong-type + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: kube-system + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + limits: + cpu: 50m + memory: 30Mi + volumeMounts: + - name: pki + mountPath: /etc/srv/kubernetes/pki/konnectivity-agent +`, + expectedResult: nil, + expectedError: strptr("invalid service configuration object \"DaemonSet\""), + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + proxyConfig := fmt.Sprintf("test-egress-selector-config-%s", tc.name) + if tc.createFile { + f, err := ioutil.TempFile("", proxyConfig) + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + if err := ioutil.WriteFile(f.Name(), []byte(tc.contents), os.FileMode(0755)); err != nil { + t.Fatal(err) + } + proxyConfig = f.Name() + } + config, err := ReadEgressSelectorConfiguration(proxyConfig) + if err == nil && tc.expectedError != nil { + t.Errorf("calling ReadEgressSelectorConfiguration expected error: %s, did not get it", *tc.expectedError) + } + if err != nil && tc.expectedError == nil { + t.Errorf("unexpected error calling ReadEgressSelectorConfiguration got: %#v", err) + } + if err != nil && tc.expectedError != nil && err.Error() != *tc.expectedError { + t.Errorf("calling ReadEgressSelectorConfiguration expected error: %s, got %#v", *tc.expectedError, err) + } + if !reflect.DeepEqual(config, tc.expectedResult) { + t.Errorf("problem with configuration returned from ReadEgressSelectorConfiguration expected: %#v, got: %#v", tc.expectedResult, config) + } + }) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/BUILD b/staging/src/k8s.io/apiserver/pkg/server/options/BUILD index ec477368a61..a1ae49080f9 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/options/BUILD @@ -11,6 +11,7 @@ go_library( "coreapi.go", "deprecated_insecure_serving.go", "doc.go", + "egress_selector.go", "etcd.go", "events.go", "feature.go", @@ -56,6 +57,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server/egressselector:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/resourceconfig:go_default_library", @@ -83,6 +85,7 @@ go_library( "//vendor/gopkg.in/natefinch/lumberjack.v2:go_default_library", "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library", + "//vendor/k8s.io/utils/path:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/egress_selector.go b/staging/src/k8s.io/apiserver/pkg/server/options/egress_selector.go new file mode 100644 index 00000000000..837e32fcfaf --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/options/egress_selector.go @@ -0,0 +1,92 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "fmt" + "github.com/spf13/pflag" + "k8s.io/utils/path" + + "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/egressselector" +) + +// EgressSelectorOptions holds the api server egress selector options. +// See https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190226-network-proxy.md +type EgressSelectorOptions struct { + // ConfigFile is the file path with api-server egress selector configuration. + ConfigFile string +} + +// NewEgressSelectorOptions creates a new instance of EgressSelectorOptions +// +// The option is to point to a configuration file for egress/konnectivity. +// This determines which types of requests use egress/konnectivity and how they use it. +// If empty the API Server will attempt to connect directly using the network. +func NewEgressSelectorOptions() *EgressSelectorOptions { + return &EgressSelectorOptions{} +} + +// AddFlags adds flags related to admission for a specific APIServer to the specified FlagSet +func (o *EgressSelectorOptions) AddFlags(fs *pflag.FlagSet) { + if o == nil { + return + } + + fs.StringVar(&o.ConfigFile, "egress-selector-config-file", o.ConfigFile, + "File with apiserver egress selector configuration.") +} + +// ApplyTo adds the egress selector settings to the server configuration. +// In case egress selector settings were not provided by a cluster-admin +// they will be prepared from the recommended/default/no-op values. +func (o *EgressSelectorOptions) ApplyTo(c *server.Config) error { + if o == nil { + return nil + } + + npConfig, err := egressselector.ReadEgressSelectorConfiguration(o.ConfigFile) + if err != nil { + return fmt.Errorf("failed to read egress selector config: %v", err) + } + errs := egressselector.ValidateEgressSelectorConfiguration(npConfig) + if len(errs) > 0 { + return fmt.Errorf("failed to validate egress selector configuration: %v", errs.ToAggregate()) + } + + cs, err := server.NewEgressSelector(npConfig) + if err != nil { + return fmt.Errorf("failed to setup egress selector with config %#v: %v", npConfig, err) + } + c.EgressSelector = cs + return nil +} + +// Validate verifies flags passed to EgressSelectorOptions. +func (o *EgressSelectorOptions) Validate() []error { + if o == nil || o.ConfigFile == "" { + return nil + } + + errs := []error{} + + if exists, err := path.Exists(path.CheckFollowSymlink, o.ConfigFile); exists == false || err != nil { + errs = append(errs, fmt.Errorf("egress-selector-config-file %s does not exist", o.ConfigFile)) + } + + return errs +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go b/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go index 631b4f43e39..2ddb901a922 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go @@ -44,6 +44,8 @@ type RecommendedOptions struct { // ProcessInfo is used to identify events created by the server. ProcessInfo *ProcessInfo Webhook *WebhookOptions + // API Server Egress Selector is used to control outbound traffic from the API Server + EgressSelector *EgressSelectorOptions } func NewRecommendedOptions(prefix string, codec runtime.Codec, processInfo *ProcessInfo) *RecommendedOptions { @@ -67,6 +69,7 @@ func NewRecommendedOptions(prefix string, codec runtime.Codec, processInfo *Proc Admission: NewAdmissionOptions(), ProcessInfo: processInfo, Webhook: NewWebhookOptions(), + EgressSelector: NewEgressSelectorOptions(), } } @@ -79,6 +82,7 @@ func (o *RecommendedOptions) AddFlags(fs *pflag.FlagSet) { o.Features.AddFlags(fs) o.CoreAPI.AddFlags(fs) o.Admission.AddFlags(fs) + o.EgressSelector.AddFlags(fs) } // ApplyTo adds RecommendedOptions to the server configuration. @@ -110,6 +114,9 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error { } else if err := o.Admission.ApplyTo(&config.Config, config.SharedInformerFactory, config.ClientConfig, initializers...); err != nil { return err } + if err := o.EgressSelector.ApplyTo(&config.Config); err != nil { + return err + } return nil } @@ -124,6 +131,7 @@ func (o *RecommendedOptions) Validate() []error { errors = append(errors, o.Features.Validate()...) errors = append(errors, o.CoreAPI.Validate()...) errors = append(errors, o.Admission.Validate()...) + errors = append(errors, o.EgressSelector.Validate()...) return errors } diff --git a/staging/src/k8s.io/code-generator/_examples/HyphenGroup/apis/example/v1/zz_generated.defaults.go b/staging/src/k8s.io/code-generator/_examples/HyphenGroup/apis/example/v1/zz_generated.defaults.go new file mode 100644 index 00000000000..cce2e603a69 --- /dev/null +++ b/staging/src/k8s.io/code-generator/_examples/HyphenGroup/apis/example/v1/zz_generated.defaults.go @@ -0,0 +1,32 @@ +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by defaulter-gen. DO NOT EDIT. + +package v1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// RegisterDefaults adds defaulters functions to the given scheme. +// Public to allow building arbitrary schemes. +// All generated defaulters are covering - they call all nested defaulters. +func RegisterDefaults(scheme *runtime.Scheme) error { + return nil +} diff --git a/staging/src/k8s.io/kube-aggregator/go.sum b/staging/src/k8s.io/kube-aggregator/go.sum index eb46d25ae64..0bd22d1e01a 100644 --- a/staging/src/k8s.io/kube-aggregator/go.sum +++ b/staging/src/k8s.io/kube-aggregator/go.sum @@ -171,6 +171,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/soheilhy/cmux v0.1.3 h1:09wy7WZk4AqO03yH85Ex1X+Uo3vDsil3Fa9AgF8Emss= github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.4 h1:S0tLZ3VOKl2Te0hpq8+ke0eSJPfCnNTPiDlsfwi1/NE= diff --git a/staging/src/k8s.io/sample-apiserver/go.sum b/staging/src/k8s.io/sample-apiserver/go.sum index 79a4b96887f..92956bad004 100644 --- a/staging/src/k8s.io/sample-apiserver/go.sum +++ b/staging/src/k8s.io/sample-apiserver/go.sum @@ -168,6 +168,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/soheilhy/cmux v0.1.3 h1:09wy7WZk4AqO03yH85Ex1X+Uo3vDsil3Fa9AgF8Emss= github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.4 h1:S0tLZ3VOKl2Te0hpq8+ke0eSJPfCnNTPiDlsfwi1/NE= diff --git a/vendor/modules.txt b/vendor/modules.txt index 186bae47edb..c2412826ba8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1257,6 +1257,7 @@ k8s.io/apiserver/pkg/registry/generic/testing k8s.io/apiserver/pkg/registry/rest k8s.io/apiserver/pkg/registry/rest/resttest k8s.io/apiserver/pkg/server +k8s.io/apiserver/pkg/server/egressselector k8s.io/apiserver/pkg/server/filters k8s.io/apiserver/pkg/server/healthz k8s.io/apiserver/pkg/server/httplog