diff --git a/build/lib/release.sh b/build/lib/release.sh index 4c5e0962be3..73c0bcca62d 100644 --- a/build/lib/release.sh +++ b/build/lib/release.sh @@ -432,6 +432,7 @@ function kube::release::package_kube_manifests_tarball() { cp "${src_dir}/etcd.manifest" "${dst_dir}" cp "${src_dir}/kube-scheduler.manifest" "${dst_dir}" cp "${src_dir}/kube-apiserver.manifest" "${dst_dir}" + cp "${src_dir}/konnectivity-server.yaml" "${dst_dir}" cp "${src_dir}/abac-authz-policy.jsonl" "${dst_dir}" cp "${src_dir}/kube-controller-manager.manifest" "${dst_dir}" cp "${src_dir}/kube-addon-manager.yaml" "${dst_dir}" diff --git a/cluster/gce/addons/konnectivity-agent/daemonset.yaml b/cluster/gce/addons/konnectivity-agent/daemonset.yaml new file mode 100644 index 00000000000..1edd52c511a --- /dev/null +++ b/cluster/gce/addons/konnectivity-agent/daemonset.yaml @@ -0,0 +1,66 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + addonmanager.kubernetes.io/mode: Reconcile + k8s-app: konnectivity-agent + namespace: kube-system + name: konnectivity-agent +spec: + selector: + matchLabels: + k8s-app: konnectivity-agent + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: konnectivity-agent + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + priorityClassName: system-cluster-critical + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + hostNetwork: true + volumes: + - name: pki + hostPath: + path: /etc/srv/kubernetes/pki/konnectivity-agent + containers: + - image: gcr.io/google-containers/proxy-agent:v0.0.3 + name: konnectivity-agent + command: ["/proxy-agent"] + args: [ + "--logtostderr=true", + "--ca-cert=/etc/srv/kubernetes/pki/konnectivity-agent/ca.crt", + "--agent-cert=/etc/srv/kubernetes/pki/konnectivity-agent/client.crt", + "--agent-key=/etc/srv/kubernetes/pki/konnectivity-agent/client.key", + "--proxy-server-host=__APISERVER_IP__", + "--proxy-server-port=8132" + ] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + limits: + cpu: 50m + memory: 30Mi + livenessProbe: + httpGet: + host: 127.0.0.1 + port: 8093 + path: /healthz + initialDelaySeconds: 15 + timeoutSeconds: 15 + volumeMounts: + - name: pki + mountPath: /etc/srv/kubernetes/pki/konnectivity-agent diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index fe68c3816ab..fbea33f15ed 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -480,3 +480,6 @@ WINDOWS_NODE_TAINTS="${WINDOWS_NODE_TAINTS:-node.kubernetes.io/os=win1809:NoSche # Whether to set up a private GCE cluster, i.e. a cluster where nodes have only private IPs. GCE_PRIVATE_CLUSTER="${KUBE_GCE_PRIVATE_CLUSTER:-false}" + +# Optional: Create apiserver konnectivity server and agent. +ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE="${KUBE_ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index c9ebe210c70..53dc09f076d 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -495,6 +495,16 @@ function create-node-pki { KUBELET_KEY_PATH="${pki_dir}/kubelet.key" write-pki-data "${KUBELET_KEY}" "${KUBELET_KEY_PATH}" fi + + if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + mkdir -p "${pki_dir}/konnectivity-agent" + KONNECTIVITY_AGENT_CA_CERT_PATH="${pki_dir}/konnectivity-agent/ca.crt" + KONNECTIVITY_AGENT_CLIENT_KEY_PATH="${pki_dir}/konnectivity-agent/client.key" + KONNECTIVITY_AGENT_CLIENT_CERT_PATH="${pki_dir}/konnectivity-agent/client.crt" + write-pki-data "${KONNECTIVITY_AGENT_CA_CERT}" "${KONNECTIVITY_AGENT_CA_CERT_PATH}" + write-pki-data "${KONNECTIVITY_AGENT_CLIENT_KEY}" "${KONNECTIVITY_AGENT_CLIENT_KEY_PATH}" + write-pki-data "${KONNECTIVITY_AGENT_CLIENT_CERT}" "${KONNECTIVITY_AGENT_CLIENT_CERT_PATH}" + fi } function create-master-pki { @@ -558,6 +568,42 @@ function create-master-pki { PROXY_CLIENT_CERT_PATH="${pki_dir}/proxy_client.crt" write-pki-data "${PROXY_CLIENT_CERT}" "${PROXY_CLIENT_CERT_PATH}" fi + + if [[ ! -z "${KONNECTIVITY_SERVER_CA_CERT:-}" ]]; then + mkdir -p "${pki_dir}"/konnectivity-server + #KONNECTIVITY_SERVER_CA_KEY_PATH="${pki_dir}/konnectivity-server/ca.key" + #write-pki-data "${KONNECTIVITY_SERVER_CA_KEY}" "${KONNECTIVITY_SERVER_CA_KEY_PATH}" + + KONNECTIVITY_SERVER_CA_CERT_PATH="${pki_dir}/konnectivity-server/ca.crt" + write-pki-data "${KONNECTIVITY_SERVER_CA_CERT}" "${KONNECTIVITY_SERVER_CA_CERT_PATH}" + + KONNECTIVITY_SERVER_KEY_PATH="${pki_dir}/konnectivity-server/server.key" + write-pki-data "${KONNECTIVITY_SERVER_KEY}" "${KONNECTIVITY_SERVER_KEY_PATH}" + + KONNECTIVITY_SERVER_CERT_PATH="${pki_dir}/konnectivity-server/server.crt" + write-pki-data "${KONNECTIVITY_SERVER_CERT}" "${KONNECTIVITY_SERVER_CERT_PATH}" + + KONNECTIVITY_SERVER_CLIENT_KEY_PATH="${pki_dir}/konnectivity-server/client.key" + write-pki-data "${KONNECTIVITY_SERVER_CLIENT_KEY}" "${KONNECTIVITY_SERVER_CLIENT_KEY_PATH}" + + KONNECTIVITY_SERVER_CLIENT_CERT_PATH="${pki_dir}/konnectivity-server/client.crt" + write-pki-data "${KONNECTIVITY_SERVER_CLIENT_CERT}" "${KONNECTIVITY_SERVER_CLIENT_CERT_PATH}" + fi + + if [[ ! -z "${KONNECTIVITY_AGENT_CA_CERT:-}" ]]; then + mkdir -p "${pki_dir}"/konnectivity-agent + KONNECTIVITY_AGENT_CA_KEY_PATH="${pki_dir}/konnectivity-agent/ca.key" + write-pki-data "${KONNECTIVITY_AGENT_CA_KEY}" "${KONNECTIVITY_AGENT_CA_KEY_PATH}" + + KONNECTIVITY_AGENT_CA_CERT_PATH="${pki_dir}/konnectivity-agent/ca.crt" + write-pki-data "${KONNECTIVITY_AGENT_CA_CERT}" "${KONNECTIVITY_AGENT_CA_CERT_PATH}" + + KONNECTIVITY_AGENT_KEY_PATH="${pki_dir}/konnectivity-agent/server.key" + write-pki-data "${KONNECTIVITY_AGENT_KEY}" "${KONNECTIVITY_AGENT_KEY_PATH}" + + KONNECTIVITY_AGENT_CERT_PATH="${pki_dir}/konnectivity-agent/server.crt" + write-pki-data "${KONNECTIVITY_AGENT_CERT}" "${KONNECTIVITY_AGENT_CERT_PATH}" + fi } # After the first boot and on upgrade, these files exist on the master-pd @@ -757,6 +803,27 @@ contexts: name: webhook EOF fi + if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + cat </etc/srv/kubernetes/egress_selector_configuration.yaml +apiVersion: apiserver.k8s.io/v1alpha1 +kind: EgressSelectorConfiguration +egressSelections: +- name: cluster + connection: + type: http-connect + httpConnect: + url: https://127.0.0.1:8131 + caBundle: /etc/srv/kubernetes/pki/konnectivity-server/ca.crt + clientKey: /etc/srv/kubernetes/pki/konnectivity-server/client.key + clientCert: /etc/srv/kubernetes/pki/konnectivity-server/client.crt +- name: master + connection: + type: direct +- name: etcd + connection: + type: direct +EOF + fi if [[ -n "${WEBHOOK_GKE_EXEC_AUTH:-}" ]]; then if [[ -z "${EXEC_AUTH_PLUGIN_URL:-}" ]]; then @@ -1214,6 +1281,10 @@ function create-master-etcd-apiserver-auth { fi } +function create-master-konnectivity-server-apiserver-auth { + echo TODO: implement create-master-konnectivity-server-apiserver-auth +} + function assemble-docker-flags { echo "Assemble docker command line flags" local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false" @@ -1562,6 +1633,49 @@ function start-etcd-servers { prepare-etcd-manifest "-events" "4002" "2381" "100m" "etcd-events.manifest" } +# Replaces the variables in the konnectivity-server manifest file with the real values, and then +# copy the file to the manifest dir +# $1: value for variable "server_port" +# $2: value for variable "agent_port" +# $3: value for bariable "admin_port" +function prepare-konnectivity-server-manifest { + local -r temp_file="/tmp/konnectivity-server.yaml" + params=() + cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/konnectivity-server.yaml" "${temp_file}" + params+=("--log-file=/var/log/konnectivity-server.log") + params+=("--logtostderr=false") + params+=("--log-file-max-size=0") + params+=("--server-ca-cert=${KONNECTIVITY_SERVER_CA_CERT_PATH}") + params+=("--server-cert=${KONNECTIVITY_SERVER_CERT_PATH}") + params+=("--server-key=${KONNECTIVITY_SERVER_KEY_PATH}") + params+=("--cluster-ca-cert=${KONNECTIVITY_AGENT_CA_CERT_PATH}") + params+=("--cluster-cert=${KONNECTIVITY_AGENT_CERT_PATH}") + params+=("--cluster-key=${KONNECTIVITY_AGENT_KEY_PATH}") + params+=("--mode=http-connect") + params+=("--server-port=$1") + params+=("--agent-port=$2") + params+=("--admin-port=$3") + konnectivity_args="" + for param in "${params[@]}"; do + konnectivity_args+=", \"${param}\"" + done + sed -i -e "s@{{ *konnectivity_args *}}@${konnectivity_args}@g" "${temp_file}" + sed -i -e "s@{{ *server_port *}}@$1@g" "${temp_file}" + sed -i -e "s@{{ *agent_port *}}@$2@g" "${temp_file}" + sed -i -e "s@{{ *admin_port *}}@$3@g" "${temp_file}" + sed -i -e "s@{{ *liveness_probe_initial_delay *}}@30@g" "${temp_file}" + mv "${temp_file}" /etc/kubernetes/manifests +} + +# Starts konnectivity server pod. +# More specifically, it prepares dirs and files, sets the variable value +# in the manifests, and copies them to /etc/kubernetes/manifests. +function start-konnectivity-server { + echo "Start konnectivity server pods" + prepare-log-file /var/log/konnectivity-server.log + prepare-konnectivity-server-manifest "8131" "8132" "8133" +} + # Calculates the following variables based on env variables, which will be used # by the manifests of several kube-master components. # CLOUD_CONFIG_OPT @@ -1905,6 +2019,15 @@ function start-kube-apiserver { authorization_mode="Node,${authorization_mode}" params+=" --authorization-mode=${authorization_mode}" + local csc_config_mount="" + local csc_config_volume="" + if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + # Create the EgressSelectorConfiguration yaml file to control the Egress Selector. + csc_config_mount="{\"name\": \"cscconfigmount\",\"mountPath\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"readOnly\": false}," + csc_config_volume="{\"name\": \"cscconfigmount\",\"hostPath\": {\"path\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"type\": \"FileOrCreate\"}}," + params+=" --egress-selector-config-file=/etc/srv/kubernetes/egress_selector_configuration.yaml" + fi + local container_env="" if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then container_env+="{\"name\": \"KUBE_CACHE_MUTATION_DETECTOR\", \"value\": \"${ENABLE_CACHE_MUTATION_DETECTOR}\"}" @@ -1943,6 +2066,8 @@ function start-kube-apiserver { sed -i -e "s@{{webhook_authn_config_volume}}@${webhook_authn_config_volume}@g" "${src_file}" sed -i -e "s@{{webhook_config_mount}}@${webhook_config_mount}@g" "${src_file}" sed -i -e "s@{{webhook_config_volume}}@${webhook_config_volume}@g" "${src_file}" + sed -i -e "s@{{csc_config_mount}}@${csc_config_mount}@g" "${src_file}" + sed -i -e "s@{{csc_config_volume}}@${csc_config_volume}@g" "${src_file}" sed -i -e "s@{{audit_policy_config_mount}}@${audit_policy_config_mount}@g" "${src_file}" sed -i -e "s@{{audit_policy_config_volume}}@${audit_policy_config_volume}@g" "${src_file}" sed -i -e "s@{{audit_webhook_config_mount}}@${audit_webhook_config_mount}@g" "${src_file}" @@ -2675,6 +2800,11 @@ EOF setup-addon-manifests "addons" "node-termination-handler" setup-node-termination-handler-manifest fi + # Setting up the konnectivity-agent daemonset + if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + setup-addon-manifests "addons" "konnectivity-agent" + setup-konnectivity-agent-manifest + fi if [[ "${ENABLE_CLUSTER_DNS:-}" == "true" ]]; then # Create a new directory for the DNS addon and prepend a "0" on the name. # Prepending "0" to the directory ensures that add-on manager @@ -2773,6 +2903,11 @@ function setup-node-termination-handler-manifest { fi } +function setup-konnectivity-agent-manifest { + local -r manifest="/etc/kubernetes/addons/konnectivity-agent/daemonset.yaml" + sed -i "s|__APISERVER_IP__|${KUBERNETES_MASTER_NAME}|g" "${manifest}" +} + # Setups manifests for ingress controller and gce-specific policies for service controller. function start-lb-controller { setup-addon-manifests "addons" "loadbalancing" @@ -3055,6 +3190,9 @@ function main() { create-master-pki create-master-auth ensure-master-bootstrap-kubectl-auth + if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + create-master-konnectivity-server-apiserver-auth + fi create-master-kubelet-auth create-master-etcd-auth create-master-etcd-apiserver-auth @@ -3088,6 +3226,9 @@ function main() { start-etcd-empty-dir-cleanup-pod fi start-kube-apiserver + if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + start-konnectivity-server + fi start-kube-controller-manager start-kube-scheduler wait-till-apiserver-ready diff --git a/cluster/gce/manifests/BUILD b/cluster/gce/manifests/BUILD index 2af9c9b46d8..9c7379aa26b 100644 --- a/cluster/gce/manifests/BUILD +++ b/cluster/gce/manifests/BUILD @@ -18,6 +18,7 @@ filegroup( "etcd.manifest", "etcd-empty-dir-cleanup.yaml", "glbc.manifest", + "konnectivity-server.yaml", "kube-addon-manager.yaml", "kube-apiserver.manifest", "kube-controller-manager.manifest", diff --git a/cluster/gce/manifests/konnectivity-server.yaml b/cluster/gce/manifests/konnectivity-server.yaml new file mode 100644 index 00000000000..59236a579b5 --- /dev/null +++ b/cluster/gce/manifests/konnectivity-server.yaml @@ -0,0 +1,58 @@ +apiVersion: v1 +kind: Pod +metadata: + name: konnectivity-server + namespace: kube-system + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + seccomp.security.alpha.kubernetes.io/pod: 'docker/default' + component: konnectivity-server +spec: + hostNetwork: true + containers: + - name: konnectivity-server-container + image: gcr.io/google-containers/proxy-server:v0.0.3 + resources: + requests: + cpu: 40m + command: [ "/proxy-server"{{ konnectivity_args }} ] + livenessProbe: + httpGet: + scheme: HTTP + host: 127.0.0.1 + port: {{ admin_port }} + path: /healthz + initialDelaySeconds: {{ liveness_probe_initial_delay }} + timeoutSeconds: 60 + ports: + - name: serverport + containerPort: {{ server_port }} + hostPort: {{ server_port }} + - name: agentport + containerPort: {{ agent_port }} + hostPort: {{ agent_port }} + - name: adminport + containerPort: {{ admin_port }} + hostPort: {{ admin_port }} + volumeMounts: + - name: varlogkonnectivityserver + mountPath: /var/log/konnectivity-server.log + readOnly: false + - name: pkiserver + mountPath: /etc/srv/kubernetes/pki/konnectivity-server + readOnly: true + - name: pkiagent + mountPath: /etc/srv/kubernetes/pki/konnectivity-agent + readOnly: true + volumes: + - name: varlogkonnectivityserver + hostPath: + path: /var/log/konnectivity-server.log + type: FileOrCreate + - name: pkiserver + hostPath: + path: /etc/srv/kubernetes/pki/konnectivity-server + - name: pkiagent + hostPath: + path: /etc/srv/kubernetes/pki/konnectivity-agent + diff --git a/cluster/gce/manifests/kube-apiserver.manifest b/cluster/gce/manifests/kube-apiserver.manifest index 4e882da5533..02c3021282f 100644 --- a/cluster/gce/manifests/kube-apiserver.manifest +++ b/cluster/gce/manifests/kube-apiserver.manifest @@ -63,6 +63,7 @@ {{additional_cloud_config_mount}} {{webhook_config_mount}} {{webhook_authn_config_mount}} + {{csc_config_mount}} {{audit_policy_config_mount}} {{audit_webhook_config_mount}} {{webhook_exec_auth_plugin_mount}} @@ -103,6 +104,7 @@ {{additional_cloud_config_volume}} {{webhook_config_volume}} {{webhook_authn_config_volume}} + {{csc_config_volume}} {{audit_policy_config_volume}} {{audit_webhook_config_volume}} {{webhook_exec_auth_plugin_volume}} diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 01140aeecd2..1246b193eb5 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -1066,6 +1066,18 @@ ETCD_APISERVER_SERVER_KEY: $(yaml-quote ${ETCD_APISERVER_SERVER_KEY_BASE64:-}) ETCD_APISERVER_SERVER_CERT: $(yaml-quote ${ETCD_APISERVER_SERVER_CERT_BASE64:-}) ETCD_APISERVER_CLIENT_KEY: $(yaml-quote ${ETCD_APISERVER_CLIENT_KEY_BASE64:-}) ETCD_APISERVER_CLIENT_CERT: $(yaml-quote ${ETCD_APISERVER_CLIENT_CERT_BASE64:-}) +KONNECTIVITY_SERVER_CA_KEY: $(yaml-quote ${KONNECTIVITY_SERVER_CA_KEY_BASE64:-}) +KONNECTIVITY_SERVER_CA_CERT: $(yaml-quote ${KONNECTIVITY_SERVER_CA_CERT_BASE64:-}) +KONNECTIVITY_SERVER_CERT: $(yaml-quote ${KONNECTIVITY_SERVER_CERT_BASE64:-}) +KONNECTIVITY_SERVER_KEY: $(yaml-quote ${KONNECTIVITY_SERVER_KEY_BASE64:-}) +KONNECTIVITY_SERVER_CLIENT_CERT: $(yaml-quote ${KONNECTIVITY_SERVER_CLIENT_CERT_BASE64:-}) +KONNECTIVITY_SERVER_CLIENT_KEY: $(yaml-quote ${KONNECTIVITY_SERVER_CLIENT_KEY_BASE64:-}) +KONNECTIVITY_AGENT_CA_KEY: $(yaml-quote ${KONNECTIVITY_AGENT_CA_KEY_BASE64:-}) +KONNECTIVITY_AGENT_CA_CERT: $(yaml-quote ${KONNECTIVITY_AGENT_CA_CERT_BASE64:-}) +KONNECTIVITY_AGENT_CERT: $(yaml-quote ${KONNECTIVITY_AGENT_CERT_BASE64:-}) +KONNECTIVITY_AGENT_KEY: $(yaml-quote ${KONNECTIVITY_AGENT_KEY_BASE64:-}) +KONNECTIVITY_AGENT_CLIENT_CERT: $(yaml-quote ${KONNECTIVITY_AGENT_CLIENT_CERT_BASE64:-}) +KONNECTIVITY_AGENT_CLIENT_KEY: $(yaml-quote ${KONNECTIVITY_AGENT_CLIENT_KEY_BASE64:-}) EOF } @@ -1206,6 +1218,13 @@ EOF [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "cos" ]]; then cat >>$file <>$file <>$file <>$file <${cert_create_debug_output} || true + cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/aggregator + mkdir easy-rsa-master/konnectivity-server + cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-server + mkdir easy-rsa-master/konnectivity-agent + cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-agent) &>${cert_create_debug_output} || true CERT_DIR="${KUBE_TEMP}/easy-rsa-master/easyrsa3" AGGREGATOR_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/aggregator" + KONNECTIVITY_SERVER_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-server" + KONNECTIVITY_AGENT_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-agent" if [ ! -x "${CERT_DIR}/easyrsa" -o ! -x "${AGGREGATOR_CERT_DIR}/easyrsa" ]; then # TODO(roberthbailey,porridge): add better error handling here, # see https://github.com/kubernetes/kubernetes/issues/55229 @@ -1726,7 +1773,92 @@ function generate-aggregator-certs { fi } +# Runs the easy RSA commands to generate server side certificate files +# for the konnectivity server. This includes both server side to both +# konnectivity-server and konnectivity-agent. +# The generated files are in ${KONNECTIVITY_SERVER_CERT_DIR} and +# ${KONNECTIVITY_AGENT_CERT_DIR} # +# Assumed vars +# KUBE_TEMP +# KONNECTIVITY_SERVER_CERT_DIR +# KONNECTIVITY_SERVER_PRIMARY_CN: Primary canonical name +# KONNECTIVITY_SERVER_SANS: Subject alternate names +# +function generate-konnectivity-server-certs { + local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX") + # Note: This was heavily cribbed from make-ca-cert.sh + (set -x + # Make the client <-> konnectivity server side certificates. + cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-server" + ./easyrsa init-pki + # this puts the cert into pki/ca.crt and the key into pki/private/ca.key + ./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass + ./easyrsa --subject-alt-name="IP:127.0.0.1,${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass + ./easyrsa build-client-full client nopass + + kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl" + + # make the config for the signer + echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json" + # create the konnectivity server cert with the correct groups + echo '{"CN":"konnectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-server + rm -f "konnectivity-server.csr" + + # Make the agent <-> konnectivity server side certificates. + cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-agent" + ./easyrsa init-pki + # this puts the cert into pki/ca.crt and the key into pki/private/ca.key + ./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass + ./easyrsa --subject-alt-name="${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass + ./easyrsa build-client-full client nopass + + kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl" + + # make the config for the signer + echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","agent auth"]}}}' > "ca-config.json" + # create the konnectivity server cert with the correct groups + echo '{"CN":"koonectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent + rm -f "konnectivity-agent.csr" + + echo `ls ${KONNECTIVITY_SERVER_CERT_DIR}/pki/` + echo `ls ${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/` + echo `ls ${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/` + echo `ls ${KONNECTIVITY_AGENT_CERT_DIR}/pki/` + echo `ls ${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/` + echo `ls ${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/` + echo "completed main certificate section") &>${cert_create_debug_output} || true + + local output_file_missing=0 + local output_file + for output_file in \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/ca.key" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/ca.crt" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/server.crt" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/server.key" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/client.crt" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/client.key" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/ca.key" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/ca.crt" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/server.crt" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key" + do + if [[ ! -s "${output_file}" ]]; then + echo "Expected file ${output_file} not created" >&2 + output_file_missing=1 + fi + done + if (( $output_file_missing )); then + # TODO(roberthbailey,porridge): add better error handling here, + # see https://github.com/kubernetes/kubernetes/issues/55229 + cat "${cert_create_debug_output}" >&2 + echo "=== Failed to generate konnectivity-server certificates: Aborting ===" >&2 + exit 2 + fi +} + # Using provided master env, extracts value from provided key. # # Args: @@ -1766,6 +1898,16 @@ function parse-master-env() { ETCD_APISERVER_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_SERVER_CERT") ETCD_APISERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_KEY") ETCD_APISERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_CERT") + KONNECTIVITY_SERVER_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_KEY") + KONNECTIVITY_SERVER_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_CERT") + KONNECTIVITY_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CERT") + KONNECTIVITY_SERVER_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_KEY") + KONNECTIVITY_SERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_CERT") + KONNECTIVITY_SERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_KEY") + KONNECTIVITY_AGENT_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_KEY") + KONNECTIVITY_AGENT_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_CERT") + KONNECTIVITY_AGENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CERT") + KONNECTIVITY_AGENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_KEY") } # Update or verify required gcloud components are installed @@ -2505,6 +2647,15 @@ function create-master() { --target-tags "${MASTER_TAG}" \ --allow tcp:443 & + echo "Configuring firewall for apiserver konnectivity server" + if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then + gcloud compute firewall-rules create "${MASTER_NAME}-konnectivity-server" \ + --project "${NETWORK_PROJECT}" \ + --network "${NETWORK}" \ + --target-tags "${MASTER_TAG}" \ + --allow tcp:8132 & + fi + # We have to make sure the disk is created before creating the master VM, so # run this in the foreground. gcloud compute disks create "${MASTER_NAME}-pd" \ @@ -3197,7 +3348,7 @@ function kube-down() { # If there are no more remaining master replicas, we should delete all remaining network resources. if [[ "${REMAINING_MASTER_COUNT}" -eq 0 ]]; then # Delete firewall rule for the master, etcd servers, and nodes. - delete-firewall-rules "${MASTER_NAME}-https" "${MASTER_NAME}-etcd" "${NODE_TAG}-all" + delete-firewall-rules "${MASTER_NAME}-https" "${MASTER_NAME}-etcd" "${NODE_TAG}-all" "${MASTER_NAME}-konnectivity-server" # Delete the master's reserved IP if gcloud compute addresses describe "${MASTER_NAME}-ip" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then gcloud compute addresses delete \ diff --git a/cmd/kube-apiserver/app/options/options.go b/cmd/kube-apiserver/app/options/options.go index f89910ac46b..5ea40f7cdfc 100644 --- a/cmd/kube-apiserver/app/options/options.go +++ b/cmd/kube-apiserver/app/options/options.go @@ -48,6 +48,7 @@ type ServerRunOptions struct { Authorization *kubeoptions.BuiltInAuthorizationOptions CloudProvider *kubeoptions.CloudProviderOptions APIEnablement *genericoptions.APIEnablementOptions + EgressSelector *genericoptions.EgressSelectorOptions AllowPrivileged bool EnableLogsHandler bool @@ -87,6 +88,7 @@ func NewServerRunOptions() *ServerRunOptions { Authorization: kubeoptions.NewBuiltInAuthorizationOptions(), CloudProvider: kubeoptions.NewCloudProviderOptions(), APIEnablement: genericoptions.NewAPIEnablementOptions(), + EgressSelector: genericoptions.NewEgressSelectorOptions(), EnableLogsHandler: true, EventTTL: 1 * time.Hour, @@ -134,6 +136,7 @@ func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) { s.Authorization.AddFlags(fss.FlagSet("authorization")) s.CloudProvider.AddFlags(fss.FlagSet("cloud provider")) s.APIEnablement.AddFlags(fss.FlagSet("api enablement")) + s.EgressSelector.AddFlags(fss.FlagSet("egress selector")) s.Admission.AddFlags(fss.FlagSet("admission")) // Note: the weird ""+ in below lines seems to be the only way to get gofmt to diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index 917a5661f0f..8cf22957405 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -94,6 +94,7 @@ func TestAddFlags(t *testing.T) { "--cloud-provider=azure", "--cors-allowed-origins=10.10.10.100,10.10.10.200", "--contention-profiling=true", + "--egress-selector-config-file=/var/run/kubernetes/egress-selector/connectivity.yaml", "--enable-aggregator-routing=true", "--enable-logs-handler=false", "--endpoint-reconciler-type=" + string(reconcilers.LeaseEndpointReconcilerType), @@ -293,6 +294,9 @@ func TestAddFlags(t *testing.T) { APIEnablement: &apiserveroptions.APIEnablementOptions{ RuntimeConfig: cliflag.ConfigurationMap{}, }, + EgressSelector: &apiserveroptions.EgressSelectorOptions{ + ConfigFile: "/var/run/kubernetes/egress-selector/connectivity.yaml", + }, EnableLogsHandler: false, EnableAggregatorRouting: true, ProxyClientKeyFile: "/var/run/kubernetes/proxy.key", diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 490dad60c18..43782b4b7c8 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -362,6 +362,10 @@ func CreateKubeAPIServerConfig( // Use the nodeTunneler's dialer to connect to the kubelet config.ExtraConfig.KubeletClientConfig.Dial = nodeTunneler.Dial } + if config.GenericConfig.EgressSelector != nil { + // Use the config.GenericConfig.EgressSelector lookup to find the dialer to connect to the kubelet + config.ExtraConfig.KubeletClientConfig.Lookup = config.GenericConfig.EgressSelector.Lookup + } return } @@ -402,6 +406,9 @@ func buildGenericConfig( if lastErr = s.APIEnablement.ApplyTo(genericConfig, master.DefaultAPIResourceConfigSource(), legacyscheme.Scheme); lastErr != nil { return } + if lastErr = s.EgressSelector.ApplyTo(genericConfig); lastErr != nil { + return + } genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme)) genericConfig.OpenAPIConfig.Info.Title = "Kubernetes" diff --git a/pkg/kubelet/client/BUILD b/pkg/kubelet/client/BUILD index f6791f4f780..36b6a282919 100644 --- a/pkg/kubelet/client/BUILD +++ b/pkg/kubelet/client/BUILD @@ -16,6 +16,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/transport:go_default_library", ], diff --git a/pkg/kubelet/client/kubelet_client.go b/pkg/kubelet/client/kubelet_client.go index bde19b76e66..3a6e7aee8e8 100644 --- a/pkg/kubelet/client/kubelet_client.go +++ b/pkg/kubelet/client/kubelet_client.go @@ -18,6 +18,7 @@ package client import ( "context" + "fmt" "net/http" "strconv" "time" @@ -26,6 +27,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apiserver/pkg/server" restclient "k8s.io/client-go/rest" "k8s.io/client-go/transport" nodeutil "k8s.io/kubernetes/pkg/util/node" @@ -51,6 +53,9 @@ type KubeletClientConfig struct { // Dial is a custom dialer used for the client Dial utilnet.DialFunc + + // Lookup will give us a dialer if the egress selector is configured for it + Lookup server.EgressSelectorLookup } // ConnectionInfo provides the information needed to connect to a kubelet @@ -73,9 +78,20 @@ func MakeTransport(config *KubeletClientConfig) (http.RoundTripper, error) { } rt := http.DefaultTransport - if config.Dial != nil || tlsConfig != nil { + dialer := config.Dial + if dialer == nil && config.Lookup != nil { + // Assuming EgressSelector if SSHTunnel is not turned on. + // We will not get a dialer if egress selector is disabled. + networkContext := server.NetworkContext{EgressSelectionName: server.Cluster} + dialer, err = config.Lookup(networkContext) + if err != nil { + return nil, fmt.Errorf("failed to get context dialer for 'cluster': got %v", err) + } + } + if dialer != nil || tlsConfig != nil { + // If SSH Tunnel is turned on rt = utilnet.SetOldTransportDefaults(&http.Transport{ - DialContext: config.Dial, + DialContext: dialer, TLSClientConfig: tlsConfig, }) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/go.sum b/staging/src/k8s.io/apiextensions-apiserver/go.sum index 0c965371948..9b9702714d3 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/go.sum +++ b/staging/src/k8s.io/apiextensions-apiserver/go.sum @@ -215,6 +215,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/soheilhy/cmux v0.1.3 h1:09wy7WZk4AqO03yH85Ex1X+Uo3vDsil3Fa9AgF8Emss= github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.4 h1:S0tLZ3VOKl2Te0hpq8+ke0eSJPfCnNTPiDlsfwi1/NE= diff --git a/staging/src/k8s.io/apiserver/go.sum b/staging/src/k8s.io/apiserver/go.sum index b15f3d30c15..2744ecbeab0 100644 --- a/staging/src/k8s.io/apiserver/go.sum +++ b/staging/src/k8s.io/apiserver/go.sum @@ -156,6 +156,7 @@ github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/soheilhy/cmux v0.1.3 h1:09wy7WZk4AqO03yH85Ex1X+Uo3vDsil3Fa9AgF8Emss= github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/register.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/register.go index ffe9942a6cc..15519d1c47b 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/register.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/register.go @@ -45,6 +45,7 @@ var ( func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, &AdmissionConfiguration{}, + &EgressSelectorConfiguration{}, ) return nil } diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go index e55da95f95d..d21e7d631d7 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go @@ -48,3 +48,52 @@ type AdmissionPluginConfiguration struct { // +optional Configuration *runtime.Unknown } + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// EgressSelectorConfiguration provides versioned configuration for egress selector clients. +type EgressSelectorConfiguration struct { + metav1.TypeMeta + + // EgressSelections contains a list of egress selection client configurations + EgressSelections []EgressSelection +} + +// EgressSelection provides the configuration for a single egress selection client. +type EgressSelection struct { + // Name is the name of the egress selection. + // Currently supported values are "Master", "Etcd" and "Cluster" + Name string + + // Connection is the exact information used to configure the egress selection + Connection Connection +} + +// Connection provides the configuration for a single egress selection client. +type Connection struct { + // Type is the type of connection used to connect from client to konnectivity server. + // Currently supported values are "http-connect" and "direct". + Type string + + // httpConnect is the config needed to use http-connect to the konnectivity server. + // +optional + HTTPConnect *HTTPConnectConfig +} + +type HTTPConnectConfig struct { + // URL is the location of the konnectivity server to connect to. + // As an example it might be "https://127.0.0.1:8131" + URL string + + // CABundle is the file location of the CA to be used to determine trust with the konnectivity server. + // +optional + CABundle string + + // ClientKey is the file location of the client key to be used in mtls handshakes with the konnectivity server. + // +optional + ClientKey string + + // ClientCert is the file location of the client certificate to be used in mtls handshakes with the konnectivity server. + // +optional + ClientCert string +} diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/register.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/register.go index 466b19ae5bb..758be628a79 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/register.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/register.go @@ -46,6 +46,7 @@ func init() { func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, &AdmissionConfiguration{}, + &EgressSelectorConfiguration{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go index 239b8e20e04..10034d7c340 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go @@ -48,3 +48,63 @@ type AdmissionPluginConfiguration struct { // +optional Configuration *runtime.Unknown `json:"configuration"` } + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// EgressSelectorConfiguration provides versioned configuration for egress selector clients. +type EgressSelectorConfiguration struct { + metav1.TypeMeta `json:",inline"` + + // connectionServices contains a list of egress selection client configurations + EgressSelections []EgressSelection `json:"egressSelections"` +} + +// EgressSelection provides the configuration for a single egress selection client. +type EgressSelection struct { + // name is the name of the egress selection. + // Currently supported values are "Master", "Etcd" and "Cluster" + Name string `json:"name"` + + // connection is the exact information used to configure the egress selection + Connection Connection `json:"connection"` +} + +// Connection provides the configuration for a single egress selection client. +type Connection struct { + // type is the type of connection used to connect from client to network/konnectivity server. + // Currently supported values are "http-connect" and "direct". + Type string `json:"type"` + + // httpConnect is the config needed to use http-connect to the konnectivity server. + // Absence when the type is "http-connect" will cause an error + // Presence when the type is "direct" will also cause an error + // +optional + HTTPConnect *HTTPConnectConfig `json:"httpConnect,omitempty"` +} + +type HTTPConnectConfig struct { + // url is the location of the proxy server to connect to. + // As an example it might be "https://127.0.0.1:8131" + URL string `json:"url"` + + // caBundle is the file location of the CA to be used to determine trust with the konnectivity server. + // Must be absent/empty http-connect using the plain http + // Must be configured for http-connect using the https protocol + // Misconfiguration will cause an error + // +optional + CABundle string `json:"caBundle,omitempty"` + + // clientKey is the file location of the client key to be used in mtls handshakes with the konnectivity server. + // Must be absent/empty http-connect using the plain http + // Must be configured for http-connect using the https protocol + // Misconfiguration will cause an error + // +optional + ClientKey string `json:"clientKey,omitempty"` + + // clientCert is the file location of the client certificate to be used in mtls handshakes with the konnectivity server. + // Must be absent/empty http-connect using the plain http + // Must be configured for http-connect using the https protocol + // Misconfiguration will cause an error + // +optional + ClientCert string `json:"clientCert,omitempty"` +} diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go index 64909b34a8f..80352f02e91 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go @@ -55,6 +55,46 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*Connection)(nil), (*apiserver.Connection)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_Connection_To_apiserver_Connection(a.(*Connection), b.(*apiserver.Connection), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*apiserver.Connection)(nil), (*Connection)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_apiserver_Connection_To_v1alpha1_Connection(a.(*apiserver.Connection), b.(*Connection), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*EgressSelection)(nil), (*apiserver.EgressSelection)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_EgressSelection_To_apiserver_EgressSelection(a.(*EgressSelection), b.(*apiserver.EgressSelection), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*apiserver.EgressSelection)(nil), (*EgressSelection)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_apiserver_EgressSelection_To_v1alpha1_EgressSelection(a.(*apiserver.EgressSelection), b.(*EgressSelection), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*EgressSelectorConfiguration)(nil), (*apiserver.EgressSelectorConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration(a.(*EgressSelectorConfiguration), b.(*apiserver.EgressSelectorConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*apiserver.EgressSelectorConfiguration)(nil), (*EgressSelectorConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration(a.(*apiserver.EgressSelectorConfiguration), b.(*EgressSelectorConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*HTTPConnectConfig)(nil), (*apiserver.HTTPConnectConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(a.(*HTTPConnectConfig), b.(*apiserver.HTTPConnectConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*apiserver.HTTPConnectConfig)(nil), (*HTTPConnectConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(a.(*apiserver.HTTPConnectConfig), b.(*HTTPConnectConfig), scope) + }); err != nil { + return err + } return nil } @@ -101,3 +141,97 @@ func autoConvert_apiserver_AdmissionPluginConfiguration_To_v1alpha1_AdmissionPlu func Convert_apiserver_AdmissionPluginConfiguration_To_v1alpha1_AdmissionPluginConfiguration(in *apiserver.AdmissionPluginConfiguration, out *AdmissionPluginConfiguration, s conversion.Scope) error { return autoConvert_apiserver_AdmissionPluginConfiguration_To_v1alpha1_AdmissionPluginConfiguration(in, out, s) } + +func autoConvert_v1alpha1_Connection_To_apiserver_Connection(in *Connection, out *apiserver.Connection, s conversion.Scope) error { + out.Type = in.Type + out.HTTPConnect = (*apiserver.HTTPConnectConfig)(unsafe.Pointer(in.HTTPConnect)) + return nil +} + +// Convert_v1alpha1_Connection_To_apiserver_Connection is an autogenerated conversion function. +func Convert_v1alpha1_Connection_To_apiserver_Connection(in *Connection, out *apiserver.Connection, s conversion.Scope) error { + return autoConvert_v1alpha1_Connection_To_apiserver_Connection(in, out, s) +} + +func autoConvert_apiserver_Connection_To_v1alpha1_Connection(in *apiserver.Connection, out *Connection, s conversion.Scope) error { + out.Type = in.Type + out.HTTPConnect = (*HTTPConnectConfig)(unsafe.Pointer(in.HTTPConnect)) + return nil +} + +// Convert_apiserver_Connection_To_v1alpha1_Connection is an autogenerated conversion function. +func Convert_apiserver_Connection_To_v1alpha1_Connection(in *apiserver.Connection, out *Connection, s conversion.Scope) error { + return autoConvert_apiserver_Connection_To_v1alpha1_Connection(in, out, s) +} + +func autoConvert_v1alpha1_EgressSelection_To_apiserver_EgressSelection(in *EgressSelection, out *apiserver.EgressSelection, s conversion.Scope) error { + out.Name = in.Name + if err := Convert_v1alpha1_Connection_To_apiserver_Connection(&in.Connection, &out.Connection, s); err != nil { + return err + } + return nil +} + +// Convert_v1alpha1_EgressSelection_To_apiserver_EgressSelection is an autogenerated conversion function. +func Convert_v1alpha1_EgressSelection_To_apiserver_EgressSelection(in *EgressSelection, out *apiserver.EgressSelection, s conversion.Scope) error { + return autoConvert_v1alpha1_EgressSelection_To_apiserver_EgressSelection(in, out, s) +} + +func autoConvert_apiserver_EgressSelection_To_v1alpha1_EgressSelection(in *apiserver.EgressSelection, out *EgressSelection, s conversion.Scope) error { + out.Name = in.Name + if err := Convert_apiserver_Connection_To_v1alpha1_Connection(&in.Connection, &out.Connection, s); err != nil { + return err + } + return nil +} + +// Convert_apiserver_EgressSelection_To_v1alpha1_EgressSelection is an autogenerated conversion function. +func Convert_apiserver_EgressSelection_To_v1alpha1_EgressSelection(in *apiserver.EgressSelection, out *EgressSelection, s conversion.Scope) error { + return autoConvert_apiserver_EgressSelection_To_v1alpha1_EgressSelection(in, out, s) +} + +func autoConvert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration(in *EgressSelectorConfiguration, out *apiserver.EgressSelectorConfiguration, s conversion.Scope) error { + out.EgressSelections = *(*[]apiserver.EgressSelection)(unsafe.Pointer(&in.EgressSelections)) + return nil +} + +// Convert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration is an autogenerated conversion function. +func Convert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration(in *EgressSelectorConfiguration, out *apiserver.EgressSelectorConfiguration, s conversion.Scope) error { + return autoConvert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration(in, out, s) +} + +func autoConvert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration(in *apiserver.EgressSelectorConfiguration, out *EgressSelectorConfiguration, s conversion.Scope) error { + out.EgressSelections = *(*[]EgressSelection)(unsafe.Pointer(&in.EgressSelections)) + return nil +} + +// Convert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration is an autogenerated conversion function. +func Convert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration(in *apiserver.EgressSelectorConfiguration, out *EgressSelectorConfiguration, s conversion.Scope) error { + return autoConvert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration(in, out, s) +} + +func autoConvert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(in *HTTPConnectConfig, out *apiserver.HTTPConnectConfig, s conversion.Scope) error { + out.URL = in.URL + out.CABundle = in.CABundle + out.ClientKey = in.ClientKey + out.ClientCert = in.ClientCert + return nil +} + +// Convert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig is an autogenerated conversion function. +func Convert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(in *HTTPConnectConfig, out *apiserver.HTTPConnectConfig, s conversion.Scope) error { + return autoConvert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(in, out, s) +} + +func autoConvert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(in *apiserver.HTTPConnectConfig, out *HTTPConnectConfig, s conversion.Scope) error { + out.URL = in.URL + out.CABundle = in.CABundle + out.ClientKey = in.ClientKey + out.ClientCert = in.ClientCert + return nil +} + +// Convert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig is an autogenerated conversion function. +func Convert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(in *apiserver.HTTPConnectConfig, out *HTTPConnectConfig, s conversion.Scope) error { + return autoConvert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(in, out, s) +} diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.deepcopy.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.deepcopy.go index 24151bbd22b..e8d6086798b 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.deepcopy.go @@ -76,3 +76,89 @@ func (in *AdmissionPluginConfiguration) DeepCopy() *AdmissionPluginConfiguration in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Connection) DeepCopyInto(out *Connection) { + *out = *in + if in.HTTPConnect != nil { + in, out := &in.HTTPConnect, &out.HTTPConnect + *out = new(HTTPConnectConfig) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Connection. +func (in *Connection) DeepCopy() *Connection { + if in == nil { + return nil + } + out := new(Connection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressSelection) DeepCopyInto(out *EgressSelection) { + *out = *in + in.Connection.DeepCopyInto(&out.Connection) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressSelection. +func (in *EgressSelection) DeepCopy() *EgressSelection { + if in == nil { + return nil + } + out := new(EgressSelection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressSelectorConfiguration) DeepCopyInto(out *EgressSelectorConfiguration) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.EgressSelections != nil { + in, out := &in.EgressSelections, &out.EgressSelections + *out = make([]EgressSelection, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressSelectorConfiguration. +func (in *EgressSelectorConfiguration) DeepCopy() *EgressSelectorConfiguration { + if in == nil { + return nil + } + out := new(EgressSelectorConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EgressSelectorConfiguration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HTTPConnectConfig) DeepCopyInto(out *HTTPConnectConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPConnectConfig. +func (in *HTTPConnectConfig) DeepCopy() *HTTPConnectConfig { + if in == nil { + return nil + } + out := new(HTTPConnectConfig) + in.DeepCopyInto(out) + return out +} diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/zz_generated.deepcopy.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/zz_generated.deepcopy.go index 542ef977b6b..3159f7c1a34 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/zz_generated.deepcopy.go @@ -76,3 +76,89 @@ func (in *AdmissionPluginConfiguration) DeepCopy() *AdmissionPluginConfiguration in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Connection) DeepCopyInto(out *Connection) { + *out = *in + if in.HTTPConnect != nil { + in, out := &in.HTTPConnect, &out.HTTPConnect + *out = new(HTTPConnectConfig) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Connection. +func (in *Connection) DeepCopy() *Connection { + if in == nil { + return nil + } + out := new(Connection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressSelection) DeepCopyInto(out *EgressSelection) { + *out = *in + in.Connection.DeepCopyInto(&out.Connection) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressSelection. +func (in *EgressSelection) DeepCopy() *EgressSelection { + if in == nil { + return nil + } + out := new(EgressSelection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressSelectorConfiguration) DeepCopyInto(out *EgressSelectorConfiguration) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.EgressSelections != nil { + in, out := &in.EgressSelections, &out.EgressSelections + *out = make([]EgressSelection, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressSelectorConfiguration. +func (in *EgressSelectorConfiguration) DeepCopy() *EgressSelectorConfiguration { + if in == nil { + return nil + } + out := new(EgressSelectorConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EgressSelectorConfiguration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HTTPConnectConfig) DeepCopyInto(out *HTTPConnectConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPConnectConfig. +func (in *HTTPConnectConfig) DeepCopy() *HTTPConnectConfig { + if in == nil { + return nil + } + out := new(HTTPConnectConfig) + in.DeepCopyInto(out) + return out +} diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/rest/streamer.go b/staging/src/k8s.io/apiserver/pkg/registry/generic/rest/streamer.go index 562288a3aee..be911fab2f1 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/rest/streamer.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/rest/streamer.go @@ -62,6 +62,7 @@ func (s *LocationStreamer) InputStream(ctx context.Context, apiVersion, acceptHe if transport == nil { transport = http.DefaultTransport } + client := &http.Client{ Transport: transport, CheckRedirect: s.RedirectChecker, diff --git a/staging/src/k8s.io/apiserver/pkg/server/BUILD b/staging/src/k8s.io/apiserver/pkg/server/BUILD index 43bf3731c4a..de85f2dcc42 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/BUILD @@ -11,6 +11,7 @@ go_test( srcs = [ "config_selfclient_test.go", "config_test.go", + "egress_selector_test.go", "genericapiserver_test.go", "healthz_test.go", ], @@ -21,9 +22,11 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/version:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/apiserver:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/example:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/example/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", @@ -49,6 +52,7 @@ go_library( "config_selfclient.go", "deprecated_insecure_serving.go", "doc.go", + "egress_selector.go", "genericapiserver.go", "handler.go", "healthz.go", @@ -70,6 +74,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library", @@ -79,6 +84,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/apiserver:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/apiserver/install:go_default_library", "//staging/src/k8s.io/apiserver/pkg/audit:go_default_library", "//staging/src/k8s.io/apiserver/pkg/audit/policy:go_default_library", @@ -133,6 +139,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//staging/src/k8s.io/apiserver/pkg/server/egressselector:all-srcs", "//staging/src/k8s.io/apiserver/pkg/server/filters:all-srcs", "//staging/src/k8s.io/apiserver/pkg/server/healthz:all-srcs", "//staging/src/k8s.io/apiserver/pkg/server/httplog:all-srcs", diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index d00637e5515..91fac23ddcd 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -29,7 +29,7 @@ import ( "sync/atomic" "time" - jsonpatch "github.com/evanphx/json-patch" + "github.com/evanphx/json-patch" "github.com/go-openapi/spec" "github.com/pborman/uuid" @@ -94,6 +94,11 @@ type Config struct { // This is required for proper functioning of the PostStartHooks on a GenericAPIServer // TODO: move into SecureServing(WithLoopback) as soon as insecure serving is gone LoopbackClientConfig *restclient.Config + + // EgressSelector provides a lookup mechanism for dialing outbound connections. + // It does so based on a EgressSelectorConfiguration which was read at startup. + EgressSelector *EgressSelector + // RuleResolver is required to get the list of rules that apply to a given user // in a given namespace RuleResolver authorizer.RuleResolver diff --git a/staging/src/k8s.io/apiserver/pkg/server/egress_selector.go b/staging/src/k8s.io/apiserver/pkg/server/egress_selector.go new file mode 100644 index 00000000000..58a9dd320ad --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/egress_selector.go @@ -0,0 +1,192 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "bufio" + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apiserver/pkg/apis/apiserver" + "k8s.io/klog" + "net" + "net/http" + "net/url" + "strings" +) + +var directDialer utilnet.DialFunc = http.DefaultTransport.(*http.Transport).DialContext + +type EgressSelector struct { + egressToDialer map[EgressType]utilnet.DialFunc +} + +// EgressType is an indicator of which egress selection should be used for sending traffic. +// See https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190226-network-proxy.md#network-context +type EgressType int + +const ( + // Master is the EgressType for traffic intended to go to the control plane. + Master EgressType = iota + // Etcd is the EgressType for traffic intended to go to Kubernetes persistence store. + Etcd + // Cluster is the EgressType for traffic intended to go to the system being managed by Kubernetes. + Cluster +) + +// NetworkContext is the struct used by Kubernetes API Server to indicate where it intends traffic to be sent. +type NetworkContext struct { + // EgressSelectionName is the unique name of the + // EgressSelectorConfiguration which determines + // the network we route the traffic to. + EgressSelectionName EgressType +} + +// EgressSelectorLookup is the interface to get the dialer function for the network context. +type EgressSelectorLookup func(networkContext NetworkContext) (utilnet.DialFunc, error) + +func (s EgressType) String() string { + switch s { + case Master: + return "master" + case Etcd: + return "etcd" + case Cluster: + return "cluster" + default: + return "invalid" + } +} + +func lookupServiceName(name string) (EgressType, error) { + switch strings.ToLower(name) { + case "master": + return Master, nil + case "etcd": + return Etcd, nil + case "cluster": + return Cluster, nil + } + return -1, fmt.Errorf("unrecognized service name %s", name) +} + +func createConnectDialer(connectConfig *apiserver.HTTPConnectConfig) (utilnet.DialFunc, error) { + clientCert := connectConfig.ClientCert + clientKey := connectConfig.ClientKey + caCert := connectConfig.CABundle + proxyURL, err := url.Parse(connectConfig.URL) + if err != nil { + return nil, fmt.Errorf("invalid proxy server url %q: %v", connectConfig.URL, err) + } + proxyAddress := proxyURL.Host + + clientCerts, err := tls.LoadX509KeyPair(clientCert, clientKey) + if err != nil { + return nil, fmt.Errorf("failed to read key pair %s & %s, got %v", clientCert, clientKey, err) + } + certPool := x509.NewCertPool() + certBytes, err := ioutil.ReadFile(caCert) + if err != nil { + return nil, fmt.Errorf("failed to read cert file %s, got %v", caCert, err) + } + ok := certPool.AppendCertsFromPEM(certBytes) + if !ok { + return nil, fmt.Errorf("failed to append CA cert to the cert pool") + } + contextDialer := func(ctx context.Context, network, addr string) (net.Conn, error) { + klog.V(4).Infof("Sending request to %q.", addr) + proxyConn, err := tls.Dial("tcp", proxyAddress, + &tls.Config{ + Certificates: []tls.Certificate{clientCerts}, + RootCAs: certPool, + }, + ) + if err != nil { + return nil, fmt.Errorf("dialing proxy %q failed: %v", proxyAddress, err) + } + fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", addr, "127.0.0.1") + br := bufio.NewReader(proxyConn) + res, err := http.ReadResponse(br, nil) + if err != nil { + proxyConn.Close() + return nil, fmt.Errorf("reading HTTP response from CONNECT to %s via proxy %s failed: %v", + addr, proxyAddress, err) + } + if res.StatusCode != 200 { + proxyConn.Close() + return nil, fmt.Errorf("proxy error from %s while dialing %s, code %d: %v", + proxyAddress, addr, res.StatusCode, res.Status) + } + + // It's safe to discard the bufio.Reader here and return the + // original TCP conn directly because we only use this for + // TLS, and in TLS the client speaks first, so we know there's + // no unbuffered data. But we can double-check. + if br.Buffered() > 0 { + proxyConn.Close() + return nil, fmt.Errorf("unexpected %d bytes of buffered data from CONNECT proxy %q", + br.Buffered(), proxyAddress) + } + klog.V(4).Infof("About to proxy request to %s over %s.", addr, proxyAddress) + return proxyConn, nil + } + return contextDialer, nil +} + +// NewEgressSelector configures lookup mechanism for Lookup. +// It does so based on a EgressSelectorConfiguration which was read at startup. +func NewEgressSelector(config *apiserver.EgressSelectorConfiguration) (*EgressSelector, error) { + if config == nil || config.EgressSelections == nil { + // No Connection Services configured, leaving the serviceMap empty, will return default dialer. + return nil, nil + } + cs := &EgressSelector{ + egressToDialer: make(map[EgressType]utilnet.DialFunc), + } + for _, service := range config.EgressSelections { + name, err := lookupServiceName(service.Name) + if err != nil { + return nil, err + } + switch service.Connection.Type { + case "http-connect": + contextDialer, err := createConnectDialer(service.Connection.HTTPConnect) + if err != nil { + return nil, fmt.Errorf("failed to create http-connect dialer: %v", err) + } + cs.egressToDialer[name] = contextDialer + case "direct": + cs.egressToDialer[name] = directDialer + default: + return nil, fmt.Errorf("unrecognized service connection type %q", service.Connection.Type) + } + } + return cs, nil +} + +// Lookup gets the dialer function for the network context. +// This is configured for the Kubernetes API Server at startup. +func (cs *EgressSelector) Lookup(networkContext NetworkContext) (utilnet.DialFunc, error) { + if cs.egressToDialer == nil { + // The round trip wrapper will over-ride the dialContext method appropriately + return nil, nil + } + return cs.egressToDialer[networkContext.EgressSelectionName], nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/egress_selector_test.go b/staging/src/k8s.io/apiserver/pkg/server/egress_selector_test.go new file mode 100644 index 00000000000..825e53415d8 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/egress_selector_test.go @@ -0,0 +1,183 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "context" + "net" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apiserver/pkg/apis/apiserver" +) + +type fakeEgressSelection struct { + directDialerCalled bool +} + +func TestEgressSelector(t *testing.T) { + testcases := []struct { + name string + input *apiserver.EgressSelectorConfiguration + services []struct { + egressType EgressType + validateDialer func(dialer utilnet.DialFunc, s *fakeEgressSelection) (bool, error) + lookupError *string + dialerError *string + } + expectedError *string + }{ + { + name: "direct", + input: &apiserver.EgressSelectorConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "", + APIVersion: "", + }, + EgressSelections: []apiserver.EgressSelection{ + { + Name: "cluster", + Connection: apiserver.Connection{ + Type: "direct", + HTTPConnect: &apiserver.HTTPConnectConfig{ + URL: "", + CABundle: "", + ClientKey: "", + ClientCert: "", + }, + }, + }, + { + Name: "master", + Connection: apiserver.Connection{ + Type: "direct", + HTTPConnect: &apiserver.HTTPConnectConfig{ + URL: "", + CABundle: "", + ClientKey: "", + ClientCert: "", + }, + }, + }, + { + Name: "etcd", + Connection: apiserver.Connection{ + Type: "direct", + HTTPConnect: &apiserver.HTTPConnectConfig{ + URL: "", + CABundle: "", + ClientKey: "", + ClientCert: "", + }, + }, + }, + }, + }, + services: []struct { + egressType EgressType + validateDialer func(dialer utilnet.DialFunc, s *fakeEgressSelection) (bool, error) + lookupError *string + dialerError *string + }{ + { + Cluster, + validateDirectDialer, + nil, + nil, + }, + { + Master, + validateDirectDialer, + nil, + nil, + }, + { + Etcd, + validateDirectDialer, + nil, + nil, + }, + }, + expectedError: nil, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + // Setup the various pieces such as the fake dialer prior to initializing the egress selector. + // Go doesn't allow function pointer comparison, nor does its reflect package + // So overriding the default dialer to detect if it is returned. + fake := &fakeEgressSelection{} + directDialer = fake.fakeDirectDialer + cs, err := NewEgressSelector(tc.input) + if err == nil && tc.expectedError != nil { + t.Errorf("calling NewEgressSelector expected error: %s, did not get it", *tc.expectedError) + } + if err != nil && tc.expectedError == nil { + t.Errorf("unexpected error calling NewEgressSelector got: %#v", err) + } + if err != nil && tc.expectedError != nil && err.Error() != *tc.expectedError { + t.Errorf("calling NewEgressSelector expected error: %s, got %#v", *tc.expectedError, err) + } + + for _, service := range tc.services { + networkContext := NetworkContext{EgressSelectionName: service.egressType} + dialer, lookupErr := cs.Lookup(networkContext) + if lookupErr == nil && service.lookupError != nil { + t.Errorf("calling Lookup expected error: %s, did not get it", *service.lookupError) + } + if lookupErr != nil && service.lookupError == nil { + t.Errorf("unexpected error calling Lookup got: %#v", lookupErr) + } + if lookupErr != nil && service.lookupError != nil && lookupErr.Error() != *service.lookupError { + t.Errorf("calling Lookup expected error: %s, got %#v", *service.lookupError, lookupErr) + } + fake.directDialerCalled = false + ok, dialerErr := service.validateDialer(dialer, fake) + if dialerErr == nil && service.dialerError != nil { + t.Errorf("calling Lookup expected error: %s, did not get it", *service.dialerError) + } + if dialerErr != nil && service.dialerError == nil { + t.Errorf("unexpected error calling Lookup got: %#v", dialerErr) + } + if dialerErr != nil && service.dialerError != nil && dialerErr.Error() != *service.dialerError { + t.Errorf("calling Lookup expected error: %s, got %#v", *service.dialerError, dialerErr) + } + if !ok { + t.Errorf("Could not validate dialer for service %q", service.egressType) + } + } + }) + } +} + +func (s *fakeEgressSelection) fakeDirectDialer(ctx context.Context, network, address string) (net.Conn, error) { + s.directDialerCalled = true + return nil, nil +} + +func validateDirectDialer(dialer utilnet.DialFunc, s *fakeEgressSelection) (bool, error) { + conn, err := dialer(context.Background(), "tcp", "127.0.0.1:8080") + if err != nil { + return false, err + } + if conn != nil { + return false, nil + } + return s.directDialerCalled, nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/egressselector/BUILD b/staging/src/k8s.io/apiserver/pkg/server/egressselector/BUILD new file mode 100644 index 00000000000..8df8af8ac63 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/egressselector/BUILD @@ -0,0 +1,42 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["config.go"], + importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/egressselector", + importpath = "k8s.io/apiserver/pkg/server/egressselector", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/apiserver:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/apiserver/install:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1:go_default_library", + "//vendor/k8s.io/utils/path:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = ["config_test.go"], + embed = [":go_default_library"], + deps = [ + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/apiserver:go_default_library", + ], +) diff --git a/staging/src/k8s.io/apiserver/pkg/server/egressselector/config.go b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config.go new file mode 100644 index 00000000000..a14714edaf1 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config.go @@ -0,0 +1,179 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package egressselector + +import ( + "fmt" + "io/ioutil" + "strings" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apiserver/pkg/apis/apiserver" + "k8s.io/apiserver/pkg/apis/apiserver/install" + "k8s.io/apiserver/pkg/apis/apiserver/v1alpha1" + "k8s.io/utils/path" + "sigs.k8s.io/yaml" +) + +var cfgScheme = runtime.NewScheme() + +func init() { + install.Install(cfgScheme) +} + +// ReadEgressSelectorConfiguration reads the egress selector configuration at the specified path. +// It returns the loaded egress selector configuration if the input file aligns with the required syntax. +// If it does not align with the provided syntax, it returns a default configuration which should function as a no-op. +// It does this by returning a nil configuration, which preserves backward compatibility. +// This works because prior to this there was no egress selector configuration. +// It returns an error if the file did not exist. +func ReadEgressSelectorConfiguration(configFilePath string) (*apiserver.EgressSelectorConfiguration, error) { + if configFilePath == "" { + return nil, nil + } + // a file was provided, so we just read it. + data, err := ioutil.ReadFile(configFilePath) + if err != nil { + return nil, fmt.Errorf("unable to read egress selector configuration from %q [%v]", configFilePath, err) + } + var decodedConfig v1alpha1.EgressSelectorConfiguration + err = yaml.Unmarshal(data, &decodedConfig) + if err != nil { + // we got an error where the decode wasn't related to a missing type + return nil, err + } + if decodedConfig.Kind != "EgressSelectorConfiguration" { + return nil, fmt.Errorf("invalid service configuration object %q", decodedConfig.Kind) + } + config, err := cfgScheme.ConvertToVersion(&decodedConfig, apiserver.SchemeGroupVersion) + if err != nil { + // we got an error where the decode wasn't related to a missing type + return nil, err + } + if internalConfig, ok := config.(*apiserver.EgressSelectorConfiguration); ok { + return internalConfig, nil + } + return nil, fmt.Errorf("unable to convert %T to *apiserver.EgressSelectorConfiguration", config) +} + +// ValidateEgressSelectorConfiguration checks the apiserver.EgressSelectorConfiguration for +// common configuration errors. It will return error for problems such as configuring mtls/cert +// settings for protocol which do not support security. It will also try to catch errors such as +// incorrect file paths. It will return nil if it does not find anything wrong. +func ValidateEgressSelectorConfiguration(config *apiserver.EgressSelectorConfiguration) field.ErrorList { + allErrs := field.ErrorList{} + if config == nil { + return allErrs // Treating a nil configuration as valid + } + for _, service := range config.EgressSelections { + base := field.NewPath("service", "connection") + switch service.Connection.Type { + case "direct": + allErrs = append(allErrs, validateDirectConnection(service.Connection, base)...) + case "http-connect": + allErrs = append(allErrs, validateHTTPConnection(service.Connection, base)...) + default: + allErrs = append(allErrs, field.NotSupported( + base.Child("type"), + service.Connection.Type, + []string{"direct", "http-connect"})) + } + } + + return allErrs +} + +func validateDirectConnection(connection apiserver.Connection, fldPath *field.Path) field.ErrorList { + if connection.HTTPConnect != nil { + return field.ErrorList{field.Invalid( + fldPath.Child("httpConnect"), + "direct", + "httpConnect config should be absent for direct connect"), + } + } + return nil +} + +func validateHTTPConnection(connection apiserver.Connection, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if connection.HTTPConnect == nil { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect"), + "nil", + "httpConnect config should be present for http-connect")) + } else if strings.HasPrefix(connection.HTTPConnect.URL, "https://") { + if connection.HTTPConnect.CABundle == "" { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "caBundle"), + "nil", + "http-connect via https requires caBundle")) + } else if exists, err := path.Exists(path.CheckFollowSymlink, connection.HTTPConnect.CABundle); exists == false || err != nil { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "caBundle"), + connection.HTTPConnect.CABundle, + "http-connect ca bundle does not exist")) + } + if connection.HTTPConnect.ClientCert == "" { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "clientCert"), + "nil", + "http-connect via https requires clientCert")) + } else if exists, err := path.Exists(path.CheckFollowSymlink, connection.HTTPConnect.ClientCert); exists == false || err != nil { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "clientCert"), + connection.HTTPConnect.ClientCert, + "http-connect client cert does not exist")) + } + if connection.HTTPConnect.ClientKey == "" { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "clientKey"), + "nil", + "http-connect via https requires clientKey")) + } else if exists, err := path.Exists(path.CheckFollowSymlink, connection.HTTPConnect.ClientKey); exists == false || err != nil { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "clientKey"), + connection.HTTPConnect.ClientKey, + "http-connect client key does not exist")) + } + } else if strings.HasPrefix(connection.HTTPConnect.URL, "http://") { + if connection.HTTPConnect.CABundle != "" { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "caBundle"), + connection.HTTPConnect.CABundle, + "http-connect via http does not support caBundle")) + } + if connection.HTTPConnect.ClientCert != "" { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "clientCert"), + connection.HTTPConnect.ClientCert, + "http-connect via http does not support clientCert")) + } + if connection.HTTPConnect.ClientKey != "" { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "clientKey"), + connection.HTTPConnect.ClientKey, + "http-connect via http does not support clientKey")) + } + } else { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("httpConnect", "url"), + connection.HTTPConnect.URL, + "supported connection protocols are http:// and https://")) + } + return allErrs +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go new file mode 100644 index 00000000000..feefe5fe11e --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go @@ -0,0 +1,214 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package egressselector + +import ( + "fmt" + "io/ioutil" + "os" + "reflect" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/apis/apiserver" +) + +func strptr(s string) *string { + return &s +} + +func TestReadEgressSelectorConfiguration(t *testing.T) { + testcases := []struct { + name string + contents string + createFile bool + expectedResult *apiserver.EgressSelectorConfiguration + expectedError *string + }{ + { + name: "empty", + createFile: true, + contents: ``, + expectedResult: nil, + expectedError: strptr("invalid service configuration object \"\""), + }, + { + name: "absent", + createFile: false, + contents: ``, + expectedResult: nil, + expectedError: strptr("unable to read egress selector configuration from \"test-egress-selector-config-absent\" [open test-egress-selector-config-absent: no such file or directory]"), + }, + { + name: "v1alpha1", + createFile: true, + contents: ` +apiVersion: apiserver.k8s.io/v1alpha1 +kind: EgressSelectorConfiguration +egressSelections: +- name: "cluster" + connection: + type: "http-connect" + httpConnect: + url: "https://127.0.0.1:8131" + caBundle: "/etc/srv/kubernetes/pki/konnectivity-server/ca.crt" + clientKey: "/etc/srv/kubernetes/pki/konnectivity-server/client.key" + clientCert: "/etc/srv/kubernetes/pki/konnectivity-server/client.crt" +- name: "master" + connection: + type: "http-connect" + httpConnect: + url: "https://127.0.0.1:8132" + caBundle: "/etc/srv/kubernetes/pki/konnectivity-server-master/ca.crt" + clientKey: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.key" + clientCert: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.crt" +- name: "etcd" + connection: + type: "direct" +`, + expectedResult: &apiserver.EgressSelectorConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "", + APIVersion: "", + }, + EgressSelections: []apiserver.EgressSelection{ + { + Name: "cluster", + Connection: apiserver.Connection{ + Type: "http-connect", + HTTPConnect: &apiserver.HTTPConnectConfig{ + URL: "https://127.0.0.1:8131", + CABundle: "/etc/srv/kubernetes/pki/konnectivity-server/ca.crt", + ClientKey: "/etc/srv/kubernetes/pki/konnectivity-server/client.key", + ClientCert: "/etc/srv/kubernetes/pki/konnectivity-server/client.crt", + }, + }, + }, + { + Name: "master", + Connection: apiserver.Connection{ + Type: "http-connect", + HTTPConnect: &apiserver.HTTPConnectConfig{ + URL: "https://127.0.0.1:8132", + CABundle: "/etc/srv/kubernetes/pki/konnectivity-server-master/ca.crt", + ClientKey: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.key", + ClientCert: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.crt", + }, + }, + }, + { + Name: "etcd", + Connection: apiserver.Connection{ + Type: "direct", + }, + }, + }, + }, + expectedError: nil, + }, + { + name: "wrong_type", + createFile: true, + contents: ` +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + addonmanager.kubernetes.io/mode: Reconcile + k8s-app: konnectivity-agent + namespace: kube-system + name: proxy-agent +spec: + selector: + matchLabels: + k8s-app: konnectivity-agent + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: proxy-agent + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + priorityClassName: system-cluster-critical + # Necessary to reboot node + hostPID: true + volumes: + - name: pki + hostPath: + path: /etc/srv/kubernetes/pki/konnectivity-agent + containers: + - image: gcr.io/google-containers/proxy-agent:v0.0.3 + name: proxy-agent + command: ["/proxy-agent"] + args: ["--caCert=/etc/srv/kubernetes/pki/proxy-agent/ca.crt", "--agentCert=/etc/srv/kubernetes/pki/proxy-agent/client.crt", "--agentKey=/etc/srv/kubernetes/pki/proxy-agent/client.key", "--proxyServerHost=127.0.0.1", "--proxyServerPort=8132"] + securityContext: + capabilities: + add: ["SYS_BOOT"] + env: + - name: wrong-type + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: kube-system + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + limits: + cpu: 50m + memory: 30Mi + volumeMounts: + - name: pki + mountPath: /etc/srv/kubernetes/pki/konnectivity-agent +`, + expectedResult: nil, + expectedError: strptr("invalid service configuration object \"DaemonSet\""), + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + proxyConfig := fmt.Sprintf("test-egress-selector-config-%s", tc.name) + if tc.createFile { + f, err := ioutil.TempFile("", proxyConfig) + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + if err := ioutil.WriteFile(f.Name(), []byte(tc.contents), os.FileMode(0755)); err != nil { + t.Fatal(err) + } + proxyConfig = f.Name() + } + config, err := ReadEgressSelectorConfiguration(proxyConfig) + if err == nil && tc.expectedError != nil { + t.Errorf("calling ReadEgressSelectorConfiguration expected error: %s, did not get it", *tc.expectedError) + } + if err != nil && tc.expectedError == nil { + t.Errorf("unexpected error calling ReadEgressSelectorConfiguration got: %#v", err) + } + if err != nil && tc.expectedError != nil && err.Error() != *tc.expectedError { + t.Errorf("calling ReadEgressSelectorConfiguration expected error: %s, got %#v", *tc.expectedError, err) + } + if !reflect.DeepEqual(config, tc.expectedResult) { + t.Errorf("problem with configuration returned from ReadEgressSelectorConfiguration expected: %#v, got: %#v", tc.expectedResult, config) + } + }) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/BUILD b/staging/src/k8s.io/apiserver/pkg/server/options/BUILD index ec477368a61..a1ae49080f9 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/options/BUILD @@ -11,6 +11,7 @@ go_library( "coreapi.go", "deprecated_insecure_serving.go", "doc.go", + "egress_selector.go", "etcd.go", "events.go", "feature.go", @@ -56,6 +57,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server/egressselector:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/resourceconfig:go_default_library", @@ -83,6 +85,7 @@ go_library( "//vendor/gopkg.in/natefinch/lumberjack.v2:go_default_library", "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library", + "//vendor/k8s.io/utils/path:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/egress_selector.go b/staging/src/k8s.io/apiserver/pkg/server/options/egress_selector.go new file mode 100644 index 00000000000..837e32fcfaf --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/options/egress_selector.go @@ -0,0 +1,92 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "fmt" + "github.com/spf13/pflag" + "k8s.io/utils/path" + + "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/egressselector" +) + +// EgressSelectorOptions holds the api server egress selector options. +// See https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190226-network-proxy.md +type EgressSelectorOptions struct { + // ConfigFile is the file path with api-server egress selector configuration. + ConfigFile string +} + +// NewEgressSelectorOptions creates a new instance of EgressSelectorOptions +// +// The option is to point to a configuration file for egress/konnectivity. +// This determines which types of requests use egress/konnectivity and how they use it. +// If empty the API Server will attempt to connect directly using the network. +func NewEgressSelectorOptions() *EgressSelectorOptions { + return &EgressSelectorOptions{} +} + +// AddFlags adds flags related to admission for a specific APIServer to the specified FlagSet +func (o *EgressSelectorOptions) AddFlags(fs *pflag.FlagSet) { + if o == nil { + return + } + + fs.StringVar(&o.ConfigFile, "egress-selector-config-file", o.ConfigFile, + "File with apiserver egress selector configuration.") +} + +// ApplyTo adds the egress selector settings to the server configuration. +// In case egress selector settings were not provided by a cluster-admin +// they will be prepared from the recommended/default/no-op values. +func (o *EgressSelectorOptions) ApplyTo(c *server.Config) error { + if o == nil { + return nil + } + + npConfig, err := egressselector.ReadEgressSelectorConfiguration(o.ConfigFile) + if err != nil { + return fmt.Errorf("failed to read egress selector config: %v", err) + } + errs := egressselector.ValidateEgressSelectorConfiguration(npConfig) + if len(errs) > 0 { + return fmt.Errorf("failed to validate egress selector configuration: %v", errs.ToAggregate()) + } + + cs, err := server.NewEgressSelector(npConfig) + if err != nil { + return fmt.Errorf("failed to setup egress selector with config %#v: %v", npConfig, err) + } + c.EgressSelector = cs + return nil +} + +// Validate verifies flags passed to EgressSelectorOptions. +func (o *EgressSelectorOptions) Validate() []error { + if o == nil || o.ConfigFile == "" { + return nil + } + + errs := []error{} + + if exists, err := path.Exists(path.CheckFollowSymlink, o.ConfigFile); exists == false || err != nil { + errs = append(errs, fmt.Errorf("egress-selector-config-file %s does not exist", o.ConfigFile)) + } + + return errs +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go b/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go index 631b4f43e39..2ddb901a922 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go @@ -44,6 +44,8 @@ type RecommendedOptions struct { // ProcessInfo is used to identify events created by the server. ProcessInfo *ProcessInfo Webhook *WebhookOptions + // API Server Egress Selector is used to control outbound traffic from the API Server + EgressSelector *EgressSelectorOptions } func NewRecommendedOptions(prefix string, codec runtime.Codec, processInfo *ProcessInfo) *RecommendedOptions { @@ -67,6 +69,7 @@ func NewRecommendedOptions(prefix string, codec runtime.Codec, processInfo *Proc Admission: NewAdmissionOptions(), ProcessInfo: processInfo, Webhook: NewWebhookOptions(), + EgressSelector: NewEgressSelectorOptions(), } } @@ -79,6 +82,7 @@ func (o *RecommendedOptions) AddFlags(fs *pflag.FlagSet) { o.Features.AddFlags(fs) o.CoreAPI.AddFlags(fs) o.Admission.AddFlags(fs) + o.EgressSelector.AddFlags(fs) } // ApplyTo adds RecommendedOptions to the server configuration. @@ -110,6 +114,9 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error { } else if err := o.Admission.ApplyTo(&config.Config, config.SharedInformerFactory, config.ClientConfig, initializers...); err != nil { return err } + if err := o.EgressSelector.ApplyTo(&config.Config); err != nil { + return err + } return nil } @@ -124,6 +131,7 @@ func (o *RecommendedOptions) Validate() []error { errors = append(errors, o.Features.Validate()...) errors = append(errors, o.CoreAPI.Validate()...) errors = append(errors, o.Admission.Validate()...) + errors = append(errors, o.EgressSelector.Validate()...) return errors } diff --git a/staging/src/k8s.io/code-generator/_examples/HyphenGroup/apis/example/v1/zz_generated.defaults.go b/staging/src/k8s.io/code-generator/_examples/HyphenGroup/apis/example/v1/zz_generated.defaults.go new file mode 100644 index 00000000000..cce2e603a69 --- /dev/null +++ b/staging/src/k8s.io/code-generator/_examples/HyphenGroup/apis/example/v1/zz_generated.defaults.go @@ -0,0 +1,32 @@ +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by defaulter-gen. DO NOT EDIT. + +package v1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// RegisterDefaults adds defaulters functions to the given scheme. +// Public to allow building arbitrary schemes. +// All generated defaulters are covering - they call all nested defaulters. +func RegisterDefaults(scheme *runtime.Scheme) error { + return nil +} diff --git a/staging/src/k8s.io/kube-aggregator/go.sum b/staging/src/k8s.io/kube-aggregator/go.sum index eb46d25ae64..0bd22d1e01a 100644 --- a/staging/src/k8s.io/kube-aggregator/go.sum +++ b/staging/src/k8s.io/kube-aggregator/go.sum @@ -171,6 +171,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/soheilhy/cmux v0.1.3 h1:09wy7WZk4AqO03yH85Ex1X+Uo3vDsil3Fa9AgF8Emss= github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.4 h1:S0tLZ3VOKl2Te0hpq8+ke0eSJPfCnNTPiDlsfwi1/NE= diff --git a/staging/src/k8s.io/sample-apiserver/go.sum b/staging/src/k8s.io/sample-apiserver/go.sum index 79a4b96887f..92956bad004 100644 --- a/staging/src/k8s.io/sample-apiserver/go.sum +++ b/staging/src/k8s.io/sample-apiserver/go.sum @@ -168,6 +168,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/soheilhy/cmux v0.1.3 h1:09wy7WZk4AqO03yH85Ex1X+Uo3vDsil3Fa9AgF8Emss= github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.4 h1:S0tLZ3VOKl2Te0hpq8+ke0eSJPfCnNTPiDlsfwi1/NE= diff --git a/vendor/modules.txt b/vendor/modules.txt index 186bae47edb..c2412826ba8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1257,6 +1257,7 @@ k8s.io/apiserver/pkg/registry/generic/testing k8s.io/apiserver/pkg/registry/rest k8s.io/apiserver/pkg/registry/rest/resttest k8s.io/apiserver/pkg/server +k8s.io/apiserver/pkg/server/egressselector k8s.io/apiserver/pkg/server/filters k8s.io/apiserver/pkg/server/healthz k8s.io/apiserver/pkg/server/httplog