Merge pull request #78543 from cheftako/kas-np3

Get network-proxy working with GCE.

build/lib/release.sh

@@ -432,6 +432,7 @@ function kube::release::package_kube_manifests_tarball() {
   cp "${src_dir}/etcd.manifest" "${dst_dir}"
   cp "${src_dir}/kube-scheduler.manifest" "${dst_dir}"
   cp "${src_dir}/kube-apiserver.manifest" "${dst_dir}"
+  cp "${src_dir}/konnectivity-server.yaml" "${dst_dir}"
   cp "${src_dir}/abac-authz-policy.jsonl" "${dst_dir}"
   cp "${src_dir}/kube-controller-manager.manifest" "${dst_dir}"
   cp "${src_dir}/kube-addon-manager.yaml" "${dst_dir}"

cluster/gce/addons/konnectivity-agent/daemonset.yaml

@@ -0,0 +1,66 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    k8s-app: konnectivity-agent
+  namespace: kube-system
+  name: konnectivity-agent
+spec:
+  selector:
+    matchLabels:
+      k8s-app: konnectivity-agent
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        k8s-app: konnectivity-agent
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      priorityClassName: system-cluster-critical
+      tolerations:
+        - key: "CriticalAddonsOnly"
+          operator: "Exists"
+      hostNetwork: true
+      volumes:
+        - name: pki
+          hostPath:
+            path: /etc/srv/kubernetes/pki/konnectivity-agent
+      containers:
+        - image: gcr.io/google-containers/proxy-agent:v0.0.3
+          name: konnectivity-agent
+          command: ["/proxy-agent"]
+          args: [
+                  "--logtostderr=true",
+                  "--ca-cert=/etc/srv/kubernetes/pki/konnectivity-agent/ca.crt",
+                  "--agent-cert=/etc/srv/kubernetes/pki/konnectivity-agent/client.crt",
+                  "--agent-key=/etc/srv/kubernetes/pki/konnectivity-agent/client.key",
+                  "--proxy-server-host=__APISERVER_IP__",
+                  "--proxy-server-port=8132"
+                  ]
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          resources:
+            limits:
+              cpu: 50m
+              memory: 30Mi
+          livenessProbe:
+            httpGet:
+              host: 127.0.0.1
+              port: 8093
+              path: /healthz
+            initialDelaySeconds: 15
+            timeoutSeconds: 15
+          volumeMounts:
+            - name: pki
+              mountPath: /etc/srv/kubernetes/pki/konnectivity-agent

cluster/gce/config-default.sh

@@ -480,3 +480,6 @@ WINDOWS_NODE_TAINTS="${WINDOWS_NODE_TAINTS:-node.kubernetes.io/os=win1809:NoSche
 
 # Whether to set up a private GCE cluster, i.e. a cluster where nodes have only private IPs.
 GCE_PRIVATE_CLUSTER="${KUBE_GCE_PRIVATE_CLUSTER:-false}"
+
+# Optional: Create apiserver konnectivity server and agent.
+ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE="${KUBE_ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}"

cluster/gce/gci/configure-helper.sh

@@ -495,6 +495,16 @@ function create-node-pki {
     KUBELET_KEY_PATH="${pki_dir}/kubelet.key"
     write-pki-data "${KUBELET_KEY}" "${KUBELET_KEY_PATH}"
   fi
+
+  if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then
+    mkdir -p "${pki_dir}/konnectivity-agent"
+    KONNECTIVITY_AGENT_CA_CERT_PATH="${pki_dir}/konnectivity-agent/ca.crt"
+    KONNECTIVITY_AGENT_CLIENT_KEY_PATH="${pki_dir}/konnectivity-agent/client.key"
+    KONNECTIVITY_AGENT_CLIENT_CERT_PATH="${pki_dir}/konnectivity-agent/client.crt"
+    write-pki-data "${KONNECTIVITY_AGENT_CA_CERT}" "${KONNECTIVITY_AGENT_CA_CERT_PATH}"
+    write-pki-data "${KONNECTIVITY_AGENT_CLIENT_KEY}" "${KONNECTIVITY_AGENT_CLIENT_KEY_PATH}"
+    write-pki-data "${KONNECTIVITY_AGENT_CLIENT_CERT}" "${KONNECTIVITY_AGENT_CLIENT_CERT_PATH}"
+  fi
 }
 
 function create-master-pki {
@@ -558,6 +568,42 @@ function create-master-pki {
     PROXY_CLIENT_CERT_PATH="${pki_dir}/proxy_client.crt"
     write-pki-data "${PROXY_CLIENT_CERT}" "${PROXY_CLIENT_CERT_PATH}"
   fi
+
+  if [[ ! -z "${KONNECTIVITY_SERVER_CA_CERT:-}" ]]; then
+    mkdir -p "${pki_dir}"/konnectivity-server
+    #KONNECTIVITY_SERVER_CA_KEY_PATH="${pki_dir}/konnectivity-server/ca.key"
+    #write-pki-data "${KONNECTIVITY_SERVER_CA_KEY}" "${KONNECTIVITY_SERVER_CA_KEY_PATH}"
+
+    KONNECTIVITY_SERVER_CA_CERT_PATH="${pki_dir}/konnectivity-server/ca.crt"
+    write-pki-data "${KONNECTIVITY_SERVER_CA_CERT}" "${KONNECTIVITY_SERVER_CA_CERT_PATH}"
+
+    KONNECTIVITY_SERVER_KEY_PATH="${pki_dir}/konnectivity-server/server.key"
+    write-pki-data "${KONNECTIVITY_SERVER_KEY}" "${KONNECTIVITY_SERVER_KEY_PATH}"
+
+    KONNECTIVITY_SERVER_CERT_PATH="${pki_dir}/konnectivity-server/server.crt"
+    write-pki-data "${KONNECTIVITY_SERVER_CERT}" "${KONNECTIVITY_SERVER_CERT_PATH}"
+
+    KONNECTIVITY_SERVER_CLIENT_KEY_PATH="${pki_dir}/konnectivity-server/client.key"
+    write-pki-data "${KONNECTIVITY_SERVER_CLIENT_KEY}" "${KONNECTIVITY_SERVER_CLIENT_KEY_PATH}"
+
+    KONNECTIVITY_SERVER_CLIENT_CERT_PATH="${pki_dir}/konnectivity-server/client.crt"
+    write-pki-data "${KONNECTIVITY_SERVER_CLIENT_CERT}" "${KONNECTIVITY_SERVER_CLIENT_CERT_PATH}"
+  fi
+
+  if [[ ! -z "${KONNECTIVITY_AGENT_CA_CERT:-}" ]]; then
+    mkdir -p "${pki_dir}"/konnectivity-agent
+    KONNECTIVITY_AGENT_CA_KEY_PATH="${pki_dir}/konnectivity-agent/ca.key"
+    write-pki-data "${KONNECTIVITY_AGENT_CA_KEY}" "${KONNECTIVITY_AGENT_CA_KEY_PATH}"
+
+    KONNECTIVITY_AGENT_CA_CERT_PATH="${pki_dir}/konnectivity-agent/ca.crt"
+    write-pki-data "${KONNECTIVITY_AGENT_CA_CERT}" "${KONNECTIVITY_AGENT_CA_CERT_PATH}"
+
+    KONNECTIVITY_AGENT_KEY_PATH="${pki_dir}/konnectivity-agent/server.key"
+    write-pki-data "${KONNECTIVITY_AGENT_KEY}" "${KONNECTIVITY_AGENT_KEY_PATH}"
+
+    KONNECTIVITY_AGENT_CERT_PATH="${pki_dir}/konnectivity-agent/server.crt"
+    write-pki-data "${KONNECTIVITY_AGENT_CERT}" "${KONNECTIVITY_AGENT_CERT_PATH}"
+  fi
 }
 
 # After the first boot and on upgrade, these files exist on the master-pd
@@ -757,6 +803,27 @@ contexts:
   name: webhook
 EOF
   fi
+  if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then
+    cat <<EOF >/etc/srv/kubernetes/egress_selector_configuration.yaml
+apiVersion: apiserver.k8s.io/v1alpha1
+kind: EgressSelectorConfiguration
+egressSelections:
+- name: cluster
+  connection:
+    type: http-connect
+    httpConnect:
+      url: https://127.0.0.1:8131
+      caBundle: /etc/srv/kubernetes/pki/konnectivity-server/ca.crt
+      clientKey: /etc/srv/kubernetes/pki/konnectivity-server/client.key
+      clientCert: /etc/srv/kubernetes/pki/konnectivity-server/client.crt
+- name: master
+  connection:
+    type: direct
+- name: etcd
+  connection:
+    type: direct
+EOF
+  fi
 
   if [[ -n "${WEBHOOK_GKE_EXEC_AUTH:-}" ]]; then
     if [[ -z "${EXEC_AUTH_PLUGIN_URL:-}" ]]; then
@@ -1214,6 +1281,10 @@ function create-master-etcd-apiserver-auth {
    fi
 }
 
+function create-master-konnectivity-server-apiserver-auth {
+  echo TODO: implement create-master-konnectivity-server-apiserver-auth
+}
+
 function assemble-docker-flags {
   echo "Assemble docker command line flags"
   local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false"
@@ -1562,6 +1633,49 @@ function start-etcd-servers {
   prepare-etcd-manifest "-events" "4002" "2381" "100m" "etcd-events.manifest"
 }
 
+# Replaces the variables in the konnectivity-server manifest file with the real values, and then
+# copy the file to the manifest dir
+# $1: value for variable "server_port"
+# $2: value for variable "agent_port"
+# $3: value for bariable "admin_port"
+function prepare-konnectivity-server-manifest {
+  local -r temp_file="/tmp/konnectivity-server.yaml"
+  params=()
+  cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/konnectivity-server.yaml" "${temp_file}"
+  params+=("--log-file=/var/log/konnectivity-server.log")
+  params+=("--logtostderr=false")
+  params+=("--log-file-max-size=0")
+  params+=("--server-ca-cert=${KONNECTIVITY_SERVER_CA_CERT_PATH}")
+  params+=("--server-cert=${KONNECTIVITY_SERVER_CERT_PATH}")
+  params+=("--server-key=${KONNECTIVITY_SERVER_KEY_PATH}")
+  params+=("--cluster-ca-cert=${KONNECTIVITY_AGENT_CA_CERT_PATH}")
+  params+=("--cluster-cert=${KONNECTIVITY_AGENT_CERT_PATH}")
+  params+=("--cluster-key=${KONNECTIVITY_AGENT_KEY_PATH}")
+  params+=("--mode=http-connect")
+  params+=("--server-port=$1")
+  params+=("--agent-port=$2")
+  params+=("--admin-port=$3")
+  konnectivity_args=""
+  for param in "${params[@]}"; do
+    konnectivity_args+=", \"${param}\""
+  done
+  sed -i -e "s@{{ *konnectivity_args *}}@${konnectivity_args}@g" "${temp_file}"
+  sed -i -e "s@{{ *server_port *}}@$1@g" "${temp_file}"
+  sed -i -e "s@{{ *agent_port *}}@$2@g" "${temp_file}"
+  sed -i -e "s@{{ *admin_port *}}@$3@g" "${temp_file}"
+  sed -i -e "s@{{ *liveness_probe_initial_delay *}}@30@g" "${temp_file}"
+  mv "${temp_file}" /etc/kubernetes/manifests
+}
+
+# Starts konnectivity server pod.
+# More specifically, it prepares dirs and files, sets the variable value
+# in the manifests, and copies them to /etc/kubernetes/manifests.
+function start-konnectivity-server {
+  echo "Start konnectivity server pods"
+  prepare-log-file /var/log/konnectivity-server.log
+  prepare-konnectivity-server-manifest "8131" "8132" "8133"
+}
+
 # Calculates the following variables based on env variables, which will be used
 # by the manifests of several kube-master components.
 #   CLOUD_CONFIG_OPT
@@ -1905,6 +2019,15 @@ function start-kube-apiserver {
   authorization_mode="Node,${authorization_mode}"
   params+=" --authorization-mode=${authorization_mode}"
 
+  local csc_config_mount=""
+  local csc_config_volume=""
+  if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then
+    # Create the EgressSelectorConfiguration yaml file to control the Egress Selector.
+    csc_config_mount="{\"name\": \"cscconfigmount\",\"mountPath\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"readOnly\": false},"
+    csc_config_volume="{\"name\": \"cscconfigmount\",\"hostPath\": {\"path\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"type\": \"FileOrCreate\"}},"
+    params+=" --egress-selector-config-file=/etc/srv/kubernetes/egress_selector_configuration.yaml"
+  fi
+
   local container_env=""
   if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then
     container_env+="{\"name\": \"KUBE_CACHE_MUTATION_DETECTOR\", \"value\": \"${ENABLE_CACHE_MUTATION_DETECTOR}\"}"
@@ -1943,6 +2066,8 @@ function start-kube-apiserver {
   sed -i -e "s@{{webhook_authn_config_volume}}@${webhook_authn_config_volume}@g" "${src_file}"
   sed -i -e "s@{{webhook_config_mount}}@${webhook_config_mount}@g" "${src_file}"
   sed -i -e "s@{{webhook_config_volume}}@${webhook_config_volume}@g" "${src_file}"
+  sed -i -e "s@{{csc_config_mount}}@${csc_config_mount}@g" "${src_file}"
+  sed -i -e "s@{{csc_config_volume}}@${csc_config_volume}@g" "${src_file}"
   sed -i -e "s@{{audit_policy_config_mount}}@${audit_policy_config_mount}@g" "${src_file}"
   sed -i -e "s@{{audit_policy_config_volume}}@${audit_policy_config_volume}@g" "${src_file}"
   sed -i -e "s@{{audit_webhook_config_mount}}@${audit_webhook_config_mount}@g" "${src_file}"
@@ -2675,6 +2800,11 @@ EOF
       setup-addon-manifests "addons" "node-termination-handler"
       setup-node-termination-handler-manifest
   fi
+  # Setting up the konnectivity-agent daemonset
+  if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then
+    setup-addon-manifests "addons" "konnectivity-agent"
+    setup-konnectivity-agent-manifest
+  fi
   if [[ "${ENABLE_CLUSTER_DNS:-}" == "true" ]]; then
     # Create a new directory for the DNS addon and prepend a "0" on the name.
     # Prepending "0" to the directory ensures that add-on manager
@@ -2773,6 +2903,11 @@ function setup-node-termination-handler-manifest {
     fi
 }
 
+function setup-konnectivity-agent-manifest {
+    local -r manifest="/etc/kubernetes/addons/konnectivity-agent/daemonset.yaml"
+    sed -i "s|__APISERVER_IP__|${KUBERNETES_MASTER_NAME}|g" "${manifest}"
+}
+
 # Setups manifests for ingress controller and gce-specific policies for service controller.
 function start-lb-controller {
   setup-addon-manifests "addons" "loadbalancing"
@@ -3055,6 +3190,9 @@ function main() {
     create-master-pki
     create-master-auth
     ensure-master-bootstrap-kubectl-auth
+    if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then
+      create-master-konnectivity-server-apiserver-auth
+    fi
     create-master-kubelet-auth
     create-master-etcd-auth
     create-master-etcd-apiserver-auth
@@ -3088,6 +3226,9 @@ function main() {
       start-etcd-empty-dir-cleanup-pod
     fi
     start-kube-apiserver
+    if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then
+      start-konnectivity-server
+    fi
     start-kube-controller-manager
     start-kube-scheduler
     wait-till-apiserver-ready

cluster/gce/manifests/BUILD

@@ -18,6 +18,7 @@ filegroup(
         "etcd.manifest",
         "etcd-empty-dir-cleanup.yaml",
         "glbc.manifest",
+        "konnectivity-server.yaml",
         "kube-addon-manager.yaml",
         "kube-apiserver.manifest",
         "kube-controller-manager.manifest",

cluster/gce/manifests/konnectivity-server.yaml

@@ -0,0 +1,58 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: konnectivity-server
+  namespace: kube-system
+  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ''
+    seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
+  component: konnectivity-server
+spec:
+  hostNetwork: true
+  containers:
+  - name: konnectivity-server-container
+    image: gcr.io/google-containers/proxy-server:v0.0.3
+    resources:
+      requests:
+        cpu: 40m
+    command: [ "/proxy-server"{{ konnectivity_args }} ]
+    livenessProbe:
+      httpGet:
+        scheme: HTTP
+        host: 127.0.0.1
+        port: {{ admin_port }}
+        path: /healthz
+      initialDelaySeconds: {{ liveness_probe_initial_delay }}
+      timeoutSeconds: 60
+    ports:
+    - name: serverport
+      containerPort: {{ server_port }}
+      hostPort: {{ server_port }}
+    - name: agentport
+      containerPort: {{ agent_port }}
+      hostPort: {{ agent_port }}
+    - name: adminport
+      containerPort: {{ admin_port }}
+      hostPort: {{ admin_port }}
+    volumeMounts:
+    - name: varlogkonnectivityserver
+      mountPath: /var/log/konnectivity-server.log
+      readOnly: false
+    - name: pkiserver
+      mountPath: /etc/srv/kubernetes/pki/konnectivity-server
+      readOnly: true
+    - name: pkiagent
+      mountPath: /etc/srv/kubernetes/pki/konnectivity-agent
+      readOnly: true
+  volumes:
+  - name: varlogkonnectivityserver
+    hostPath:
+      path: /var/log/konnectivity-server.log
+      type: FileOrCreate
+  - name: pkiserver
+    hostPath:
+      path: /etc/srv/kubernetes/pki/konnectivity-server
+  - name: pkiagent
+    hostPath:
+      path: /etc/srv/kubernetes/pki/konnectivity-agent
+

cluster/gce/manifests/kube-apiserver.manifest

@@ -63,6 +63,7 @@
         {{additional_cloud_config_mount}}
         {{webhook_config_mount}}
         {{webhook_authn_config_mount}}
+        {{csc_config_mount}}
         {{audit_policy_config_mount}}
         {{audit_webhook_config_mount}}
 	{{webhook_exec_auth_plugin_mount}}
@@ -103,6 +104,7 @@
   {{additional_cloud_config_volume}}
   {{webhook_config_volume}}
   {{webhook_authn_config_volume}}
+  {{csc_config_volume}}
   {{audit_policy_config_volume}}
   {{audit_webhook_config_volume}}
   {{webhook_exec_auth_plugin_volume}}

cluster/gce/util.sh

@@ -1066,6 +1066,18 @@ ETCD_APISERVER_SERVER_KEY: $(yaml-quote ${ETCD_APISERVER_SERVER_KEY_BASE64:-})
 ETCD_APISERVER_SERVER_CERT: $(yaml-quote ${ETCD_APISERVER_SERVER_CERT_BASE64:-})
 ETCD_APISERVER_CLIENT_KEY: $(yaml-quote ${ETCD_APISERVER_CLIENT_KEY_BASE64:-})
 ETCD_APISERVER_CLIENT_CERT: $(yaml-quote ${ETCD_APISERVER_CLIENT_CERT_BASE64:-})
+KONNECTIVITY_SERVER_CA_KEY: $(yaml-quote ${KONNECTIVITY_SERVER_CA_KEY_BASE64:-})
+KONNECTIVITY_SERVER_CA_CERT: $(yaml-quote ${KONNECTIVITY_SERVER_CA_CERT_BASE64:-})
+KONNECTIVITY_SERVER_CERT: $(yaml-quote ${KONNECTIVITY_SERVER_CERT_BASE64:-})
+KONNECTIVITY_SERVER_KEY: $(yaml-quote ${KONNECTIVITY_SERVER_KEY_BASE64:-})
+KONNECTIVITY_SERVER_CLIENT_CERT: $(yaml-quote ${KONNECTIVITY_SERVER_CLIENT_CERT_BASE64:-})
+KONNECTIVITY_SERVER_CLIENT_KEY: $(yaml-quote ${KONNECTIVITY_SERVER_CLIENT_KEY_BASE64:-})
+KONNECTIVITY_AGENT_CA_KEY: $(yaml-quote ${KONNECTIVITY_AGENT_CA_KEY_BASE64:-})
+KONNECTIVITY_AGENT_CA_CERT: $(yaml-quote ${KONNECTIVITY_AGENT_CA_CERT_BASE64:-})
+KONNECTIVITY_AGENT_CERT: $(yaml-quote ${KONNECTIVITY_AGENT_CERT_BASE64:-})
+KONNECTIVITY_AGENT_KEY: $(yaml-quote ${KONNECTIVITY_AGENT_KEY_BASE64:-})
+KONNECTIVITY_AGENT_CLIENT_CERT: $(yaml-quote ${KONNECTIVITY_AGENT_CLIENT_CERT_BASE64:-})
+KONNECTIVITY_AGENT_CLIENT_KEY: $(yaml-quote ${KONNECTIVITY_AGENT_CLIENT_KEY_BASE64:-})
 EOF
 }
 
@@ -1206,6 +1218,13 @@ EOF
      [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "cos" ]]; then
     cat >>$file <<EOF
 REMOUNT_VOLUME_PLUGIN_DIR: $(yaml-quote ${REMOUNT_VOLUME_PLUGIN_DIR:-true})
+EOF
+  fi
+  if [[ "${master}" == "false" ]]; then
+    cat >>$file <<EOF
+KONNECTIVITY_AGENT_CA_CERT: $(yaml-quote ${KONNECTIVITY_AGENT_CA_CERT_BASE64:-})
+KONNECTIVITY_AGENT_CLIENT_KEY: $(yaml-quote ${KONNECTIVITY_AGENT_CLIENT_KEY_BASE64:-})
+KONNECTIVITY_AGENT_CLIENT_CERT: $(yaml-quote ${KONNECTIVITY_AGENT_CLIENT_CERT_BASE64:-})
 EOF
   fi
   if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then
@@ -1462,6 +1481,11 @@ EOF
   if [ -n "${MAX_PODS_PER_NODE:-}" ]; then
     cat >>$file <<EOF
 MAX_PODS_PER_NODE: $(yaml-quote ${MAX_PODS_PER_NODE})
+EOF
+  fi
+  if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then
+      cat >>$file <<EOF
+ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE: $(yaml-quote ${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE})
 EOF
   fi
 }
@@ -1549,6 +1573,7 @@ function create-certs {
   setup-easyrsa
   PRIMARY_CN="${primary_cn}" SANS="${sans}" generate-certs
   AGGREGATOR_PRIMARY_CN="${primary_cn}" AGGREGATOR_SANS="${sans}" generate-aggregator-certs
+  KONNECTIVITY_SERVER_PRIMARY_CN="${primary_cn}" KONNECTIVITY_SERVER_SANS="${sans}" generate-konnectivity-server-certs
 
   # By default, linux wraps base64 output every 76 cols, so we use 'tr -d' to remove whitespaces.
   # Note 'base64 -w0' doesn't work on Mac OS X, which has different flags.
@@ -1570,6 +1595,22 @@ function create-certs {
   REQUESTHEADER_CA_CERT_BASE64=$(cat "${AGGREGATOR_CERT_DIR}/pki/ca.crt" | base64 | tr -d '\r\n')
   PROXY_CLIENT_CERT_BASE64=$(cat "${AGGREGATOR_CERT_DIR}/pki/issued/proxy-client.crt" | base64 | tr -d '\r\n')
   PROXY_CLIENT_KEY_BASE64=$(cat "${AGGREGATOR_CERT_DIR}/pki/private/proxy-client.key" | base64 | tr -d '\r\n')
+
+  # Setting up the Kubernetes API Server Konnectivity Server auth.
+  # This includes certs for both API Server to Konnectivity Server and
+  # Konnectivity Agent to Konnectivity Server.
+  KONNECTIVITY_SERVER_CA_KEY_BASE64=$(cat "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/ca.key" | base64 | tr -d '\r\n')
+  KONNECTIVITY_SERVER_CA_CERT_BASE64=$(cat "${KONNECTIVITY_SERVER_CERT_DIR}/pki/ca.crt" | base64 | tr -d '\r\n')
+  KONNECTIVITY_SERVER_CERT_BASE64=$(cat "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/server.crt" | base64 | tr -d '\r\n')
+  KONNECTIVITY_SERVER_KEY_BASE64=$(cat "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/server.key" | base64 | tr -d '\r\n')
+  KONNECTIVITY_SERVER_CLIENT_CERT_BASE64=$(cat "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/client.crt" | base64 | tr -d '\r\n')
+  KONNECTIVITY_SERVER_CLIENT_KEY_BASE64=$(cat "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/client.key" | base64 | tr -d '\r\n')
+  KONNECTIVITY_AGENT_CA_KEY_BASE64=$(cat "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/ca.key" | base64 | tr -d '\r\n')
+  KONNECTIVITY_AGENT_CA_CERT_BASE64=$(cat "${KONNECTIVITY_AGENT_CERT_DIR}/pki/ca.crt" | base64 | tr -d '\r\n')
+  KONNECTIVITY_AGENT_CERT_BASE64=$(cat "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/server.crt" | base64 | tr -d '\r\n')
+  KONNECTIVITY_AGENT_KEY_BASE64=$(cat "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" | base64 | tr -d '\r\n')
+  KONNECTIVITY_AGENT_CLIENT_CERT_BASE64=$(cat "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" | base64 | tr -d '\r\n')
+  KONNECTIVITY_AGENT_CLIENT_KEY_BASE64=$(cat "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key" | base64 | tr -d '\r\n')
 }
 
 # Set up easy-rsa directory structure.
@@ -1590,9 +1631,15 @@ function setup-easyrsa {
     mkdir easy-rsa-master/kubelet
     cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/kubelet
     mkdir easy-rsa-master/aggregator
-    cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/aggregator) &>${cert_create_debug_output} || true
+    cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/aggregator
+    mkdir easy-rsa-master/konnectivity-server
+    cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-server
+    mkdir easy-rsa-master/konnectivity-agent
+    cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-agent) &>${cert_create_debug_output} || true
   CERT_DIR="${KUBE_TEMP}/easy-rsa-master/easyrsa3"
   AGGREGATOR_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/aggregator"
+  KONNECTIVITY_SERVER_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-server"
+  KONNECTIVITY_AGENT_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-agent"
   if [ ! -x "${CERT_DIR}/easyrsa" -o ! -x "${AGGREGATOR_CERT_DIR}/easyrsa" ]; then
     # TODO(roberthbailey,porridge): add better error handling here,
     # see https://github.com/kubernetes/kubernetes/issues/55229
@@ -1726,7 +1773,92 @@ function generate-aggregator-certs {
   fi
 }
 
+# Runs the easy RSA commands to generate server side certificate files
+# for the konnectivity server. This includes both server side to both
+# konnectivity-server and konnectivity-agent.
+# The generated files are in ${KONNECTIVITY_SERVER_CERT_DIR} and
+# ${KONNECTIVITY_AGENT_CERT_DIR}
 #
+# Assumed vars
+#   KUBE_TEMP
+#   KONNECTIVITY_SERVER_CERT_DIR
+#   KONNECTIVITY_SERVER_PRIMARY_CN: Primary canonical name
+#   KONNECTIVITY_SERVER_SANS: Subject alternate names
+#
+function generate-konnectivity-server-certs {
+  local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
+  # Note: This was heavily cribbed from make-ca-cert.sh
+  (set -x
+    # Make the client <-> konnectivity server side certificates.
+    cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-server"
+    ./easyrsa init-pki
+    # this puts the cert into pki/ca.crt and the key into pki/private/ca.key
+    ./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass
+    ./easyrsa --subject-alt-name="IP:127.0.0.1,${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass
+    ./easyrsa build-client-full client nopass
+
+    kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
+
+    # make the config for the signer
+    echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json"
+    # create the konnectivity server cert with the correct groups
+    echo '{"CN":"konnectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-server
+    rm -f "konnectivity-server.csr"
+
+    # Make the agent <-> konnectivity server side certificates.
+    cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-agent"
+    ./easyrsa init-pki
+    # this puts the cert into pki/ca.crt and the key into pki/private/ca.key
+    ./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass
+    ./easyrsa --subject-alt-name="${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass
+    ./easyrsa build-client-full client nopass
+
+    kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
+
+    # make the config for the signer
+    echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","agent auth"]}}}' > "ca-config.json"
+    # create the konnectivity server cert with the correct groups
+    echo '{"CN":"koonectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent
+    rm -f "konnectivity-agent.csr"
+
+    echo `ls ${KONNECTIVITY_SERVER_CERT_DIR}/pki/`
+    echo `ls ${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/`
+    echo `ls ${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/`
+    echo `ls ${KONNECTIVITY_AGENT_CERT_DIR}/pki/`
+    echo `ls ${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/`
+    echo `ls ${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/`
+    echo "completed main certificate section") &>${cert_create_debug_output} || true
+
+  local output_file_missing=0
+  local output_file
+  for output_file in \
+    "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/ca.key" \
+    "${KONNECTIVITY_SERVER_CERT_DIR}/pki/ca.crt" \
+    "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/server.crt" \
+    "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/server.key" \
+    "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/client.crt" \
+    "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/client.key" \
+    "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/ca.key" \
+    "${KONNECTIVITY_AGENT_CERT_DIR}/pki/ca.crt" \
+    "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/server.crt" \
+    "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" \
+    "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" \
+    "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key"
+  do
+    if [[ ! -s "${output_file}" ]]; then
+      echo "Expected file ${output_file} not created" >&2
+      output_file_missing=1
+    fi
+  done
+  if (( $output_file_missing )); then
+    # TODO(roberthbailey,porridge): add better error handling here,
+    # see https://github.com/kubernetes/kubernetes/issues/55229
+    cat "${cert_create_debug_output}" >&2
+    echo "=== Failed to generate konnectivity-server certificates: Aborting ===" >&2
+    exit 2
+  fi
+}
+
 # Using provided master env, extracts value from provided key.
 #
 # Args:
@@ -1766,6 +1898,16 @@ function parse-master-env() {
   ETCD_APISERVER_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_SERVER_CERT")
   ETCD_APISERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_KEY")
   ETCD_APISERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_CERT")
+  KONNECTIVITY_SERVER_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_KEY")
+  KONNECTIVITY_SERVER_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_CERT")
+  KONNECTIVITY_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CERT")
+  KONNECTIVITY_SERVER_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_KEY")
+  KONNECTIVITY_SERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_CERT")
+  KONNECTIVITY_SERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_KEY")
+  KONNECTIVITY_AGENT_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_KEY")
+  KONNECTIVITY_AGENT_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_CERT")
+  KONNECTIVITY_AGENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CERT")
+  KONNECTIVITY_AGENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_KEY")
 }
 
 # Update or verify required gcloud components are installed
@@ -2505,6 +2647,15 @@ function create-master() {
     --target-tags "${MASTER_TAG}" \
     --allow tcp:443 &
 
+  echo "Configuring firewall for apiserver konnectivity server"
+  if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then
+    gcloud compute firewall-rules create "${MASTER_NAME}-konnectivity-server" \
+      --project "${NETWORK_PROJECT}" \
+      --network "${NETWORK}" \
+      --target-tags "${MASTER_TAG}" \
+      --allow tcp:8132 &
+  fi
+
   # We have to make sure the disk is created before creating the master VM, so
   # run this in the foreground.
   gcloud compute disks create "${MASTER_NAME}-pd" \
@@ -3197,7 +3348,7 @@ function kube-down() {
   # If there are no more remaining master replicas, we should delete all remaining network resources.
   if [[ "${REMAINING_MASTER_COUNT}" -eq 0 ]]; then
     # Delete firewall rule for the master, etcd servers, and nodes.
-    delete-firewall-rules "${MASTER_NAME}-https" "${MASTER_NAME}-etcd" "${NODE_TAG}-all"
+    delete-firewall-rules "${MASTER_NAME}-https" "${MASTER_NAME}-etcd" "${NODE_TAG}-all" "${MASTER_NAME}-konnectivity-server"
     # Delete the master's reserved IP
     if gcloud compute addresses describe "${MASTER_NAME}-ip" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
       gcloud compute addresses delete \

cmd/kube-apiserver/app/options/options.go

@@ -48,6 +48,7 @@ type ServerRunOptions struct {
 	Authorization           *kubeoptions.BuiltInAuthorizationOptions
 	CloudProvider           *kubeoptions.CloudProviderOptions
 	APIEnablement           *genericoptions.APIEnablementOptions
+	EgressSelector          *genericoptions.EgressSelectorOptions
 
 	AllowPrivileged           bool
 	EnableLogsHandler         bool
@@ -87,6 +88,7 @@ func NewServerRunOptions() *ServerRunOptions {
 		Authorization:           kubeoptions.NewBuiltInAuthorizationOptions(),
 		CloudProvider:           kubeoptions.NewCloudProviderOptions(),
 		APIEnablement:           genericoptions.NewAPIEnablementOptions(),
+		EgressSelector:          genericoptions.NewEgressSelectorOptions(),
 
 		EnableLogsHandler:      true,
 		EventTTL:               1 * time.Hour,
@@ -134,6 +136,7 @@ func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
 	s.Authorization.AddFlags(fss.FlagSet("authorization"))
 	s.CloudProvider.AddFlags(fss.FlagSet("cloud provider"))
 	s.APIEnablement.AddFlags(fss.FlagSet("api enablement"))
+	s.EgressSelector.AddFlags(fss.FlagSet("egress selector"))
 	s.Admission.AddFlags(fss.FlagSet("admission"))
 
 	// Note: the weird ""+ in below lines seems to be the only way to get gofmt to

cmd/kube-apiserver/app/options/options_test.go

@@ -94,6 +94,7 @@ func TestAddFlags(t *testing.T) {
 		"--cloud-provider=azure",
 		"--cors-allowed-origins=10.10.10.100,10.10.10.200",
 		"--contention-profiling=true",
+		"--egress-selector-config-file=/var/run/kubernetes/egress-selector/connectivity.yaml",
 		"--enable-aggregator-routing=true",
 		"--enable-logs-handler=false",
 		"--endpoint-reconciler-type=" + string(reconcilers.LeaseEndpointReconcilerType),
@@ -293,6 +294,9 @@ func TestAddFlags(t *testing.T) {
 		APIEnablement: &apiserveroptions.APIEnablementOptions{
 			RuntimeConfig: cliflag.ConfigurationMap{},
 		},
+		EgressSelector: &apiserveroptions.EgressSelectorOptions{
+			ConfigFile: "/var/run/kubernetes/egress-selector/connectivity.yaml",
+		},
 		EnableLogsHandler:       false,
 		EnableAggregatorRouting: true,
 		ProxyClientKeyFile:      "/var/run/kubernetes/proxy.key",

cmd/kube-apiserver/app/server.go

@@ -362,6 +362,10 @@ func CreateKubeAPIServerConfig(
 		// Use the nodeTunneler's dialer to connect to the kubelet
 		config.ExtraConfig.KubeletClientConfig.Dial = nodeTunneler.Dial
 	}
+	if config.GenericConfig.EgressSelector != nil {
+		// Use the config.GenericConfig.EgressSelector lookup to find the dialer to connect to the kubelet
+		config.ExtraConfig.KubeletClientConfig.Lookup = config.GenericConfig.EgressSelector.Lookup
+	}
 
 	return
 }
@@ -402,6 +406,9 @@ func buildGenericConfig(
 	if lastErr = s.APIEnablement.ApplyTo(genericConfig, master.DefaultAPIResourceConfigSource(), legacyscheme.Scheme); lastErr != nil {
 		return
 	}
+	if lastErr = s.EgressSelector.ApplyTo(genericConfig); lastErr != nil {
+		return
+	}
 
 	genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme))
 	genericConfig.OpenAPIConfig.Info.Title = "Kubernetes"

pkg/kubelet/client/BUILD

@@ -16,6 +16,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
         "//staging/src/k8s.io/client-go/rest:go_default_library",
         "//staging/src/k8s.io/client-go/transport:go_default_library",
     ],

pkg/kubelet/client/kubelet_client.go

@@ -18,6 +18,7 @@ package client
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"strconv"
 	"time"
@@ -26,6 +27,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apiserver/pkg/server"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/transport"
 	nodeutil "k8s.io/kubernetes/pkg/util/node"
@@ -56,6 +58,9 @@ type KubeletClientConfig struct {
 
 	// Dial is a custom dialer used for the client
 	Dial utilnet.DialFunc
+
+	// Lookup will give us a dialer if the egress selector is configured for it
+	Lookup server.EgressSelectorLookup
 }
 
 // ConnectionInfo provides the information needed to connect to a kubelet
@@ -79,9 +84,20 @@ func MakeTransport(config *KubeletClientConfig) (http.RoundTripper, error) {
 	}
 
 	rt := http.DefaultTransport
-	if config.Dial != nil || tlsConfig != nil {
+	dialer := config.Dial
+	if dialer == nil && config.Lookup != nil {
+		// Assuming EgressSelector if SSHTunnel is not turned on.
+		// We will not get a dialer if egress selector is disabled.
+		networkContext := server.NetworkContext{EgressSelectionName: server.Cluster}
+		dialer, err = config.Lookup(networkContext)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get context dialer for 'cluster': got %v", err)
+		}
+	}
+	if dialer != nil || tlsConfig != nil {
+		// If SSH Tunnel is turned on
 		rt = utilnet.SetOldTransportDefaults(&http.Transport{
-			DialContext:     config.Dial,
+			DialContext:     dialer,
 			TLSClientConfig: tlsConfig,
 		})
 	}

staging/src/k8s.io/apiextensions-apiserver/go.sum

@@ -215,6 +215,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
 github.com/soheilhy/cmux v0.1.3 h1:09wy7WZk4AqO03yH85Ex1X+Uo3vDsil3Fa9AgF8Emss=
 github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.4 h1:S0tLZ3VOKl2Te0hpq8+ke0eSJPfCnNTPiDlsfwi1/NE=

staging/src/k8s.io/apiserver/go.sum

@@ -156,6 +156,7 @@ github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/soheilhy/cmux v0.1.3 h1:09wy7WZk4AqO03yH85Ex1X+Uo3vDsil3Fa9AgF8Emss=
 github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=

staging/src/k8s.io/apiserver/pkg/apis/apiserver/register.go

@@ -45,6 +45,7 @@ var (
 func addKnownTypes(scheme *runtime.Scheme) error {
 	scheme.AddKnownTypes(SchemeGroupVersion,
 		&AdmissionConfiguration{},
+		&EgressSelectorConfiguration{},
 	)
 	return nil
 }

staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go

@@ -48,3 +48,52 @@ type AdmissionPluginConfiguration struct {
 	// +optional
 	Configuration *runtime.Unknown
 }
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// EgressSelectorConfiguration provides versioned configuration for egress selector clients.
+type EgressSelectorConfiguration struct {
+	metav1.TypeMeta
+
+	// EgressSelections contains a list of egress selection client configurations
+	EgressSelections []EgressSelection
+}
+
+// EgressSelection provides the configuration for a single egress selection client.
+type EgressSelection struct {
+	// Name is the name of the egress selection.
+	// Currently supported values are "Master", "Etcd" and "Cluster"
+	Name string
+
+	// Connection is the exact information used to configure the egress selection
+	Connection Connection
+}
+
+// Connection provides the configuration for a single egress selection client.
+type Connection struct {
+	// Type is the type of connection used to connect from client to konnectivity server.
+	// Currently supported values are "http-connect" and "direct".
+	Type string
+
+	// httpConnect is the config needed to use http-connect to the konnectivity server.
+	// +optional
+	HTTPConnect *HTTPConnectConfig
+}
+
+type HTTPConnectConfig struct {
+	// URL is the location of the konnectivity server to connect to.
+	// As an example it might be "https://127.0.0.1:8131"
+	URL string
+
+	// CABundle is the file location of the CA to be used to determine trust with the konnectivity server.
+	// +optional
+	CABundle string
+
+	// ClientKey is the file location of the client key to be used in mtls handshakes with the konnectivity server.
+	// +optional
+	ClientKey string
+
+	// ClientCert is the file location of the client certificate to be used in mtls handshakes with the konnectivity server.
+	// +optional
+	ClientCert string
+}

staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/register.go

@@ -46,6 +46,7 @@ func init() {
 func addKnownTypes(scheme *runtime.Scheme) error {
 	scheme.AddKnownTypes(SchemeGroupVersion,
 		&AdmissionConfiguration{},
+		&EgressSelectorConfiguration{},
 	)
 	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
 	return nil

staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go

@@ -48,3 +48,63 @@ type AdmissionPluginConfiguration struct {
 	// +optional
 	Configuration *runtime.Unknown `json:"configuration"`
 }
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// EgressSelectorConfiguration provides versioned configuration for egress selector clients.
+type EgressSelectorConfiguration struct {
+	metav1.TypeMeta `json:",inline"`
+
+	// connectionServices contains a list of egress selection client configurations
+	EgressSelections []EgressSelection `json:"egressSelections"`
+}
+
+// EgressSelection provides the configuration for a single egress selection client.
+type EgressSelection struct {
+	// name is the name of the egress selection.
+	// Currently supported values are "Master", "Etcd" and "Cluster"
+	Name string `json:"name"`
+
+	// connection is the exact information used to configure the egress selection
+	Connection Connection `json:"connection"`
+}
+
+// Connection provides the configuration for a single egress selection client.
+type Connection struct {
+	// type is the type of connection used to connect from client to network/konnectivity server.
+	// Currently supported values are "http-connect" and "direct".
+	Type string `json:"type"`
+
+	// httpConnect is the config needed to use http-connect to the konnectivity server.
+	// Absence when the type is "http-connect" will cause an error
+	// Presence when the type is "direct" will also cause an error
+	// +optional
+	HTTPConnect *HTTPConnectConfig `json:"httpConnect,omitempty"`
+}
+
+type HTTPConnectConfig struct {
+	// url is the location of the proxy server to connect to.
+	// As an example it might be "https://127.0.0.1:8131"
+	URL string `json:"url"`
+
+	// caBundle is the file location of the CA to be used to determine trust with the konnectivity server.
+	// Must be absent/empty http-connect using the plain http
+	// Must be configured for http-connect using the https protocol
+	// Misconfiguration will cause an error
+	// +optional
+	CABundle string `json:"caBundle,omitempty"`
+
+	// clientKey is the file location of the client key to be used in mtls handshakes with the konnectivity server.
+	// Must be absent/empty http-connect using the plain http
+	// Must be configured for http-connect using the https protocol
+	// Misconfiguration will cause an error
+	// +optional
+	ClientKey string `json:"clientKey,omitempty"`
+
+	// clientCert is the file location of the client certificate to be used in mtls handshakes with the konnectivity server.
+	// Must be absent/empty http-connect using the plain http
+	// Must be configured for http-connect using the https protocol
+	// Misconfiguration will cause an error
+	// +optional
+	ClientCert string `json:"clientCert,omitempty"`
+}

staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go

@@ -55,6 +55,46 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*Connection)(nil), (*apiserver.Connection)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_Connection_To_apiserver_Connection(a.(*Connection), b.(*apiserver.Connection), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.Connection)(nil), (*Connection)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_Connection_To_v1alpha1_Connection(a.(*apiserver.Connection), b.(*Connection), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*EgressSelection)(nil), (*apiserver.EgressSelection)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_EgressSelection_To_apiserver_EgressSelection(a.(*EgressSelection), b.(*apiserver.EgressSelection), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.EgressSelection)(nil), (*EgressSelection)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_EgressSelection_To_v1alpha1_EgressSelection(a.(*apiserver.EgressSelection), b.(*EgressSelection), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*EgressSelectorConfiguration)(nil), (*apiserver.EgressSelectorConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration(a.(*EgressSelectorConfiguration), b.(*apiserver.EgressSelectorConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.EgressSelectorConfiguration)(nil), (*EgressSelectorConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration(a.(*apiserver.EgressSelectorConfiguration), b.(*EgressSelectorConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*HTTPConnectConfig)(nil), (*apiserver.HTTPConnectConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(a.(*HTTPConnectConfig), b.(*apiserver.HTTPConnectConfig), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.HTTPConnectConfig)(nil), (*HTTPConnectConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(a.(*apiserver.HTTPConnectConfig), b.(*HTTPConnectConfig), scope)
+	}); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -101,3 +141,97 @@ func autoConvert_apiserver_AdmissionPluginConfiguration_To_v1alpha1_AdmissionPlu
 func Convert_apiserver_AdmissionPluginConfiguration_To_v1alpha1_AdmissionPluginConfiguration(in *apiserver.AdmissionPluginConfiguration, out *AdmissionPluginConfiguration, s conversion.Scope) error {
 	return autoConvert_apiserver_AdmissionPluginConfiguration_To_v1alpha1_AdmissionPluginConfiguration(in, out, s)
 }
+
+func autoConvert_v1alpha1_Connection_To_apiserver_Connection(in *Connection, out *apiserver.Connection, s conversion.Scope) error {
+	out.Type = in.Type
+	out.HTTPConnect = (*apiserver.HTTPConnectConfig)(unsafe.Pointer(in.HTTPConnect))
+	return nil
+}
+
+// Convert_v1alpha1_Connection_To_apiserver_Connection is an autogenerated conversion function.
+func Convert_v1alpha1_Connection_To_apiserver_Connection(in *Connection, out *apiserver.Connection, s conversion.Scope) error {
+	return autoConvert_v1alpha1_Connection_To_apiserver_Connection(in, out, s)
+}
+
+func autoConvert_apiserver_Connection_To_v1alpha1_Connection(in *apiserver.Connection, out *Connection, s conversion.Scope) error {
+	out.Type = in.Type
+	out.HTTPConnect = (*HTTPConnectConfig)(unsafe.Pointer(in.HTTPConnect))
+	return nil
+}
+
+// Convert_apiserver_Connection_To_v1alpha1_Connection is an autogenerated conversion function.
+func Convert_apiserver_Connection_To_v1alpha1_Connection(in *apiserver.Connection, out *Connection, s conversion.Scope) error {
+	return autoConvert_apiserver_Connection_To_v1alpha1_Connection(in, out, s)
+}
+
+func autoConvert_v1alpha1_EgressSelection_To_apiserver_EgressSelection(in *EgressSelection, out *apiserver.EgressSelection, s conversion.Scope) error {
+	out.Name = in.Name
+	if err := Convert_v1alpha1_Connection_To_apiserver_Connection(&in.Connection, &out.Connection, s); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Convert_v1alpha1_EgressSelection_To_apiserver_EgressSelection is an autogenerated conversion function.
+func Convert_v1alpha1_EgressSelection_To_apiserver_EgressSelection(in *EgressSelection, out *apiserver.EgressSelection, s conversion.Scope) error {
+	return autoConvert_v1alpha1_EgressSelection_To_apiserver_EgressSelection(in, out, s)
+}
+
+func autoConvert_apiserver_EgressSelection_To_v1alpha1_EgressSelection(in *apiserver.EgressSelection, out *EgressSelection, s conversion.Scope) error {
+	out.Name = in.Name
+	if err := Convert_apiserver_Connection_To_v1alpha1_Connection(&in.Connection, &out.Connection, s); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Convert_apiserver_EgressSelection_To_v1alpha1_EgressSelection is an autogenerated conversion function.
+func Convert_apiserver_EgressSelection_To_v1alpha1_EgressSelection(in *apiserver.EgressSelection, out *EgressSelection, s conversion.Scope) error {
+	return autoConvert_apiserver_EgressSelection_To_v1alpha1_EgressSelection(in, out, s)
+}
+
+func autoConvert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration(in *EgressSelectorConfiguration, out *apiserver.EgressSelectorConfiguration, s conversion.Scope) error {
+	out.EgressSelections = *(*[]apiserver.EgressSelection)(unsafe.Pointer(&in.EgressSelections))
+	return nil
+}
+
+// Convert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration is an autogenerated conversion function.
+func Convert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration(in *EgressSelectorConfiguration, out *apiserver.EgressSelectorConfiguration, s conversion.Scope) error {
+	return autoConvert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration(in, out, s)
+}
+
+func autoConvert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration(in *apiserver.EgressSelectorConfiguration, out *EgressSelectorConfiguration, s conversion.Scope) error {
+	out.EgressSelections = *(*[]EgressSelection)(unsafe.Pointer(&in.EgressSelections))
+	return nil
+}
+
+// Convert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration is an autogenerated conversion function.
+func Convert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration(in *apiserver.EgressSelectorConfiguration, out *EgressSelectorConfiguration, s conversion.Scope) error {
+	return autoConvert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration(in, out, s)
+}
+
+func autoConvert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(in *HTTPConnectConfig, out *apiserver.HTTPConnectConfig, s conversion.Scope) error {
+	out.URL = in.URL
+	out.CABundle = in.CABundle
+	out.ClientKey = in.ClientKey
+	out.ClientCert = in.ClientCert
+	return nil
+}
+
+// Convert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig is an autogenerated conversion function.
+func Convert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(in *HTTPConnectConfig, out *apiserver.HTTPConnectConfig, s conversion.Scope) error {
+	return autoConvert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(in, out, s)
+}
+
+func autoConvert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(in *apiserver.HTTPConnectConfig, out *HTTPConnectConfig, s conversion.Scope) error {
+	out.URL = in.URL
+	out.CABundle = in.CABundle
+	out.ClientKey = in.ClientKey
+	out.ClientCert = in.ClientCert
+	return nil
+}
+
+// Convert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig is an autogenerated conversion function.
+func Convert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(in *apiserver.HTTPConnectConfig, out *HTTPConnectConfig, s conversion.Scope) error {
+	return autoConvert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(in, out, s)
+}

staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.deepcopy.go

@@ -76,3 +76,89 @@ func (in *AdmissionPluginConfiguration) DeepCopy() *AdmissionPluginConfiguration
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Connection) DeepCopyInto(out *Connection) {
+	*out = *in
+	if in.HTTPConnect != nil {
+		in, out := &in.HTTPConnect, &out.HTTPConnect
+		*out = new(HTTPConnectConfig)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Connection.
+func (in *Connection) DeepCopy() *Connection {
+	if in == nil {
+		return nil
+	}
+	out := new(Connection)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EgressSelection) DeepCopyInto(out *EgressSelection) {
+	*out = *in
+	in.Connection.DeepCopyInto(&out.Connection)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressSelection.
+func (in *EgressSelection) DeepCopy() *EgressSelection {
+	if in == nil {
+		return nil
+	}
+	out := new(EgressSelection)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EgressSelectorConfiguration) DeepCopyInto(out *EgressSelectorConfiguration) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	if in.EgressSelections != nil {
+		in, out := &in.EgressSelections, &out.EgressSelections
+		*out = make([]EgressSelection, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressSelectorConfiguration.
+func (in *EgressSelectorConfiguration) DeepCopy() *EgressSelectorConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(EgressSelectorConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *EgressSelectorConfiguration) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *HTTPConnectConfig) DeepCopyInto(out *HTTPConnectConfig) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPConnectConfig.
+func (in *HTTPConnectConfig) DeepCopy() *HTTPConnectConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(HTTPConnectConfig)
+	in.DeepCopyInto(out)
+	return out
+}

staging/src/k8s.io/apiserver/pkg/apis/apiserver/zz_generated.deepcopy.go

@@ -76,3 +76,89 @@ func (in *AdmissionPluginConfiguration) DeepCopy() *AdmissionPluginConfiguration
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Connection) DeepCopyInto(out *Connection) {
+	*out = *in
+	if in.HTTPConnect != nil {
+		in, out := &in.HTTPConnect, &out.HTTPConnect
+		*out = new(HTTPConnectConfig)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Connection.
+func (in *Connection) DeepCopy() *Connection {
+	if in == nil {
+		return nil
+	}
+	out := new(Connection)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EgressSelection) DeepCopyInto(out *EgressSelection) {
+	*out = *in
+	in.Connection.DeepCopyInto(&out.Connection)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressSelection.
+func (in *EgressSelection) DeepCopy() *EgressSelection {
+	if in == nil {
+		return nil
+	}
+	out := new(EgressSelection)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EgressSelectorConfiguration) DeepCopyInto(out *EgressSelectorConfiguration) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	if in.EgressSelections != nil {
+		in, out := &in.EgressSelections, &out.EgressSelections
+		*out = make([]EgressSelection, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressSelectorConfiguration.
+func (in *EgressSelectorConfiguration) DeepCopy() *EgressSelectorConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(EgressSelectorConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *EgressSelectorConfiguration) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *HTTPConnectConfig) DeepCopyInto(out *HTTPConnectConfig) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPConnectConfig.
+func (in *HTTPConnectConfig) DeepCopy() *HTTPConnectConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(HTTPConnectConfig)
+	in.DeepCopyInto(out)
+	return out
+}

staging/src/k8s.io/apiserver/pkg/registry/generic/rest/streamer.go

@@ -62,6 +62,7 @@ func (s *LocationStreamer) InputStream(ctx context.Context, apiVersion, acceptHe
 	if transport == nil {
 		transport = http.DefaultTransport
 	}
+
 	client := &http.Client{
 		Transport:     transport,
 		CheckRedirect: s.RedirectChecker,

staging/src/k8s.io/apiserver/pkg/server/BUILD

@@ -11,6 +11,7 @@ go_test(
     srcs = [
         "config_selfclient_test.go",
         "config_test.go",
+        "egress_selector_test.go",
         "genericapiserver_test.go",
         "healthz_test.go",
     ],
@@ -21,9 +22,11 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/version:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/apis/apiserver:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/apis/example:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/apis/example/v1:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
@@ -49,6 +52,7 @@ go_library(
         "config_selfclient.go",
         "deprecated_insecure_serving.go",
         "doc.go",
+        "egress_selector.go",
         "genericapiserver.go",
         "handler.go",
         "healthz.go",
@@ -70,6 +74,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
@@ -79,6 +84,7 @@ go_library(
         "//staging/src/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/apis/apiserver:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/apis/apiserver/install:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/audit:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/audit/policy:go_default_library",
@@ -133,6 +139,7 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/server/egressselector:all-srcs",
         "//staging/src/k8s.io/apiserver/pkg/server/filters:all-srcs",
         "//staging/src/k8s.io/apiserver/pkg/server/healthz:all-srcs",
         "//staging/src/k8s.io/apiserver/pkg/server/httplog:all-srcs",

staging/src/k8s.io/apiserver/pkg/server/config.go

@@ -29,7 +29,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	jsonpatch "github.com/evanphx/json-patch"
+	"github.com/evanphx/json-patch"
 	"github.com/go-openapi/spec"
 	"github.com/pborman/uuid"
 
@@ -94,6 +94,11 @@ type Config struct {
 	// This is required for proper functioning of the PostStartHooks on a GenericAPIServer
 	// TODO: move into SecureServing(WithLoopback) as soon as insecure serving is gone
 	LoopbackClientConfig *restclient.Config
+
+	// EgressSelector provides a lookup mechanism for dialing outbound connections.
+	// It does so based on a EgressSelectorConfiguration which was read at startup.
+	EgressSelector *EgressSelector
+
 	// RuleResolver is required to get the list of rules that apply to a given user
 	// in a given namespace
 	RuleResolver authorizer.RuleResolver

staging/src/k8s.io/apiserver/pkg/server/egress_selector.go

@@ -0,0 +1,192 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apiserver/pkg/apis/apiserver"
+	"k8s.io/klog"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+var directDialer utilnet.DialFunc = http.DefaultTransport.(*http.Transport).DialContext
+
+type EgressSelector struct {
+	egressToDialer map[EgressType]utilnet.DialFunc
+}
+
+// EgressType is an indicator of which egress selection should be used for sending traffic.
+// See https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190226-network-proxy.md#network-context
+type EgressType int
+
+const (
+	// Master is the EgressType for traffic intended to go to the control plane.
+	Master EgressType = iota
+	// Etcd is the EgressType for traffic intended to go to Kubernetes persistence store.
+	Etcd
+	// Cluster is the EgressType for traffic intended to go to the system being managed by Kubernetes.
+	Cluster
+)
+
+// NetworkContext is the struct used by Kubernetes API Server to indicate where it intends traffic to be sent.
+type NetworkContext struct {
+	// EgressSelectionName is the unique name of the
+	// EgressSelectorConfiguration which determines
+	// the network we route the traffic to.
+	EgressSelectionName EgressType
+}
+
+// EgressSelectorLookup is the interface to get the dialer function for the network context.
+type EgressSelectorLookup func(networkContext NetworkContext) (utilnet.DialFunc, error)
+
+func (s EgressType) String() string {
+	switch s {
+	case Master:
+		return "master"
+	case Etcd:
+		return "etcd"
+	case Cluster:
+		return "cluster"
+	default:
+		return "invalid"
+	}
+}
+
+func lookupServiceName(name string) (EgressType, error) {
+	switch strings.ToLower(name) {
+	case "master":
+		return Master, nil
+	case "etcd":
+		return Etcd, nil
+	case "cluster":
+		return Cluster, nil
+	}
+	return -1, fmt.Errorf("unrecognized service name %s", name)
+}
+
+func createConnectDialer(connectConfig *apiserver.HTTPConnectConfig) (utilnet.DialFunc, error) {
+	clientCert := connectConfig.ClientCert
+	clientKey := connectConfig.ClientKey
+	caCert := connectConfig.CABundle
+	proxyURL, err := url.Parse(connectConfig.URL)
+	if err != nil {
+		return nil, fmt.Errorf("invalid proxy server url %q: %v", connectConfig.URL, err)
+	}
+	proxyAddress := proxyURL.Host
+
+	clientCerts, err := tls.LoadX509KeyPair(clientCert, clientKey)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read key pair %s & %s, got %v", clientCert, clientKey, err)
+	}
+	certPool := x509.NewCertPool()
+	certBytes, err := ioutil.ReadFile(caCert)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read cert file %s, got %v", caCert, err)
+	}
+	ok := certPool.AppendCertsFromPEM(certBytes)
+	if !ok {
+		return nil, fmt.Errorf("failed to append CA cert to the cert pool")
+	}
+	contextDialer := func(ctx context.Context, network, addr string) (net.Conn, error) {
+		klog.V(4).Infof("Sending request to %q.", addr)
+		proxyConn, err := tls.Dial("tcp", proxyAddress,
+			&tls.Config{
+				Certificates: []tls.Certificate{clientCerts},
+				RootCAs:      certPool,
+			},
+		)
+		if err != nil {
+			return nil, fmt.Errorf("dialing proxy %q failed: %v", proxyAddress, err)
+		}
+		fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", addr, "127.0.0.1")
+		br := bufio.NewReader(proxyConn)
+		res, err := http.ReadResponse(br, nil)
+		if err != nil {
+			proxyConn.Close()
+			return nil, fmt.Errorf("reading HTTP response from CONNECT to %s via proxy %s failed: %v",
+				addr, proxyAddress, err)
+		}
+		if res.StatusCode != 200 {
+			proxyConn.Close()
+			return nil, fmt.Errorf("proxy error from %s while dialing %s, code %d: %v",
+				proxyAddress, addr, res.StatusCode, res.Status)
+		}
+
+		// It's safe to discard the bufio.Reader here and return the
+		// original TCP conn directly because we only use this for
+		// TLS, and in TLS the client speaks first, so we know there's
+		// no unbuffered data. But we can double-check.
+		if br.Buffered() > 0 {
+			proxyConn.Close()
+			return nil, fmt.Errorf("unexpected %d bytes of buffered data from CONNECT proxy %q",
+				br.Buffered(), proxyAddress)
+		}
+		klog.V(4).Infof("About to proxy request to %s over %s.", addr, proxyAddress)
+		return proxyConn, nil
+	}
+	return contextDialer, nil
+}
+
+// NewEgressSelector configures lookup mechanism for Lookup.
+// It does so based on a EgressSelectorConfiguration which was read at startup.
+func NewEgressSelector(config *apiserver.EgressSelectorConfiguration) (*EgressSelector, error) {
+	if config == nil || config.EgressSelections == nil {
+		// No Connection Services configured, leaving the serviceMap empty, will return default dialer.
+		return nil, nil
+	}
+	cs := &EgressSelector{
+		egressToDialer: make(map[EgressType]utilnet.DialFunc),
+	}
+	for _, service := range config.EgressSelections {
+		name, err := lookupServiceName(service.Name)
+		if err != nil {
+			return nil, err
+		}
+		switch service.Connection.Type {
+		case "http-connect":
+			contextDialer, err := createConnectDialer(service.Connection.HTTPConnect)
+			if err != nil {
+				return nil, fmt.Errorf("failed to create http-connect dialer: %v", err)
+			}
+			cs.egressToDialer[name] = contextDialer
+		case "direct":
+			cs.egressToDialer[name] = directDialer
+		default:
+			return nil, fmt.Errorf("unrecognized service connection type %q", service.Connection.Type)
+		}
+	}
+	return cs, nil
+}
+
+// Lookup gets the dialer function for the network context.
+// This is configured for the Kubernetes API Server at startup.
+func (cs *EgressSelector) Lookup(networkContext NetworkContext) (utilnet.DialFunc, error) {
+	if cs.egressToDialer == nil {
+		// The round trip wrapper will over-ride the dialContext method appropriately
+		return nil, nil
+	}
+	return cs.egressToDialer[networkContext.EgressSelectionName], nil
+}

staging/src/k8s.io/apiserver/pkg/server/egress_selector_test.go

@@ -0,0 +1,183 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"net"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apiserver/pkg/apis/apiserver"
+)
+
+type fakeEgressSelection struct {
+	directDialerCalled bool
+}
+
+func TestEgressSelector(t *testing.T) {
+	testcases := []struct {
+		name     string
+		input    *apiserver.EgressSelectorConfiguration
+		services []struct {
+			egressType     EgressType
+			validateDialer func(dialer utilnet.DialFunc, s *fakeEgressSelection) (bool, error)
+			lookupError    *string
+			dialerError    *string
+		}
+		expectedError *string
+	}{
+		{
+			name: "direct",
+			input: &apiserver.EgressSelectorConfiguration{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "",
+					APIVersion: "",
+				},
+				EgressSelections: []apiserver.EgressSelection{
+					{
+						Name: "cluster",
+						Connection: apiserver.Connection{
+							Type: "direct",
+							HTTPConnect: &apiserver.HTTPConnectConfig{
+								URL:        "",
+								CABundle:   "",
+								ClientKey:  "",
+								ClientCert: "",
+							},
+						},
+					},
+					{
+						Name: "master",
+						Connection: apiserver.Connection{
+							Type: "direct",
+							HTTPConnect: &apiserver.HTTPConnectConfig{
+								URL:        "",
+								CABundle:   "",
+								ClientKey:  "",
+								ClientCert: "",
+							},
+						},
+					},
+					{
+						Name: "etcd",
+						Connection: apiserver.Connection{
+							Type: "direct",
+							HTTPConnect: &apiserver.HTTPConnectConfig{
+								URL:        "",
+								CABundle:   "",
+								ClientKey:  "",
+								ClientCert: "",
+							},
+						},
+					},
+				},
+			},
+			services: []struct {
+				egressType     EgressType
+				validateDialer func(dialer utilnet.DialFunc, s *fakeEgressSelection) (bool, error)
+				lookupError    *string
+				dialerError    *string
+			}{
+				{
+					Cluster,
+					validateDirectDialer,
+					nil,
+					nil,
+				},
+				{
+					Master,
+					validateDirectDialer,
+					nil,
+					nil,
+				},
+				{
+					Etcd,
+					validateDirectDialer,
+					nil,
+					nil,
+				},
+			},
+			expectedError: nil,
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Setup the various pieces such as the fake dialer prior to initializing the egress selector.
+			// Go doesn't allow function pointer comparison, nor does its reflect package
+			// So overriding the default dialer to detect if it is returned.
+			fake := &fakeEgressSelection{}
+			directDialer = fake.fakeDirectDialer
+			cs, err := NewEgressSelector(tc.input)
+			if err == nil && tc.expectedError != nil {
+				t.Errorf("calling NewEgressSelector expected error: %s, did not get it", *tc.expectedError)
+			}
+			if err != nil && tc.expectedError == nil {
+				t.Errorf("unexpected error calling NewEgressSelector got: %#v", err)
+			}
+			if err != nil && tc.expectedError != nil && err.Error() != *tc.expectedError {
+				t.Errorf("calling NewEgressSelector expected error: %s, got %#v", *tc.expectedError, err)
+			}
+
+			for _, service := range tc.services {
+				networkContext := NetworkContext{EgressSelectionName: service.egressType}
+				dialer, lookupErr := cs.Lookup(networkContext)
+				if lookupErr == nil && service.lookupError != nil {
+					t.Errorf("calling Lookup expected error: %s, did not get it", *service.lookupError)
+				}
+				if lookupErr != nil && service.lookupError == nil {
+					t.Errorf("unexpected error calling Lookup got: %#v", lookupErr)
+				}
+				if lookupErr != nil && service.lookupError != nil && lookupErr.Error() != *service.lookupError {
+					t.Errorf("calling Lookup expected error: %s, got %#v", *service.lookupError, lookupErr)
+				}
+				fake.directDialerCalled = false
+				ok, dialerErr := service.validateDialer(dialer, fake)
+				if dialerErr == nil && service.dialerError != nil {
+					t.Errorf("calling Lookup expected error: %s, did not get it", *service.dialerError)
+				}
+				if dialerErr != nil && service.dialerError == nil {
+					t.Errorf("unexpected error calling Lookup got: %#v", dialerErr)
+				}
+				if dialerErr != nil && service.dialerError != nil && dialerErr.Error() != *service.dialerError {
+					t.Errorf("calling Lookup expected error: %s, got %#v", *service.dialerError, dialerErr)
+				}
+				if !ok {
+					t.Errorf("Could not validate dialer for service %q", service.egressType)
+				}
+			}
+		})
+	}
+}
+
+func (s *fakeEgressSelection) fakeDirectDialer(ctx context.Context, network, address string) (net.Conn, error) {
+	s.directDialerCalled = true
+	return nil, nil
+}
+
+func validateDirectDialer(dialer utilnet.DialFunc, s *fakeEgressSelection) (bool, error) {
+	conn, err := dialer(context.Background(), "tcp", "127.0.0.1:8080")
+	if err != nil {
+		return false, err
+	}
+	if conn != nil {
+		return false, nil
+	}
+	return s.directDialerCalled, nil
+}

staging/src/k8s.io/apiserver/pkg/server/egressselector/BUILD

@@ -0,0 +1,42 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["config.go"],
+    importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/egressselector",
+    importpath = "k8s.io/apiserver/pkg/server/egressselector",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/apis/apiserver:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/apis/apiserver/install:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1:go_default_library",
+        "//vendor/k8s.io/utils/path:go_default_library",
+        "//vendor/sigs.k8s.io/yaml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["config_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/apis/apiserver:go_default_library",
+    ],
+)

staging/src/k8s.io/apiserver/pkg/server/egressselector/config.go

@@ -0,0 +1,179 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package egressselector
+
+import (
+	"fmt"
+	"io/ioutil"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/apiserver/pkg/apis/apiserver"
+	"k8s.io/apiserver/pkg/apis/apiserver/install"
+	"k8s.io/apiserver/pkg/apis/apiserver/v1alpha1"
+	"k8s.io/utils/path"
+	"sigs.k8s.io/yaml"
+)
+
+var cfgScheme = runtime.NewScheme()
+
+func init() {
+	install.Install(cfgScheme)
+}
+
+// ReadEgressSelectorConfiguration reads the egress selector configuration at the specified path.
+// It returns the loaded egress selector configuration if the input file aligns with the required syntax.
+// If it does not align with the provided syntax, it returns a default configuration which should function as a no-op.
+// It does this by returning a nil configuration, which preserves backward compatibility.
+// This works because prior to this there was no egress selector configuration.
+// It returns an error if the file did not exist.
+func ReadEgressSelectorConfiguration(configFilePath string) (*apiserver.EgressSelectorConfiguration, error) {
+	if configFilePath == "" {
+		return nil, nil
+	}
+	// a file was provided, so we just read it.
+	data, err := ioutil.ReadFile(configFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read egress selector configuration from %q [%v]", configFilePath, err)
+	}
+	var decodedConfig v1alpha1.EgressSelectorConfiguration
+	err = yaml.Unmarshal(data, &decodedConfig)
+	if err != nil {
+		// we got an error where the decode wasn't related to a missing type
+		return nil, err
+	}
+	if decodedConfig.Kind != "EgressSelectorConfiguration" {
+		return nil, fmt.Errorf("invalid service configuration object %q", decodedConfig.Kind)
+	}
+	config, err := cfgScheme.ConvertToVersion(&decodedConfig, apiserver.SchemeGroupVersion)
+	if err != nil {
+		// we got an error where the decode wasn't related to a missing type
+		return nil, err
+	}
+	if internalConfig, ok := config.(*apiserver.EgressSelectorConfiguration); ok {
+		return internalConfig, nil
+	}
+	return nil, fmt.Errorf("unable to convert %T to *apiserver.EgressSelectorConfiguration", config)
+}
+
+// ValidateEgressSelectorConfiguration checks the apiserver.EgressSelectorConfiguration for
+// common configuration errors. It will return error for problems such as configuring mtls/cert
+// settings for protocol which do not support security. It will also try to catch errors such as
+// incorrect file paths. It will return nil if it does not find anything wrong.
+func ValidateEgressSelectorConfiguration(config *apiserver.EgressSelectorConfiguration) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if config == nil {
+		return allErrs // Treating a nil configuration as valid
+	}
+	for _, service := range config.EgressSelections {
+		base := field.NewPath("service", "connection")
+		switch service.Connection.Type {
+		case "direct":
+			allErrs = append(allErrs, validateDirectConnection(service.Connection, base)...)
+		case "http-connect":
+			allErrs = append(allErrs, validateHTTPConnection(service.Connection, base)...)
+		default:
+			allErrs = append(allErrs, field.NotSupported(
+				base.Child("type"),
+				service.Connection.Type,
+				[]string{"direct", "http-connect"}))
+		}
+	}
+
+	return allErrs
+}
+
+func validateDirectConnection(connection apiserver.Connection, fldPath *field.Path) field.ErrorList {
+	if connection.HTTPConnect != nil {
+		return field.ErrorList{field.Invalid(
+			fldPath.Child("httpConnect"),
+			"direct",
+			"httpConnect config should be absent for direct connect"),
+		}
+	}
+	return nil
+}
+
+func validateHTTPConnection(connection apiserver.Connection, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if connection.HTTPConnect == nil {
+		allErrs = append(allErrs, field.Invalid(
+			fldPath.Child("httpConnect"),
+			"nil",
+			"httpConnect config should be present for http-connect"))
+	} else if strings.HasPrefix(connection.HTTPConnect.URL, "https://") {
+		if connection.HTTPConnect.CABundle == "" {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "caBundle"),
+				"nil",
+				"http-connect via https requires caBundle"))
+		} else if exists, err := path.Exists(path.CheckFollowSymlink, connection.HTTPConnect.CABundle); exists == false || err != nil {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "caBundle"),
+				connection.HTTPConnect.CABundle,
+				"http-connect ca bundle does not exist"))
+		}
+		if connection.HTTPConnect.ClientCert == "" {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "clientCert"),
+				"nil",
+				"http-connect via https requires clientCert"))
+		} else if exists, err := path.Exists(path.CheckFollowSymlink, connection.HTTPConnect.ClientCert); exists == false || err != nil {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "clientCert"),
+				connection.HTTPConnect.ClientCert,
+				"http-connect client cert does not exist"))
+		}
+		if connection.HTTPConnect.ClientKey == "" {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "clientKey"),
+				"nil",
+				"http-connect via https requires clientKey"))
+		} else if exists, err := path.Exists(path.CheckFollowSymlink, connection.HTTPConnect.ClientKey); exists == false || err != nil {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "clientKey"),
+				connection.HTTPConnect.ClientKey,
+				"http-connect client key does not exist"))
+		}
+	} else if strings.HasPrefix(connection.HTTPConnect.URL, "http://") {
+		if connection.HTTPConnect.CABundle != "" {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "caBundle"),
+				connection.HTTPConnect.CABundle,
+				"http-connect via http does not support caBundle"))
+		}
+		if connection.HTTPConnect.ClientCert != "" {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "clientCert"),
+				connection.HTTPConnect.ClientCert,
+				"http-connect via http does not support clientCert"))
+		}
+		if connection.HTTPConnect.ClientKey != "" {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "clientKey"),
+				connection.HTTPConnect.ClientKey,
+				"http-connect via http does not support clientKey"))
+		}
+	} else {
+		allErrs = append(allErrs, field.Invalid(
+			fldPath.Child("httpConnect", "url"),
+			connection.HTTPConnect.URL,
+			"supported connection protocols are http:// and https://"))
+	}
+	return allErrs
+}

staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go

@@ -0,0 +1,214 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package egressselector
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"reflect"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apiserver/pkg/apis/apiserver"
+)
+
+func strptr(s string) *string {
+	return &s
+}
+
+func TestReadEgressSelectorConfiguration(t *testing.T) {
+	testcases := []struct {
+		name           string
+		contents       string
+		createFile     bool
+		expectedResult *apiserver.EgressSelectorConfiguration
+		expectedError  *string
+	}{
+		{
+			name:           "empty",
+			createFile:     true,
+			contents:       ``,
+			expectedResult: nil,
+			expectedError:  strptr("invalid service configuration object \"\""),
+		},
+		{
+			name:           "absent",
+			createFile:     false,
+			contents:       ``,
+			expectedResult: nil,
+			expectedError:  strptr("unable to read egress selector configuration from \"test-egress-selector-config-absent\" [open test-egress-selector-config-absent: no such file or directory]"),
+		},
+		{
+			name:       "v1alpha1",
+			createFile: true,
+			contents: `
+apiVersion: apiserver.k8s.io/v1alpha1
+kind: EgressSelectorConfiguration
+egressSelections:
+- name: "cluster"
+  connection:
+    type: "http-connect"
+    httpConnect:
+      url: "https://127.0.0.1:8131"
+      caBundle: "/etc/srv/kubernetes/pki/konnectivity-server/ca.crt"
+      clientKey: "/etc/srv/kubernetes/pki/konnectivity-server/client.key"
+      clientCert: "/etc/srv/kubernetes/pki/konnectivity-server/client.crt"
+- name: "master"
+  connection:
+    type: "http-connect"
+    httpConnect:
+      url: "https://127.0.0.1:8132"
+      caBundle: "/etc/srv/kubernetes/pki/konnectivity-server-master/ca.crt"
+      clientKey: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.key"
+      clientCert: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.crt"
+- name: "etcd"
+  connection:
+    type: "direct"
+`,
+			expectedResult: &apiserver.EgressSelectorConfiguration{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "",
+					APIVersion: "",
+				},
+				EgressSelections: []apiserver.EgressSelection{
+					{
+						Name: "cluster",
+						Connection: apiserver.Connection{
+							Type: "http-connect",
+							HTTPConnect: &apiserver.HTTPConnectConfig{
+								URL:        "https://127.0.0.1:8131",
+								CABundle:   "/etc/srv/kubernetes/pki/konnectivity-server/ca.crt",
+								ClientKey:  "/etc/srv/kubernetes/pki/konnectivity-server/client.key",
+								ClientCert: "/etc/srv/kubernetes/pki/konnectivity-server/client.crt",
+							},
+						},
+					},
+					{
+						Name: "master",
+						Connection: apiserver.Connection{
+							Type: "http-connect",
+							HTTPConnect: &apiserver.HTTPConnectConfig{
+								URL:        "https://127.0.0.1:8132",
+								CABundle:   "/etc/srv/kubernetes/pki/konnectivity-server-master/ca.crt",
+								ClientKey:  "/etc/srv/kubernetes/pki/konnectivity-server-master/client.key",
+								ClientCert: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.crt",
+							},
+						},
+					},
+					{
+						Name: "etcd",
+						Connection: apiserver.Connection{
+							Type: "direct",
+						},
+					},
+				},
+			},
+			expectedError: nil,
+		},
+		{
+			name:       "wrong_type",
+			createFile: true,
+			contents: `
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    k8s-app: konnectivity-agent
+  namespace: kube-system
+  name: proxy-agent
+spec:
+  selector:
+    matchLabels:
+      k8s-app: konnectivity-agent
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        k8s-app: proxy-agent
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      priorityClassName: system-cluster-critical
+      # Necessary to reboot node
+      hostPID: true
+      volumes:
+        - name: pki
+          hostPath:
+            path: /etc/srv/kubernetes/pki/konnectivity-agent
+      containers:
+        - image: gcr.io/google-containers/proxy-agent:v0.0.3
+          name: proxy-agent
+          command: ["/proxy-agent"]
+          args: ["--caCert=/etc/srv/kubernetes/pki/proxy-agent/ca.crt", "--agentCert=/etc/srv/kubernetes/pki/proxy-agent/client.crt", "--agentKey=/etc/srv/kubernetes/pki/proxy-agent/client.key", "--proxyServerHost=127.0.0.1", "--proxyServerPort=8132"]
+          securityContext:
+            capabilities:
+              add: ["SYS_BOOT"]
+          env:
+            - name: wrong-type
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: kube-system
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          resources:
+            limits:
+              cpu: 50m
+              memory: 30Mi
+          volumeMounts:
+            - name: pki
+              mountPath: /etc/srv/kubernetes/pki/konnectivity-agent
+`,
+			expectedResult: nil,
+			expectedError:  strptr("invalid service configuration object \"DaemonSet\""),
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			proxyConfig := fmt.Sprintf("test-egress-selector-config-%s", tc.name)
+			if tc.createFile {
+				f, err := ioutil.TempFile("", proxyConfig)
+				if err != nil {
+					t.Fatal(err)
+				}
+				defer os.Remove(f.Name())
+				if err := ioutil.WriteFile(f.Name(), []byte(tc.contents), os.FileMode(0755)); err != nil {
+					t.Fatal(err)
+				}
+				proxyConfig = f.Name()
+			}
+			config, err := ReadEgressSelectorConfiguration(proxyConfig)
+			if err == nil && tc.expectedError != nil {
+				t.Errorf("calling ReadEgressSelectorConfiguration expected error: %s, did not get it", *tc.expectedError)
+			}
+			if err != nil && tc.expectedError == nil {
+				t.Errorf("unexpected error calling ReadEgressSelectorConfiguration got: %#v", err)
+			}
+			if err != nil && tc.expectedError != nil && err.Error() != *tc.expectedError {
+				t.Errorf("calling ReadEgressSelectorConfiguration expected error: %s, got %#v", *tc.expectedError, err)
+			}
+			if !reflect.DeepEqual(config, tc.expectedResult) {
+				t.Errorf("problem with configuration returned from ReadEgressSelectorConfiguration expected: %#v, got: %#v", tc.expectedResult, config)
+			}
+		})
+	}
+}

staging/src/k8s.io/apiserver/pkg/server/options/BUILD

@@ -11,6 +11,7 @@ go_library(
         "coreapi.go",
         "deprecated_insecure_serving.go",
         "doc.go",
+        "egress_selector.go",
         "etcd.go",
         "events.go",
         "feature.go",
@@ -56,6 +57,7 @@ go_library(
         "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/egressselector:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/resourceconfig:go_default_library",
@@ -83,6 +85,7 @@ go_library(
         "//vendor/gopkg.in/natefinch/lumberjack.v2:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
         "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library",
+        "//vendor/k8s.io/utils/path:go_default_library",
     ],
 )
 

staging/src/k8s.io/apiserver/pkg/server/options/egress_selector.go

@@ -0,0 +1,92 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"github.com/spf13/pflag"
+	"k8s.io/utils/path"
+
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/egressselector"
+)
+
+// EgressSelectorOptions holds the api server egress selector options.
+// See https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190226-network-proxy.md
+type EgressSelectorOptions struct {
+	// ConfigFile is the file path with api-server egress selector configuration.
+	ConfigFile string
+}
+
+// NewEgressSelectorOptions creates a new instance of EgressSelectorOptions
+//
+// The option is to point to a configuration file for egress/konnectivity.
+// This determines which types of requests use egress/konnectivity and how they use it.
+// If empty the API Server will attempt to connect directly using the network.
+func NewEgressSelectorOptions() *EgressSelectorOptions {
+	return &EgressSelectorOptions{}
+}
+
+// AddFlags adds flags related to admission for a specific APIServer to the specified FlagSet
+func (o *EgressSelectorOptions) AddFlags(fs *pflag.FlagSet) {
+	if o == nil {
+		return
+	}
+
+	fs.StringVar(&o.ConfigFile, "egress-selector-config-file", o.ConfigFile,
+		"File with apiserver egress selector configuration.")
+}
+
+// ApplyTo adds the egress selector settings to the server configuration.
+// In case egress selector settings were not provided by a cluster-admin
+// they will be prepared from the recommended/default/no-op values.
+func (o *EgressSelectorOptions) ApplyTo(c *server.Config) error {
+	if o == nil {
+		return nil
+	}
+
+	npConfig, err := egressselector.ReadEgressSelectorConfiguration(o.ConfigFile)
+	if err != nil {
+		return fmt.Errorf("failed to read egress selector config: %v", err)
+	}
+	errs := egressselector.ValidateEgressSelectorConfiguration(npConfig)
+	if len(errs) > 0 {
+		return fmt.Errorf("failed to validate egress selector configuration: %v", errs.ToAggregate())
+	}
+
+	cs, err := server.NewEgressSelector(npConfig)
+	if err != nil {
+		return fmt.Errorf("failed to setup egress selector with config %#v: %v", npConfig, err)
+	}
+	c.EgressSelector = cs
+	return nil
+}
+
+// Validate verifies flags passed to EgressSelectorOptions.
+func (o *EgressSelectorOptions) Validate() []error {
+	if o == nil || o.ConfigFile == "" {
+		return nil
+	}
+
+	errs := []error{}
+
+	if exists, err := path.Exists(path.CheckFollowSymlink, o.ConfigFile); exists == false || err != nil {
+		errs = append(errs, fmt.Errorf("egress-selector-config-file %s does not exist", o.ConfigFile))
+	}
+
+	return errs
+}

staging/src/k8s.io/apiserver/pkg/server/options/recommended.go

@@ -44,6 +44,8 @@ type RecommendedOptions struct {
 	// ProcessInfo is used to identify events created by the server.
 	ProcessInfo *ProcessInfo
 	Webhook     *WebhookOptions
+	// API Server Egress Selector is used to control outbound traffic from the API Server
+	EgressSelector *EgressSelectorOptions
 }
 
 func NewRecommendedOptions(prefix string, codec runtime.Codec, processInfo *ProcessInfo) *RecommendedOptions {
@@ -67,6 +69,7 @@ func NewRecommendedOptions(prefix string, codec runtime.Codec, processInfo *Proc
 		Admission:                  NewAdmissionOptions(),
 		ProcessInfo:                processInfo,
 		Webhook:                    NewWebhookOptions(),
+		EgressSelector:             NewEgressSelectorOptions(),
 	}
 }
 
@@ -79,6 +82,7 @@ func (o *RecommendedOptions) AddFlags(fs *pflag.FlagSet) {
 	o.Features.AddFlags(fs)
 	o.CoreAPI.AddFlags(fs)
 	o.Admission.AddFlags(fs)
+	o.EgressSelector.AddFlags(fs)
 }
 
 // ApplyTo adds RecommendedOptions to the server configuration.
@@ -110,6 +114,9 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 	} else if err := o.Admission.ApplyTo(&config.Config, config.SharedInformerFactory, config.ClientConfig, initializers...); err != nil {
 		return err
 	}
+	if err := o.EgressSelector.ApplyTo(&config.Config); err != nil {
+		return err
+	}
 
 	return nil
 }
@@ -124,6 +131,7 @@ func (o *RecommendedOptions) Validate() []error {
 	errors = append(errors, o.Features.Validate()...)
 	errors = append(errors, o.CoreAPI.Validate()...)
 	errors = append(errors, o.Admission.Validate()...)
+	errors = append(errors, o.EgressSelector.Validate()...)
 
 	return errors
 }

staging/src/k8s.io/kube-aggregator/go.sum

@@ -171,6 +171,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
 github.com/soheilhy/cmux v0.1.3 h1:09wy7WZk4AqO03yH85Ex1X+Uo3vDsil3Fa9AgF8Emss=
 github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.4 h1:S0tLZ3VOKl2Te0hpq8+ke0eSJPfCnNTPiDlsfwi1/NE=

staging/src/k8s.io/sample-apiserver/go.sum

@@ -168,6 +168,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
 github.com/soheilhy/cmux v0.1.3 h1:09wy7WZk4AqO03yH85Ex1X+Uo3vDsil3Fa9AgF8Emss=
 github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.4 h1:S0tLZ3VOKl2Te0hpq8+ke0eSJPfCnNTPiDlsfwi1/NE=

vendor/modules.txt

@@ -1257,6 +1257,7 @@ k8s.io/apiserver/pkg/registry/generic/testing
 k8s.io/apiserver/pkg/registry/rest
 k8s.io/apiserver/pkg/registry/rest/resttest
 k8s.io/apiserver/pkg/server
+k8s.io/apiserver/pkg/server/egressselector
 k8s.io/apiserver/pkg/server/filters
 k8s.io/apiserver/pkg/server/healthz
 k8s.io/apiserver/pkg/server/httplog