diff --git a/cluster/gce/addons/konnectivity-agent/daemonset.yaml b/cluster/gce/addons/konnectivity-agent/daemonset.yaml index ee770cc8683..5ea16d44bef 100644 --- a/cluster/gce/addons/konnectivity-agent/daemonset.yaml +++ b/cluster/gce/addons/konnectivity-agent/daemonset.yaml @@ -28,14 +28,12 @@ spec: hostPath: path: /etc/srv/kubernetes/pki/konnectivity-agent containers: - - image: gcr.io/google-containers/proxy-agent:v0.0.3 + - image: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent:v0.0.4 name: konnectivity-agent command: ["/proxy-agent"] args: [ "--logtostderr=true", - "--ca-cert=/etc/srv/kubernetes/pki/konnectivity-agent/ca.crt", - "--agent-cert=/etc/srv/kubernetes/pki/konnectivity-agent/client.crt", - "--agent-key=/etc/srv/kubernetes/pki/konnectivity-agent/client.key", + "--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", "--proxy-server-host=__APISERVER_IP__", "--proxy-server-port=8132" ] @@ -59,6 +57,3 @@ spec: path: /healthz initialDelaySeconds: 15 timeoutSeconds: 15 - volumeMounts: - - name: pki - mountPath: /etc/srv/kubernetes/pki/konnectivity-agent diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 481eb6b5b9d..d7af6d34f25 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -807,18 +807,16 @@ kind: EgressSelectorConfiguration egressSelections: - name: cluster connection: - type: http-connect - httpConnect: - url: https://127.0.0.1:8131 - caBundle: /etc/srv/kubernetes/pki/konnectivity-server/ca.crt - clientKey: /etc/srv/kubernetes/pki/konnectivity-server/client.key - clientCert: /etc/srv/kubernetes/pki/konnectivity-server/client.crt + proxyProtocol: HTTPConnect + transport: + uds: + udsName: /etc/srv/kubernetes/konnectivity/konnectivity-server.socket - name: master connection: - type: direct + proxyProtocol: Direct - name: etcd connection: - type: direct + proxyProtocol: Direct EOF fi @@ -1645,9 +1643,8 @@ function start-etcd-servers { # Replaces the variables in the konnectivity-server manifest file with the real values, and then # copy the file to the manifest dir -# $1: value for variable "server_port" -# $2: value for variable "agent_port" -# $3: value for bariable "admin_port" +# $1: value for variable "agent_port" +# $2: value for bariable "admin_port" function prepare-konnectivity-server-manifest { local -r temp_file="/tmp/konnectivity-server.yaml" params=() @@ -1655,24 +1652,20 @@ function prepare-konnectivity-server-manifest { params+=("--log-file=/var/log/konnectivity-server.log") params+=("--logtostderr=false") params+=("--log-file-max-size=0") - params+=("--server-ca-cert=${KONNECTIVITY_SERVER_CA_CERT_PATH}") - params+=("--server-cert=${KONNECTIVITY_SERVER_CERT_PATH}") - params+=("--server-key=${KONNECTIVITY_SERVER_KEY_PATH}") - params+=("--cluster-ca-cert=${KONNECTIVITY_AGENT_CA_CERT_PATH}") - params+=("--cluster-cert=${KONNECTIVITY_AGENT_CERT_PATH}") - params+=("--cluster-key=${KONNECTIVITY_AGENT_KEY_PATH}") + params+=("--uds-name=/etc/srv/kubernetes/konnectivity/konnectivity-server.socket") + params+=("--cluster-cert=/etc/srv/kubernetes/pki/apiserver.crt") + params+=("--cluster-key=/etc/srv/kubernetes/pki/apiserver.key") params+=("--mode=http-connect") - params+=("--server-port=$1") - params+=("--agent-port=$2") - params+=("--admin-port=$3") + params+=("--server-port=0") + params+=("--agent-port=$1") + params+=("--admin-port=$2") konnectivity_args="" for param in "${params[@]}"; do konnectivity_args+=", \"${param}\"" done sed -i -e "s@{{ *konnectivity_args *}}@${konnectivity_args}@g" "${temp_file}" - sed -i -e "s@{{ *server_port *}}@$1@g" "${temp_file}" - sed -i -e "s@{{ *agent_port *}}@$2@g" "${temp_file}" - sed -i -e "s@{{ *admin_port *}}@$3@g" "${temp_file}" + sed -i -e "s@{{ *agent_port *}}@$1@g" "${temp_file}" + sed -i -e "s@{{ *admin_port *}}@$2@g" "${temp_file}" sed -i -e "s@{{ *liveness_probe_initial_delay *}}@30@g" "${temp_file}" mv "${temp_file}" /etc/kubernetes/manifests } @@ -1683,7 +1676,7 @@ function prepare-konnectivity-server-manifest { function start-konnectivity-server { echo "Start konnectivity server pods" prepare-log-file /var/log/konnectivity-server.log - prepare-konnectivity-server-manifest "8131" "8132" "8133" + prepare-konnectivity-server-manifest "8132" "8133" } # Calculates the following variables based on env variables, which will be used diff --git a/cluster/gce/gci/configure-kubeapiserver.sh b/cluster/gce/gci/configure-kubeapiserver.sh index 44a0dcc3502..0e787c43cc8 100644 --- a/cluster/gce/gci/configure-kubeapiserver.sh +++ b/cluster/gce/gci/configure-kubeapiserver.sh @@ -324,11 +324,18 @@ function start-kube-apiserver { local csc_config_mount="" local csc_config_volume="" + local default_konnectivity_socket_vol="" + local default_konnectivity_socket_mnt="" if [[ "${ENABLE_EGRESS_VIA_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then # Create the EgressSelectorConfiguration yaml file to control the Egress Selector. csc_config_mount="{\"name\": \"cscconfigmount\",\"mountPath\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"readOnly\": false}," csc_config_volume="{\"name\": \"cscconfigmount\",\"hostPath\": {\"path\": \"/etc/srv/kubernetes/egress_selector_configuration.yaml\", \"type\": \"FileOrCreate\"}}," params+=" --egress-selector-config-file=/etc/srv/kubernetes/egress_selector_configuration.yaml" + + # UDS socket for communication between apiserver and konnectivity-server + local default_konnectivity_socket_path="/etc/srv/kubernetes/konnectivity" + default_konnectivity_socket_vol="{ \"name\": \"konnectivity-socket\", \"hostPath\": {\"path\": \"${default_konnectivity_socket_path}\", \"type\": \"DirectoryOrCreate\"}}," + default_konnectivity_socket_mnt="{ \"name\": \"konnectivity-socket\", \"mountPath\": \"${default_konnectivity_socket_path}\", \"readOnly\": false}," fi local container_env="" @@ -377,6 +384,8 @@ function start-kube-apiserver { sed -i -e "s@{{audit_webhook_config_volume}}@${audit_webhook_config_volume}@g" "${src_file}" sed -i -e "s@{{webhook_exec_auth_plugin_mount}}@${webhook_exec_auth_plugin_mount}@g" "${src_file}" sed -i -e "s@{{webhook_exec_auth_plugin_volume}}@${webhook_exec_auth_plugin_volume}@g" "${src_file}" + sed -i -e "s@{{konnectivity_socket_mount}}@${default_konnectivity_socket_mnt}@g" "${src_file}" + sed -i -e "s@{{konnectivity_socket_volume}}@${default_konnectivity_socket_vol}@g" "${src_file}" cp "${src_file}" "${ETC_MANIFESTS:-/etc/kubernetes/manifests}" } diff --git a/cluster/gce/manifests/konnectivity-server.yaml b/cluster/gce/manifests/konnectivity-server.yaml index 26be55cfd3c..6b78653add7 100644 --- a/cluster/gce/manifests/konnectivity-server.yaml +++ b/cluster/gce/manifests/konnectivity-server.yaml @@ -11,7 +11,7 @@ spec: hostNetwork: true containers: - name: konnectivity-server-container - image: gcr.io/google-containers/proxy-server:v0.0.3 + image: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server:v0.0.4 resources: requests: cpu: 25m @@ -25,9 +25,6 @@ spec: initialDelaySeconds: {{ liveness_probe_initial_delay }} timeoutSeconds: 60 ports: - - name: serverport - containerPort: {{ server_port }} - hostPort: {{ server_port }} - name: agentport containerPort: {{ agent_port }} hostPort: {{ agent_port }} @@ -38,21 +35,21 @@ spec: - name: varlogkonnectivityserver mountPath: /var/log/konnectivity-server.log readOnly: false - - name: pkiserver - mountPath: /etc/srv/kubernetes/pki/konnectivity-server - readOnly: true - - name: pkiagent - mountPath: /etc/srv/kubernetes/pki/konnectivity-agent + - name: pki + mountPath: /etc/srv/kubernetes/pki readOnly: true + - name: konnectivity-uds + mountPath: /etc/srv/kubernetes/konnectivity + readOnly: false volumes: - name: varlogkonnectivityserver hostPath: path: /var/log/konnectivity-server.log type: FileOrCreate - - name: pkiserver + - name: pki hostPath: - path: /etc/srv/kubernetes/pki/konnectivity-server - - name: pkiagent + path: /etc/srv/kubernetes/pki + - name: konnectivity-uds hostPath: - path: /etc/srv/kubernetes/pki/konnectivity-agent - + path: /etc/srv/kubernetes/konnectivity + type: DirectoryOrCreate diff --git a/cluster/gce/manifests/kube-apiserver.manifest b/cluster/gce/manifests/kube-apiserver.manifest index 6f126db1b7c..34de7e4016f 100644 --- a/cluster/gce/manifests/kube-apiserver.manifest +++ b/cluster/gce/manifests/kube-apiserver.manifest @@ -66,7 +66,8 @@ {{csc_config_mount}} {{audit_policy_config_mount}} {{audit_webhook_config_mount}} - {{webhook_exec_auth_plugin_mount}} + {{webhook_exec_auth_plugin_mount}} + {{konnectivity_socket_mount}} { "name": "srvkube", "mountPath": "/etc/srv/kubernetes", "readOnly": true}, @@ -108,6 +109,7 @@ {{audit_policy_config_volume}} {{audit_webhook_config_volume}} {{webhook_exec_auth_plugin_volume}} + {{konnectivity_socket_volume}} { "name": "srvkube", "hostPath": { "path": "/etc/srv/kubernetes"} diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go index d21e7d631d7..d6fe4e9931b 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go @@ -71,29 +71,80 @@ type EgressSelection struct { // Connection provides the configuration for a single egress selection client. type Connection struct { - // Type is the type of connection used to connect from client to konnectivity server. - // Currently supported values are "http-connect" and "direct". - Type string + // Protocol is the protocol used to connect from client to the konnectivity server. + ProxyProtocol ProtocolType - // httpConnect is the config needed to use http-connect to the konnectivity server. + // Transport defines the transport configurations we use to dial to the konnectivity server + // This is required if ProxyProtocol is HTTPConnect or GRPC // +optional - HTTPConnect *HTTPConnectConfig + Transport *Transport } -type HTTPConnectConfig struct { +// ProtocolType is a set of valid values for Connection.ProtocolType +type ProtocolType string + +// Valid types for ProtocolType for konnectivity server +const ( + // Use HTTPConnect to connect to konnectivity server + ProtocolHTTPConnect ProtocolType = "HTTPConnect" + // Use grpc to connect to konnectivity server + ProtocolGRPC ProtocolType = "GRPC" + // Connect directly (skip konnectivity server) + ProtocolDirect ProtocolType = "Direct" +) + +// Transport defines the transport configurations we use to dial to the konnectivity server +type Transport struct { + // TCP is the TCP configuration for communicating with the konnectivity server via TCP + // Requires at least one of TCP or UDS to be set + // +optional + TCP *TCPTransport + + // UDS is the UDS configuration for communicating with the konnectivity server via UDS + // Requires at least one of TCP or UDS to be set + // +optional + UDS *UDSTransport +} + +// TCPTransport provides the information to connect to konnectivity server via TCP +type TCPTransport struct { // URL is the location of the konnectivity server to connect to. // As an example it might be "https://127.0.0.1:8131" URL string - // CABundle is the file location of the CA to be used to determine trust with the konnectivity server. + // TLSConfig is the config needed to use TLS when connecting to konnectivity server // +optional - CABundle string - - // ClientKey is the file location of the client key to be used in mtls handshakes with the konnectivity server. - // +optional - ClientKey string - - // ClientCert is the file location of the client certificate to be used in mtls handshakes with the konnectivity server. - // +optional - ClientCert string + TLSConfig *TLSConfig `json:"tlsConfig,omitempty"` +} + +// UDSTransport provides the information to connect to konnectivity server via UDS +type UDSTransport struct { + // UDSName is the name of the unix domain socket to connect to konnectivity server + // This does not use a unix:// prefix. (Eg: /etc/srv/kubernetes/konnectivity/konnectivity-server.socket) + UDSName string +} + +// TLSConfig provides the authentication information to connect to konnectivity server +// Only used with TCPTransport +type TLSConfig struct { + // caBundle is the file location of the CA to be used to determine trust with the konnectivity server. + // Must be absent/empty HTTPConnect using the plain http + // Must be configured for HTTPConnect using the https protocol + // Misconfiguration will cause an error + // +optional + CABundle string `json:"caBundle,omitempty"` + + // clientKey is the file location of the client key to authenticate with the konnectivity server + // Must be absent/empty HTTPConnect using the plain http + // Must be configured for HTTPConnect using the https protocol + // Misconfiguration will cause an error + // +optional + ClientKey string `json:"clientKey,omitempty"` + + // clientCert is the file location of the client certificate to authenticate with the konnectivity server + // Must be absent/empty HTTPConnect using the plain http + // Must be configured for HTTPConnect using the https protocol + // Misconfiguration will cause an error + // +optional + ClientCert string `json:"clientCert,omitempty"` } diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go index 10034d7c340..e561017e215 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go @@ -71,39 +71,78 @@ type EgressSelection struct { // Connection provides the configuration for a single egress selection client. type Connection struct { - // type is the type of connection used to connect from client to network/konnectivity server. - // Currently supported values are "http-connect" and "direct". - Type string `json:"type"` + // Protocol is the protocol used to connect from client to the konnectivity server. + ProxyProtocol ProtocolType `json:"proxyProtocol,omitempty"` - // httpConnect is the config needed to use http-connect to the konnectivity server. - // Absence when the type is "http-connect" will cause an error - // Presence when the type is "direct" will also cause an error + // Transport defines the transport configurations we use to dial to the konnectivity server + // This is required if ProxyProtocol is HTTPConnect or GRPC // +optional - HTTPConnect *HTTPConnectConfig `json:"httpConnect,omitempty"` + Transport *Transport `json:"transport,omitempty"` } -type HTTPConnectConfig struct { - // url is the location of the proxy server to connect to. - // As an example it might be "https://127.0.0.1:8131" - URL string `json:"url"` +// ProtocolType is a set of valid values for Connection.ProtocolType +type ProtocolType string +// Valid types for ProtocolType for konnectivity server +const ( + // Use HTTPConnect to connect to konnectivity server + ProtocolHTTPConnect ProtocolType = "HTTPConnect" + // Use grpc to connect to konnectivity server + ProtocolGRPC ProtocolType = "GRPC" + // Connect directly (skip konnectivity server) + ProtocolDirect ProtocolType = "Direct" +) + +// Transport defines the transport configurations we use to dial to the konnectivity server +type Transport struct { + // TCP is the TCP configuration for communicating with the konnectivity server via TCP + // Requires at least one of TCP or UDS to be set + // +optional + TCP *TCPTransport `json:"tcp,omitempty"` + + // UDS is the UDS configuration for communicating with the konnectivity server via UDS + // Requires at least one of TCP or UDS to be set + // +optional + UDS *UDSTransport `json:"uds,omitempty"` +} + +// TCPTransport provides the information to connect to konnectivity server via TCP +type TCPTransport struct { + // URL is the location of the konnectivity server to connect to. + // As an example it might be "https://127.0.0.1:8131" + URL string + + // TLSConfig is the config needed to use TLS when connecting to konnectivity server + // +optional + TLSConfig *TLSConfig +} + +// UDSTransport provides the information to connect to konnectivity server via UDS +type UDSTransport struct { + // UDSName is the name of the unix domain socket to connect to konnectivity server + UDSName string +} + +// TLSConfig provides the authentication information to connect to konnectivity server +// Only used with TCPTransport +type TLSConfig struct { // caBundle is the file location of the CA to be used to determine trust with the konnectivity server. - // Must be absent/empty http-connect using the plain http - // Must be configured for http-connect using the https protocol + // Must be absent/empty HTTPConnect using the plain http + // Must be configured for HTTPConnect using the https protocol // Misconfiguration will cause an error // +optional CABundle string `json:"caBundle,omitempty"` // clientKey is the file location of the client key to be used in mtls handshakes with the konnectivity server. - // Must be absent/empty http-connect using the plain http - // Must be configured for http-connect using the https protocol + // Must be absent/empty HTTPConnect using the plain http + // Must be configured for HTTPConnect using the https protocol // Misconfiguration will cause an error // +optional ClientKey string `json:"clientKey,omitempty"` // clientCert is the file location of the client certificate to be used in mtls handshakes with the konnectivity server. - // Must be absent/empty http-connect using the plain http - // Must be configured for http-connect using the https protocol + // Must be absent/empty HTTPConnect using the plain http + // Must be configured for HTTPConnect using the https protocol // Misconfiguration will cause an error // +optional ClientCert string `json:"clientCert,omitempty"` diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go index 80352f02e91..9174b16df70 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go @@ -85,13 +85,43 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*HTTPConnectConfig)(nil), (*apiserver.HTTPConnectConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(a.(*HTTPConnectConfig), b.(*apiserver.HTTPConnectConfig), scope) + if err := s.AddGeneratedConversionFunc((*TCPTransport)(nil), (*apiserver.TCPTransport)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_TCPTransport_To_apiserver_TCPTransport(a.(*TCPTransport), b.(*apiserver.TCPTransport), scope) }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*apiserver.HTTPConnectConfig)(nil), (*HTTPConnectConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(a.(*apiserver.HTTPConnectConfig), b.(*HTTPConnectConfig), scope) + if err := s.AddGeneratedConversionFunc((*apiserver.TCPTransport)(nil), (*TCPTransport)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_apiserver_TCPTransport_To_v1alpha1_TCPTransport(a.(*apiserver.TCPTransport), b.(*TCPTransport), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*TLSConfig)(nil), (*apiserver.TLSConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_TLSConfig_To_apiserver_TLSConfig(a.(*TLSConfig), b.(*apiserver.TLSConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*apiserver.TLSConfig)(nil), (*TLSConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_apiserver_TLSConfig_To_v1alpha1_TLSConfig(a.(*apiserver.TLSConfig), b.(*TLSConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*Transport)(nil), (*apiserver.Transport)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_Transport_To_apiserver_Transport(a.(*Transport), b.(*apiserver.Transport), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*apiserver.Transport)(nil), (*Transport)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_apiserver_Transport_To_v1alpha1_Transport(a.(*apiserver.Transport), b.(*Transport), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*UDSTransport)(nil), (*apiserver.UDSTransport)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_UDSTransport_To_apiserver_UDSTransport(a.(*UDSTransport), b.(*apiserver.UDSTransport), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*apiserver.UDSTransport)(nil), (*UDSTransport)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_apiserver_UDSTransport_To_v1alpha1_UDSTransport(a.(*apiserver.UDSTransport), b.(*UDSTransport), scope) }); err != nil { return err } @@ -143,8 +173,8 @@ func Convert_apiserver_AdmissionPluginConfiguration_To_v1alpha1_AdmissionPluginC } func autoConvert_v1alpha1_Connection_To_apiserver_Connection(in *Connection, out *apiserver.Connection, s conversion.Scope) error { - out.Type = in.Type - out.HTTPConnect = (*apiserver.HTTPConnectConfig)(unsafe.Pointer(in.HTTPConnect)) + out.ProxyProtocol = apiserver.ProtocolType(in.ProxyProtocol) + out.Transport = (*apiserver.Transport)(unsafe.Pointer(in.Transport)) return nil } @@ -154,8 +184,8 @@ func Convert_v1alpha1_Connection_To_apiserver_Connection(in *Connection, out *ap } func autoConvert_apiserver_Connection_To_v1alpha1_Connection(in *apiserver.Connection, out *Connection, s conversion.Scope) error { - out.Type = in.Type - out.HTTPConnect = (*HTTPConnectConfig)(unsafe.Pointer(in.HTTPConnect)) + out.ProxyProtocol = ProtocolType(in.ProxyProtocol) + out.Transport = (*Transport)(unsafe.Pointer(in.Transport)) return nil } @@ -210,28 +240,90 @@ func Convert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorCon return autoConvert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration(in, out, s) } -func autoConvert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(in *HTTPConnectConfig, out *apiserver.HTTPConnectConfig, s conversion.Scope) error { +func autoConvert_v1alpha1_TCPTransport_To_apiserver_TCPTransport(in *TCPTransport, out *apiserver.TCPTransport, s conversion.Scope) error { out.URL = in.URL + out.TLSConfig = (*apiserver.TLSConfig)(unsafe.Pointer(in.TLSConfig)) + return nil +} + +// Convert_v1alpha1_TCPTransport_To_apiserver_TCPTransport is an autogenerated conversion function. +func Convert_v1alpha1_TCPTransport_To_apiserver_TCPTransport(in *TCPTransport, out *apiserver.TCPTransport, s conversion.Scope) error { + return autoConvert_v1alpha1_TCPTransport_To_apiserver_TCPTransport(in, out, s) +} + +func autoConvert_apiserver_TCPTransport_To_v1alpha1_TCPTransport(in *apiserver.TCPTransport, out *TCPTransport, s conversion.Scope) error { + out.URL = in.URL + out.TLSConfig = (*TLSConfig)(unsafe.Pointer(in.TLSConfig)) + return nil +} + +// Convert_apiserver_TCPTransport_To_v1alpha1_TCPTransport is an autogenerated conversion function. +func Convert_apiserver_TCPTransport_To_v1alpha1_TCPTransport(in *apiserver.TCPTransport, out *TCPTransport, s conversion.Scope) error { + return autoConvert_apiserver_TCPTransport_To_v1alpha1_TCPTransport(in, out, s) +} + +func autoConvert_v1alpha1_TLSConfig_To_apiserver_TLSConfig(in *TLSConfig, out *apiserver.TLSConfig, s conversion.Scope) error { out.CABundle = in.CABundle out.ClientKey = in.ClientKey out.ClientCert = in.ClientCert return nil } -// Convert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig is an autogenerated conversion function. -func Convert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(in *HTTPConnectConfig, out *apiserver.HTTPConnectConfig, s conversion.Scope) error { - return autoConvert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(in, out, s) +// Convert_v1alpha1_TLSConfig_To_apiserver_TLSConfig is an autogenerated conversion function. +func Convert_v1alpha1_TLSConfig_To_apiserver_TLSConfig(in *TLSConfig, out *apiserver.TLSConfig, s conversion.Scope) error { + return autoConvert_v1alpha1_TLSConfig_To_apiserver_TLSConfig(in, out, s) } -func autoConvert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(in *apiserver.HTTPConnectConfig, out *HTTPConnectConfig, s conversion.Scope) error { - out.URL = in.URL +func autoConvert_apiserver_TLSConfig_To_v1alpha1_TLSConfig(in *apiserver.TLSConfig, out *TLSConfig, s conversion.Scope) error { out.CABundle = in.CABundle out.ClientKey = in.ClientKey out.ClientCert = in.ClientCert return nil } -// Convert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig is an autogenerated conversion function. -func Convert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(in *apiserver.HTTPConnectConfig, out *HTTPConnectConfig, s conversion.Scope) error { - return autoConvert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(in, out, s) +// Convert_apiserver_TLSConfig_To_v1alpha1_TLSConfig is an autogenerated conversion function. +func Convert_apiserver_TLSConfig_To_v1alpha1_TLSConfig(in *apiserver.TLSConfig, out *TLSConfig, s conversion.Scope) error { + return autoConvert_apiserver_TLSConfig_To_v1alpha1_TLSConfig(in, out, s) +} + +func autoConvert_v1alpha1_Transport_To_apiserver_Transport(in *Transport, out *apiserver.Transport, s conversion.Scope) error { + out.TCP = (*apiserver.TCPTransport)(unsafe.Pointer(in.TCP)) + out.UDS = (*apiserver.UDSTransport)(unsafe.Pointer(in.UDS)) + return nil +} + +// Convert_v1alpha1_Transport_To_apiserver_Transport is an autogenerated conversion function. +func Convert_v1alpha1_Transport_To_apiserver_Transport(in *Transport, out *apiserver.Transport, s conversion.Scope) error { + return autoConvert_v1alpha1_Transport_To_apiserver_Transport(in, out, s) +} + +func autoConvert_apiserver_Transport_To_v1alpha1_Transport(in *apiserver.Transport, out *Transport, s conversion.Scope) error { + out.TCP = (*TCPTransport)(unsafe.Pointer(in.TCP)) + out.UDS = (*UDSTransport)(unsafe.Pointer(in.UDS)) + return nil +} + +// Convert_apiserver_Transport_To_v1alpha1_Transport is an autogenerated conversion function. +func Convert_apiserver_Transport_To_v1alpha1_Transport(in *apiserver.Transport, out *Transport, s conversion.Scope) error { + return autoConvert_apiserver_Transport_To_v1alpha1_Transport(in, out, s) +} + +func autoConvert_v1alpha1_UDSTransport_To_apiserver_UDSTransport(in *UDSTransport, out *apiserver.UDSTransport, s conversion.Scope) error { + out.UDSName = in.UDSName + return nil +} + +// Convert_v1alpha1_UDSTransport_To_apiserver_UDSTransport is an autogenerated conversion function. +func Convert_v1alpha1_UDSTransport_To_apiserver_UDSTransport(in *UDSTransport, out *apiserver.UDSTransport, s conversion.Scope) error { + return autoConvert_v1alpha1_UDSTransport_To_apiserver_UDSTransport(in, out, s) +} + +func autoConvert_apiserver_UDSTransport_To_v1alpha1_UDSTransport(in *apiserver.UDSTransport, out *UDSTransport, s conversion.Scope) error { + out.UDSName = in.UDSName + return nil +} + +// Convert_apiserver_UDSTransport_To_v1alpha1_UDSTransport is an autogenerated conversion function. +func Convert_apiserver_UDSTransport_To_v1alpha1_UDSTransport(in *apiserver.UDSTransport, out *UDSTransport, s conversion.Scope) error { + return autoConvert_apiserver_UDSTransport_To_v1alpha1_UDSTransport(in, out, s) } diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.deepcopy.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.deepcopy.go index e8d6086798b..4498e408f0b 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.deepcopy.go @@ -80,10 +80,10 @@ func (in *AdmissionPluginConfiguration) DeepCopy() *AdmissionPluginConfiguration // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Connection) DeepCopyInto(out *Connection) { *out = *in - if in.HTTPConnect != nil { - in, out := &in.HTTPConnect, &out.HTTPConnect - *out = new(HTTPConnectConfig) - **out = **in + if in.Transport != nil { + in, out := &in.Transport, &out.Transport + *out = new(Transport) + (*in).DeepCopyInto(*out) } return } @@ -148,17 +148,80 @@ func (in *EgressSelectorConfiguration) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *HTTPConnectConfig) DeepCopyInto(out *HTTPConnectConfig) { +func (in *TCPTransport) DeepCopyInto(out *TCPTransport) { + *out = *in + if in.TLSConfig != nil { + in, out := &in.TLSConfig, &out.TLSConfig + *out = new(TLSConfig) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TCPTransport. +func (in *TCPTransport) DeepCopy() *TCPTransport { + if in == nil { + return nil + } + out := new(TCPTransport) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSConfig) DeepCopyInto(out *TLSConfig) { *out = *in return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPConnectConfig. -func (in *HTTPConnectConfig) DeepCopy() *HTTPConnectConfig { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig. +func (in *TLSConfig) DeepCopy() *TLSConfig { if in == nil { return nil } - out := new(HTTPConnectConfig) + out := new(TLSConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Transport) DeepCopyInto(out *Transport) { + *out = *in + if in.TCP != nil { + in, out := &in.TCP, &out.TCP + *out = new(TCPTransport) + (*in).DeepCopyInto(*out) + } + if in.UDS != nil { + in, out := &in.UDS, &out.UDS + *out = new(UDSTransport) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Transport. +func (in *Transport) DeepCopy() *Transport { + if in == nil { + return nil + } + out := new(Transport) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UDSTransport) DeepCopyInto(out *UDSTransport) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UDSTransport. +func (in *UDSTransport) DeepCopy() *UDSTransport { + if in == nil { + return nil + } + out := new(UDSTransport) in.DeepCopyInto(out) return out } diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/zz_generated.deepcopy.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/zz_generated.deepcopy.go index 3159f7c1a34..622f1b5dc83 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/zz_generated.deepcopy.go @@ -80,10 +80,10 @@ func (in *AdmissionPluginConfiguration) DeepCopy() *AdmissionPluginConfiguration // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Connection) DeepCopyInto(out *Connection) { *out = *in - if in.HTTPConnect != nil { - in, out := &in.HTTPConnect, &out.HTTPConnect - *out = new(HTTPConnectConfig) - **out = **in + if in.Transport != nil { + in, out := &in.Transport, &out.Transport + *out = new(Transport) + (*in).DeepCopyInto(*out) } return } @@ -148,17 +148,80 @@ func (in *EgressSelectorConfiguration) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *HTTPConnectConfig) DeepCopyInto(out *HTTPConnectConfig) { +func (in *TCPTransport) DeepCopyInto(out *TCPTransport) { + *out = *in + if in.TLSConfig != nil { + in, out := &in.TLSConfig, &out.TLSConfig + *out = new(TLSConfig) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TCPTransport. +func (in *TCPTransport) DeepCopy() *TCPTransport { + if in == nil { + return nil + } + out := new(TCPTransport) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSConfig) DeepCopyInto(out *TLSConfig) { *out = *in return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPConnectConfig. -func (in *HTTPConnectConfig) DeepCopy() *HTTPConnectConfig { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig. +func (in *TLSConfig) DeepCopy() *TLSConfig { if in == nil { return nil } - out := new(HTTPConnectConfig) + out := new(TLSConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Transport) DeepCopyInto(out *Transport) { + *out = *in + if in.TCP != nil { + in, out := &in.TCP, &out.TCP + *out = new(TCPTransport) + (*in).DeepCopyInto(*out) + } + if in.UDS != nil { + in, out := &in.UDS, &out.UDS + *out = new(UDSTransport) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Transport. +func (in *Transport) DeepCopy() *Transport { + if in == nil { + return nil + } + out := new(Transport) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UDSTransport) DeepCopyInto(out *UDSTransport) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UDSTransport. +func (in *UDSTransport) DeepCopy() *UDSTransport { + if in == nil { + return nil + } + out := new(UDSTransport) in.DeepCopyInto(out) return out } diff --git a/staging/src/k8s.io/apiserver/pkg/server/egressselector/BUILD b/staging/src/k8s.io/apiserver/pkg/server/egressselector/BUILD index 7c0e2faad7d..beb7899b0df 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/egressselector/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/egressselector/BUILD @@ -16,8 +16,10 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/apis/apiserver:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/apiserver/install:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/path:go_default_library", + "//vendor/sigs.k8s.io/apiserver-network-proxy/pkg/agent/client:go_default_library", "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/server/egressselector/config.go b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config.go index 8cbbb3b014e..2cf06a4e0da 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/egressselector/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config.go @@ -79,16 +79,38 @@ func ValidateEgressSelectorConfiguration(config *apiserver.EgressSelectorConfigu } for _, service := range config.EgressSelections { base := field.NewPath("service", "connection") - switch service.Connection.Type { - case "direct": + switch service.Connection.ProxyProtocol { + case apiserver.ProtocolDirect: allErrs = append(allErrs, validateDirectConnection(service.Connection, base)...) - case "http-connect": - allErrs = append(allErrs, validateHTTPConnection(service.Connection, base)...) + case apiserver.ProtocolHTTPConnect: + if service.Connection.Transport.TCP != nil && service.Connection.Transport.UDS != nil { + allErrs = append(allErrs, field.Invalid( + base.Child("tcp"), + service.Connection.Transport.TCP, + "TCP and UDS cannot both be set")) + } else if service.Connection.Transport.TCP == nil && service.Connection.Transport.UDS == nil { + allErrs = append(allErrs, field.Required( + base.Child("tcp"), + "One of TCP or UDS must be set")) + } else if service.Connection.Transport.TCP != nil { + allErrs = append(allErrs, validateTCPConnection(service.Connection, base)...) + } else if service.Connection.Transport.UDS != nil { + allErrs = append(allErrs, validateUDSConnection(service.Connection, base)...) + } + case apiserver.ProtocolGRPC: + if service.Connection.Transport.UDS != nil { + allErrs = append(allErrs, validateUDSConnection(service.Connection, base)...) + } else { + allErrs = append(allErrs, field.NotSupported( + base.Child("protocol"), + service.Connection.ProxyProtocol, + []string{"uds"})) + } default: allErrs = append(allErrs, field.NotSupported( - base.Child("type"), - service.Connection.Type, - []string{"direct", "http-connect"})) + base.Child("protocol"), + service.Connection.ProxyProtocol, + []string{"direct", "HTTPConnect", "grpc"})) } } @@ -96,80 +118,92 @@ func ValidateEgressSelectorConfiguration(config *apiserver.EgressSelectorConfigu } func validateDirectConnection(connection apiserver.Connection, fldPath *field.Path) field.ErrorList { - if connection.HTTPConnect != nil { + if connection.Transport != nil { return field.ErrorList{field.Invalid( - fldPath.Child("httpConnect"), + fldPath.Child("transport"), "direct", - "httpConnect config should be absent for direct connect"), + "Transport config should be absent for direct connect"), } } + return nil } -func validateHTTPConnection(connection apiserver.Connection, fldPath *field.Path) field.ErrorList { +func validateUDSConnection(connection apiserver.Connection, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} - if connection.HTTPConnect == nil { + if connection.Transport.UDS.UDSName == "" { allErrs = append(allErrs, field.Invalid( - fldPath.Child("httpConnect"), + fldPath.Child("udsName"), "nil", - "httpConnect config should be present for http-connect")) - } else if strings.HasPrefix(connection.HTTPConnect.URL, "https://") { - if connection.HTTPConnect.CABundle == "" { + "UDSName should be present for UDS connections")) + } + return allErrs +} + +func validateTCPConnection(connection apiserver.Connection, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if connection.Transport.TCP.TLSConfig == nil { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("tlsConfig"), + "nil", + "TLSConfig config should be present for HTTPConnect via tcp")) + } else if strings.HasPrefix(connection.Transport.TCP.URL, "https://") { + if connection.Transport.TCP.TLSConfig.CABundle == "" { allErrs = append(allErrs, field.Invalid( - fldPath.Child("httpConnect", "caBundle"), + fldPath.Child("tlsConfig", "caBundle"), "nil", - "http-connect via https requires caBundle")) - } else if exists, err := path.Exists(path.CheckFollowSymlink, connection.HTTPConnect.CABundle); exists == false || err != nil { + "HTTPConnect via https requires caBundle")) + } else if exists, err := path.Exists(path.CheckFollowSymlink, connection.Transport.TCP.TLSConfig.CABundle); exists == false || err != nil { allErrs = append(allErrs, field.Invalid( - fldPath.Child("httpConnect", "caBundle"), - connection.HTTPConnect.CABundle, - "http-connect ca bundle does not exist")) + fldPath.Child("tlsConfig", "caBundle"), + connection.Transport.TCP.TLSConfig.CABundle, + "HTTPConnect ca bundle does not exist")) } - if connection.HTTPConnect.ClientCert == "" { + if connection.Transport.TCP.TLSConfig.ClientCert == "" { allErrs = append(allErrs, field.Invalid( - fldPath.Child("httpConnect", "clientCert"), + fldPath.Child("tlsConfig", "clientCert"), "nil", - "http-connect via https requires clientCert")) - } else if exists, err := path.Exists(path.CheckFollowSymlink, connection.HTTPConnect.ClientCert); exists == false || err != nil { + "HTTPConnect via https requires clientCert")) + } else if exists, err := path.Exists(path.CheckFollowSymlink, connection.Transport.TCP.TLSConfig.ClientCert); exists == false || err != nil { allErrs = append(allErrs, field.Invalid( - fldPath.Child("httpConnect", "clientCert"), - connection.HTTPConnect.ClientCert, - "http-connect client cert does not exist")) + fldPath.Child("tlsConfig", "clientCert"), + connection.Transport.TCP.TLSConfig.ClientCert, + "HTTPConnect client cert does not exist")) } - if connection.HTTPConnect.ClientKey == "" { + if connection.Transport.TCP.TLSConfig.ClientKey == "" { allErrs = append(allErrs, field.Invalid( - fldPath.Child("httpConnect", "clientKey"), + fldPath.Child("tlsConfig", "clientKey"), "nil", - "http-connect via https requires clientKey")) - } else if exists, err := path.Exists(path.CheckFollowSymlink, connection.HTTPConnect.ClientKey); exists == false || err != nil { + "HTTPConnect via https requires clientKey")) + } else if exists, err := path.Exists(path.CheckFollowSymlink, connection.Transport.TCP.TLSConfig.ClientKey); exists == false || err != nil { allErrs = append(allErrs, field.Invalid( - fldPath.Child("httpConnect", "clientKey"), - connection.HTTPConnect.ClientKey, - "http-connect client key does not exist")) + fldPath.Child("tlsConfig", "clientKey"), + connection.Transport.TCP.TLSConfig.ClientKey, + "HTTPConnect client key does not exist")) } - } else if strings.HasPrefix(connection.HTTPConnect.URL, "http://") { - if connection.HTTPConnect.CABundle != "" { + } else if strings.HasPrefix(connection.Transport.TCP.URL, "http://") { + if connection.Transport.TCP.TLSConfig.CABundle != "" { allErrs = append(allErrs, field.Invalid( - fldPath.Child("httpConnect", "caBundle"), - connection.HTTPConnect.CABundle, - "http-connect via http does not support caBundle")) + fldPath.Child("tlsConfig", "caBundle"), + connection.Transport.TCP.TLSConfig.CABundle, + "HTTPConnect via http does not support caBundle")) } - if connection.HTTPConnect.ClientCert != "" { + if connection.Transport.TCP.TLSConfig.ClientCert != "" { allErrs = append(allErrs, field.Invalid( - fldPath.Child("httpConnect", "clientCert"), - connection.HTTPConnect.ClientCert, - "http-connect via http does not support clientCert")) + fldPath.Child("tlsConfig", "clientCert"), + connection.Transport.TCP.TLSConfig.ClientCert, + "HTTPConnect via http does not support clientCert")) } - if connection.HTTPConnect.ClientKey != "" { + if connection.Transport.TCP.TLSConfig.ClientKey != "" { allErrs = append(allErrs, field.Invalid( - fldPath.Child("httpConnect", "clientKey"), - connection.HTTPConnect.ClientKey, - "http-connect via http does not support clientKey")) + fldPath.Child("tlsConfig", "clientKey"), + connection.Transport.TCP.TLSConfig.ClientKey, + "HTTPConnect via http does not support clientKey")) } } else { allErrs = append(allErrs, field.Invalid( - fldPath.Child("httpConnect", "url"), - connection.HTTPConnect.URL, + fldPath.Child("url"), + connection.Transport.TCP.URL, "supported connection protocols are http:// and https://")) } return allErrs diff --git a/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go index a524ebffae5..0f36bffbd8b 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go @@ -62,23 +62,27 @@ kind: EgressSelectorConfiguration egressSelections: - name: "cluster" connection: - type: "http-connect" - httpConnect: - url: "https://127.0.0.1:8131" - caBundle: "/etc/srv/kubernetes/pki/konnectivity-server/ca.crt" - clientKey: "/etc/srv/kubernetes/pki/konnectivity-server/client.key" - clientCert: "/etc/srv/kubernetes/pki/konnectivity-server/client.crt" + proxyProtocol: "HTTPConnect" + transport: + tcp: + url: "https://127.0.0.1:8131" + tlsConfig: + caBundle: "/etc/srv/kubernetes/pki/konnectivity-server/ca.crt" + clientKey: "/etc/srv/kubernetes/pki/konnectivity-server/client.key" + clientCert: "/etc/srv/kubernetes/pki/konnectivity-server/client.crt" - name: "master" connection: - type: "http-connect" - httpConnect: - url: "https://127.0.0.1:8132" - caBundle: "/etc/srv/kubernetes/pki/konnectivity-server-master/ca.crt" - clientKey: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.key" - clientCert: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.crt" + proxyProtocol: "HTTPConnect" + transport: + tcp: + url: "https://127.0.0.1:8132" + tlsConfig: + caBundle: "/etc/srv/kubernetes/pki/konnectivity-server-master/ca.crt" + clientKey: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.key" + clientCert: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.crt" - name: "etcd" connection: - type: "direct" + proxyProtocol: "Direct" `, expectedResult: &apiserver.EgressSelectorConfiguration{ TypeMeta: metav1.TypeMeta{ @@ -89,31 +93,40 @@ egressSelections: { Name: "cluster", Connection: apiserver.Connection{ - Type: "http-connect", - HTTPConnect: &apiserver.HTTPConnectConfig{ - URL: "https://127.0.0.1:8131", - CABundle: "/etc/srv/kubernetes/pki/konnectivity-server/ca.crt", - ClientKey: "/etc/srv/kubernetes/pki/konnectivity-server/client.key", - ClientCert: "/etc/srv/kubernetes/pki/konnectivity-server/client.crt", + ProxyProtocol: "HTTPConnect", + Transport: &apiserver.Transport{ + TCP: &apiserver.TCPTransport{ + URL: "https://127.0.0.1:8131", + + TLSConfig: &apiserver.TLSConfig{ + CABundle: "/etc/srv/kubernetes/pki/konnectivity-server/ca.crt", + ClientKey: "/etc/srv/kubernetes/pki/konnectivity-server/client.key", + ClientCert: "/etc/srv/kubernetes/pki/konnectivity-server/client.crt", + }, + }, }, }, }, { Name: "master", Connection: apiserver.Connection{ - Type: "http-connect", - HTTPConnect: &apiserver.HTTPConnectConfig{ - URL: "https://127.0.0.1:8132", - CABundle: "/etc/srv/kubernetes/pki/konnectivity-server-master/ca.crt", - ClientKey: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.key", - ClientCert: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.crt", + ProxyProtocol: "HTTPConnect", + Transport: &apiserver.Transport{ + TCP: &apiserver.TCPTransport{ + URL: "https://127.0.0.1:8132", + TLSConfig: &apiserver.TLSConfig{ + CABundle: "/etc/srv/kubernetes/pki/konnectivity-server-master/ca.crt", + ClientKey: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.key", + ClientCert: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.crt", + }, + }, }, }, }, { Name: "etcd", Connection: apiserver.Connection{ - Type: "direct", + ProxyProtocol: "Direct", }, }, }, diff --git a/staging/src/k8s.io/apiserver/pkg/server/egressselector/egress_selector.go b/staging/src/k8s.io/apiserver/pkg/server/egressselector/egress_selector.go index d043e1392cd..919741d78d6 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/egressselector/egress_selector.go +++ b/staging/src/k8s.io/apiserver/pkg/server/egressselector/egress_selector.go @@ -22,6 +22,7 @@ import ( "crypto/tls" "crypto/x509" "fmt" + "google.golang.org/grpc" "io/ioutil" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apiserver/pkg/apis/apiserver" @@ -29,6 +30,7 @@ import ( "net" "net/http" "net/url" + client "sigs.k8s.io/apiserver-network-proxy/pkg/agent/client" "strings" ) @@ -94,13 +96,40 @@ func lookupServiceName(name string) (EgressType, error) { return -1, fmt.Errorf("unrecognized service name %s", name) } -func createConnectDialer(connectConfig *apiserver.HTTPConnectConfig) (utilnet.DialFunc, error) { - clientCert := connectConfig.ClientCert - clientKey := connectConfig.ClientKey - caCert := connectConfig.CABundle - proxyURL, err := url.Parse(connectConfig.URL) +func tunnelHTTPConnect(proxyConn net.Conn, proxyAddress, addr string) (net.Conn, error) { + fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", addr, "127.0.0.1") + br := bufio.NewReader(proxyConn) + res, err := http.ReadResponse(br, nil) if err != nil { - return nil, fmt.Errorf("invalid proxy server url %q: %v", connectConfig.URL, err) + proxyConn.Close() + return nil, fmt.Errorf("reading HTTP response from CONNECT to %s via proxy %s failed: %v", + addr, proxyAddress, err) + } + if res.StatusCode != 200 { + proxyConn.Close() + return nil, fmt.Errorf("proxy error from %s while dialing %s, code %d: %v", + proxyAddress, addr, res.StatusCode, res.Status) + } + + // It's safe to discard the bufio.Reader here and return the + // original TCP conn directly because we only use this for + // TLS, and in TLS the client speaks first, so we know there's + // no unbuffered data. But we can double-check. + if br.Buffered() > 0 { + proxyConn.Close() + return nil, fmt.Errorf("unexpected %d bytes of buffered data from CONNECT proxy %q", + br.Buffered(), proxyAddress) + } + return proxyConn, nil +} + +func createConnectTCPDialer(tcpTransport *apiserver.TCPTransport) (utilnet.DialFunc, error) { + clientCert := tcpTransport.TLSConfig.ClientCert + clientKey := tcpTransport.TLSConfig.ClientKey + caCert := tcpTransport.TLSConfig.CABundle + proxyURL, err := url.Parse(tcpTransport.URL) + if err != nil { + return nil, fmt.Errorf("invalid proxy server url %q: %v", tcpTransport.URL, err) } proxyAddress := proxyURL.Host @@ -128,30 +157,42 @@ func createConnectDialer(connectConfig *apiserver.HTTPConnectConfig) (utilnet.Di if err != nil { return nil, fmt.Errorf("dialing proxy %q failed: %v", proxyAddress, err) } - fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", addr, "127.0.0.1") - br := bufio.NewReader(proxyConn) - res, err := http.ReadResponse(br, nil) + return tunnelHTTPConnect(proxyConn, proxyAddress, addr) + } + return contextDialer, nil +} + +func createConnectUDSDialer(udsName string) (utilnet.DialFunc, error) { + contextDialer := func(ctx context.Context, network, addr string) (net.Conn, error) { + proxyConn, err := net.Dial("unix", udsName) if err != nil { - proxyConn.Close() - return nil, fmt.Errorf("reading HTTP response from CONNECT to %s via proxy %s failed: %v", - addr, proxyAddress, err) + return nil, fmt.Errorf("dialing proxy %q failed: %v", udsName, err) } - if res.StatusCode != 200 { - proxyConn.Close() - return nil, fmt.Errorf("proxy error from %s while dialing %s, code %d: %v", - proxyAddress, addr, res.StatusCode, res.Status) + return tunnelHTTPConnect(proxyConn, udsName, addr) + } + return contextDialer, nil +} + +func createGRPCUDSDialer(udsName string) (utilnet.DialFunc, error) { + contextDialer := func(ctx context.Context, network, addr string) (net.Conn, error) { + + dialOption := grpc.WithContextDialer(func(context.Context, string) (net.Conn, error) { + c, err := net.Dial("unix", udsName) + if err != nil { + klog.Errorf("failed to create connection to uds name %s, error: %v", udsName, err) + } + return c, err + }) + + tunnel, err := client.CreateGrpcTunnel(udsName, dialOption, grpc.WithInsecure()) + if err != nil { + return nil, err } - // It's safe to discard the bufio.Reader here and return the - // original TCP conn directly because we only use this for - // TLS, and in TLS the client speaks first, so we know there's - // no unbuffered data. But we can double-check. - if br.Buffered() > 0 { - proxyConn.Close() - return nil, fmt.Errorf("unexpected %d bytes of buffered data from CONNECT proxy %q", - br.Buffered(), proxyAddress) + proxyConn, err := tunnel.Dial("tcp", addr) + if err != nil { + return nil, err } - klog.V(4).Infof("About to proxy request to %s over %s.", addr, proxyAddress) return proxyConn, nil } return contextDialer, nil @@ -172,17 +213,39 @@ func NewEgressSelector(config *apiserver.EgressSelectorConfiguration) (*EgressSe if err != nil { return nil, err } - switch service.Connection.Type { - case "http-connect": - contextDialer, err := createConnectDialer(service.Connection.HTTPConnect) - if err != nil { - return nil, fmt.Errorf("failed to create http-connect dialer: %v", err) + switch service.Connection.ProxyProtocol { + + case apiserver.ProtocolHTTPConnect: + if service.Connection.Transport.UDS != nil { + contextDialer, err := createConnectUDSDialer(service.Connection.Transport.UDS.UDSName) + if err != nil { + return nil, fmt.Errorf("failed to create HTTPConnect uds dialer: %v", err) + } + cs.egressToDialer[name] = contextDialer + } else if service.Connection.Transport.TCP != nil { + contextDialer, err := createConnectTCPDialer(service.Connection.Transport.TCP) + if err != nil { + return nil, fmt.Errorf("failed to create HTTPConnect dialer: %v", err) + } + cs.egressToDialer[name] = contextDialer + } else { + return nil, fmt.Errorf("Either TCP or UDP transport must be specified") } - cs.egressToDialer[name] = contextDialer - case "direct": + case apiserver.ProtocolGRPC: + if service.Connection.Transport.UDS != nil { + grpcContextDialer, err := createGRPCUDSDialer(service.Connection.Transport.UDS.UDSName) + if err != nil { + return nil, fmt.Errorf("failed to create grpc dialer: %v", err) + } + cs.egressToDialer[name] = grpcContextDialer + + } else { + return nil, fmt.Errorf("Either TCP or UDP transport must be specified") + } + case apiserver.ProtocolDirect: cs.egressToDialer[name] = directDialer default: - return nil, fmt.Errorf("unrecognized service connection type %q", service.Connection.Type) + return nil, fmt.Errorf("unrecognized service connection protocol %q", service.Connection.ProxyProtocol) } } return cs, nil diff --git a/staging/src/k8s.io/apiserver/pkg/server/egressselector/egress_selector_test.go b/staging/src/k8s.io/apiserver/pkg/server/egressselector/egress_selector_test.go index e24a5bdbb3b..dcd51906f02 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/egressselector/egress_selector_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/egressselector/egress_selector_test.go @@ -53,37 +53,19 @@ func TestEgressSelector(t *testing.T) { { Name: "cluster", Connection: apiserver.Connection{ - Type: "direct", - HTTPConnect: &apiserver.HTTPConnectConfig{ - URL: "", - CABundle: "", - ClientKey: "", - ClientCert: "", - }, + ProxyProtocol: apiserver.ProtocolDirect, }, }, { Name: "master", Connection: apiserver.Connection{ - Type: "direct", - HTTPConnect: &apiserver.HTTPConnectConfig{ - URL: "", - CABundle: "", - ClientKey: "", - ClientCert: "", - }, + ProxyProtocol: apiserver.ProtocolDirect, }, }, { Name: "etcd", Connection: apiserver.Connection{ - Type: "direct", - HTTPConnect: &apiserver.HTTPConnectConfig{ - URL: "", - CABundle: "", - ClientKey: "", - ClientCert: "", - }, + ProxyProtocol: apiserver.ProtocolDirect, }, }, },