From f7bd5455fe0b730248b4a1f7d07e8225ef34f136 Mon Sep 17 00:00:00 2001 From: immutablet Date: Mon, 4 Nov 2019 11:17:52 -0800 Subject: [PATCH] Isolate configuration of etcd related parameters into a separate function. --- cluster/gce/gci/apiserver_etcd_test.go | 138 ++++++++++++++++-- cluster/gce/gci/configure-kubeapiserver.sh | 69 +++++---- .../gci/testdata/kube-apiserver/etcd.template | 4 + 3 files changed, 172 insertions(+), 39 deletions(-) diff --git a/cluster/gce/gci/apiserver_etcd_test.go b/cluster/gce/gci/apiserver_etcd_test.go index d9fde354cb2..8036e5ea896 100644 --- a/cluster/gce/gci/apiserver_etcd_test.go +++ b/cluster/gce/gci/apiserver_etcd_test.go @@ -22,17 +22,132 @@ import ( ) type kubeAPIServeETCDEnv struct { - KubeHome string - ETCDServers string - CAKey string - CACert string - CACertPath string - APIServerKey string - APIServerCert string - APIServerCertPath string - APIServerKeyPath string - ETCDKey string - ETCDCert string + KubeHome string + ETCDServers string + ETCDServersOverride string + CAKey string + CACert string + CACertPath string + APIServerKey string + APIServerCert string + APIServerCertPath string + APIServerKeyPath string + ETCDKey string + ETCDCert string + StorageBackend string + StorageMediaType string + CompactionInterval string +} + +func TestServerOverride(t *testing.T) { + testCases := []struct { + desc string + env kubeAPIServeETCDEnv + want []string + }{ + { + desc: "ETCD-SERVERS is not set - default override", + want: []string{ + "--etcd-servers-overrides=/events#http://127.0.0.1:4002", + }, + }, + { + desc: "ETCD-SERVERS and ETCD_SERVERS_OVERRIDES iare set", + env: kubeAPIServeETCDEnv{ + ETCDServers: "ETCDServers", + ETCDServersOverride: "ETCDServersOverrides", + }, + want: []string{ + "--etcd-servers-overrides=ETCDServersOverrides", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + c := newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, nil) + defer c.tearDown() + tc.env.KubeHome = c.kubeHome + + c.mustInvokeFunc( + tc.env, + kubeAPIServerConfigScriptName, + "etcd.template", + "testdata/kube-apiserver/base.template", + "testdata/kube-apiserver/etcd.template", + ) + c.mustLoadPodFromManifest() + + execArgs := c.pod.Spec.Containers[0].Command[2] + for _, f := range tc.want { + if !strings.Contains(execArgs, f) { + t.Fatalf("Got %q, want it to contain %q", execArgs, f) + } + } + }) + } +} + +func TestStorageOptions(t *testing.T) { + testCases := []struct { + desc string + env kubeAPIServeETCDEnv + want []string + dontWant []string + }{ + { + desc: "storage options are supplied", + env: kubeAPIServeETCDEnv{ + StorageBackend: "StorageBackend", + StorageMediaType: "StorageMediaType", + CompactionInterval: "1s", + }, + want: []string{ + "--storage-backend=StorageBackend", + "--storage-media-type=StorageMediaType", + "--etcd-compaction-interval=1s", + }, + }, + { + desc: "storage options not not supplied", + env: kubeAPIServeETCDEnv{}, + dontWant: []string{ + "--storage-backend", + "--storage-media-type", + "--etcd-compaction-interval", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + c := newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, nil) + defer c.tearDown() + tc.env.KubeHome = c.kubeHome + + c.mustInvokeFunc( + tc.env, + kubeAPIServerConfigScriptName, + "etcd.template", + "testdata/kube-apiserver/base.template", + "testdata/kube-apiserver/etcd.template", + ) + c.mustLoadPodFromManifest() + + execArgs := c.pod.Spec.Containers[0].Command[2] + for _, f := range tc.want { + if !strings.Contains(execArgs, f) { + t.Fatalf("Got %q, want it to contain %q", execArgs, f) + } + } + + for _, f := range tc.dontWant { + if strings.Contains(execArgs, f) { + t.Fatalf("Got %q, but it was not expected it to contain %q", execArgs, f) + } + } + }) + } } func TestTLSFlags(t *testing.T) { @@ -89,7 +204,6 @@ func TestTLSFlags(t *testing.T) { t.Fatalf("Got %q, want it to contain %q", execArgs, f) } } - }) } } diff --git a/cluster/gce/gci/configure-kubeapiserver.sh b/cluster/gce/gci/configure-kubeapiserver.sh index edc0af586c5..44a0dcc3502 100644 --- a/cluster/gce/gci/configure-kubeapiserver.sh +++ b/cluster/gce/gci/configure-kubeapiserver.sh @@ -13,6 +13,43 @@ # See the License for the specific language governing permissions and # limitations under the License. + +# Configures etcd related flags of kube-apiserver. +function configure-etcd-params { + local -n params_ref=$1 + + if [[ -n "${ETCD_APISERVER_CA_KEY:-}" && -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_SERVER_KEY:-}" && -n "${ETCD_APISERVER_SERVER_CERT:-}" && -n "${ETCD_APISERVER_CLIENT_KEY:-}" && -n "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then + params_ref+=" --etcd-servers=${ETCD_SERVERS:-https://127.0.0.1:2379}" + params_ref+=" --etcd-cafile=${ETCD_APISERVER_CA_CERT_PATH}" + params_ref+=" --etcd-certfile=${ETCD_APISERVER_CLIENT_CERT_PATH}" + params_ref+=" --etcd-keyfile=${ETCD_APISERVER_CLIENT_KEY_PATH}" + elif [[ -z "${ETCD_APISERVER_CA_KEY:-}" && -z "${ETCD_APISERVER_CA_CERT:-}" && -z "${ETCD_APISERVER_SERVER_KEY:-}" && -z "${ETCD_APISERVER_SERVER_CERT:-}" && -z "${ETCD_APISERVER_CLIENT_KEY:-}" && -z "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then + params_ref+=" --etcd-servers=${ETCD_SERVERS:-http://127.0.0.1:2379}" + echo "WARNING: ALL of ETCD_APISERVER_CA_KEY, ETCD_APISERVER_CA_CERT, ETCD_APISERVER_SERVER_KEY, ETCD_APISERVER_SERVER_CERT, ETCD_APISERVER_CLIENT_KEY and ETCD_APISERVER_CLIENT_CERT are missing, mTLS between etcd server and kube-apiserver is not enabled." + else + echo "ERROR: Some of ETCD_APISERVER_CA_KEY, ETCD_APISERVER_CA_CERT, ETCD_APISERVER_SERVER_KEY, ETCD_APISERVER_SERVER_CERT, ETCD_APISERVER_CLIENT_KEY and ETCD_APISERVER_CLIENT_CERT are missing, mTLS between etcd server and kube-apiserver cannot be enabled. Please provide all mTLS credential." + exit 1 + fi + + if [[ -z "${ETCD_SERVERS:-}" ]]; then + params_ref+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-/events#http://127.0.0.1:4002}" + elif [[ -n "${ETCD_SERVERS_OVERRIDES:-}" ]]; then + params_ref+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-}" + fi + + if [[ -n "${STORAGE_BACKEND:-}" ]]; then + params_ref+=" --storage-backend=${STORAGE_BACKEND}" + fi + + if [[ -n "${STORAGE_MEDIA_TYPE:-}" ]]; then + params_ref+=" --storage-media-type=${STORAGE_MEDIA_TYPE}" + fi + + if [[ -n "${ETCD_COMPACTION_INTERVAL_SEC:-}" ]]; then + params_ref+=" --etcd-compaction-interval=${ETCD_COMPACTION_INTERVAL_SEC}s" + fi +} + # Starts kubernetes apiserver. # It prepares the log file, loads the docker image, calculates variables, sets them # in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. @@ -34,23 +71,10 @@ function start-kube-apiserver { params+=" --allow-privileged=true" params+=" --cloud-provider=gce" params+=" --client-ca-file=${CA_CERT_BUNDLE_PATH}" - if [[ -n "${ETCD_APISERVER_CA_KEY:-}" && -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_SERVER_KEY:-}" && -n "${ETCD_APISERVER_SERVER_CERT:-}" && -n "${ETCD_APISERVER_CLIENT_KEY:-}" && -n "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then - params+=" --etcd-servers=${ETCD_SERVERS:-https://127.0.0.1:2379}" - params+=" --etcd-cafile=${ETCD_APISERVER_CA_CERT_PATH}" - params+=" --etcd-certfile=${ETCD_APISERVER_CLIENT_CERT_PATH}" - params+=" --etcd-keyfile=${ETCD_APISERVER_CLIENT_KEY_PATH}" - elif [[ -z "${ETCD_APISERVER_CA_KEY:-}" && -z "${ETCD_APISERVER_CA_CERT:-}" && -z "${ETCD_APISERVER_SERVER_KEY:-}" && -z "${ETCD_APISERVER_SERVER_CERT:-}" && -z "${ETCD_APISERVER_CLIENT_KEY:-}" && -z "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then - params+=" --etcd-servers=${ETCD_SERVERS:-http://127.0.0.1:2379}" - echo "WARNING: ALL of ETCD_APISERVER_CA_KEY, ETCD_APISERVER_CA_CERT, ETCD_APISERVER_SERVER_KEY, ETCD_APISERVER_SERVER_CERT, ETCD_APISERVER_CLIENT_KEY and ETCD_APISERVER_CLIENT_CERT are missing, mTLS between etcd server and kube-apiserver is not enabled." - else - echo "ERROR: Some of ETCD_APISERVER_CA_KEY, ETCD_APISERVER_CA_CERT, ETCD_APISERVER_SERVER_KEY, ETCD_APISERVER_SERVER_CERT, ETCD_APISERVER_CLIENT_KEY and ETCD_APISERVER_CLIENT_CERT are missing, mTLS between etcd server and kube-apiserver cannot be enabled. Please provide all mTLS credential." - exit 1 - fi - if [[ -z "${ETCD_SERVERS:-}" ]]; then - params+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-/events#http://127.0.0.1:4002}" - elif [[ -n "${ETCD_SERVERS_OVERRIDES:-}" ]]; then - params+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-}" - fi + + # params is passed by reference, so no "$" + configure-etcd-params params + params+=" --secure-port=443" if [[ "${ENABLE_APISERVER_INSECURE_PORT:-false}" != "true" ]]; then # Default is :8080 @@ -80,15 +104,7 @@ function start-kube-apiserver { if [[ -n "${KUBE_PASSWORD:-}" && -n "${KUBE_USER:-}" ]]; then params+=" --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv" fi - if [[ -n "${STORAGE_BACKEND:-}" ]]; then - params+=" --storage-backend=${STORAGE_BACKEND}" - fi - if [[ -n "${STORAGE_MEDIA_TYPE:-}" ]]; then - params+=" --storage-media-type=${STORAGE_MEDIA_TYPE}" - fi - if [[ -n "${ETCD_COMPACTION_INTERVAL_SEC:-}" ]]; then - params+=" --etcd-compaction-interval=${ETCD_COMPACTION_INTERVAL_SEC}s" - fi + if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT_SEC:-}" ]]; then params+=" --request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT_SEC}s" fi @@ -268,7 +284,6 @@ function start-kube-apiserver { fi fi - local authorization_mode="RBAC" local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty" diff --git a/cluster/gce/gci/testdata/kube-apiserver/etcd.template b/cluster/gce/gci/testdata/kube-apiserver/etcd.template index ccc10e2dbb6..1624451c56b 100644 --- a/cluster/gce/gci/testdata/kube-apiserver/etcd.template +++ b/cluster/gce/gci/testdata/kube-apiserver/etcd.template @@ -9,3 +9,7 @@ readonly ETCD_SERVERS={{.ETCDServers}} readonly ETCD_APISERVER_CA_CERT_PATH={{.CACertPath}} readonly ETCD_APISERVER_CLIENT_CERT_PATH={{.APIServerCertPath}} readonly ETCD_APISERVER_CLIENT_KEY_PATH={{.APIServerKeyPath}} +readonly ETCD_SERVERS_OVERRIDES={{.ETCDServersOverride}} +readonly STORAGE_BACKEND={{.StorageBackend}} +readonly STORAGE_MEDIA_TYPE={{.StorageMediaType}} +readonly ETCD_COMPACTION_INTERVAL_SEC={{.CompactionInterval}}