From b42023eb03639d03bd0be286404c7d1ddb395f82 Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Fri, 12 Aug 2016 08:52:34 +0200 Subject: [PATCH] Support for etcd migration --- cluster/gce/gci/configure-helper.sh | 1 + cluster/gce/trusty/configure-helper.sh | 1 + cluster/images/etcd/Dockerfile | 2 +- cluster/images/etcd/migrate-if-needed.sh | 65 ++++++++++++++++++++++++ cluster/saltbase/salt/etcd/etcd.manifest | 29 +++++++---- hack/verify-flags/exceptions.txt | 18 ++++--- 6 files changed, 97 insertions(+), 19 deletions(-) create mode 100755 cluster/images/etcd/migrate-if-needed.sh diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 99fe866feac..70230d58095 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -543,6 +543,7 @@ function prepare-etcd-manifest { sed -i -e "s@{{ *cpulimit *}}@\"$4\"@g" "${temp_file}" sed -i -e "s@{{ *hostname *}}@$host_name@g" "${temp_file}" sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${temp_file}" + sed -i -e "s@{{ *storage_backend *}}@${STORAGE_BACKEND:-}@g" "${temp_file}" sed -i -e "s@{{ *cluster_state *}}@$cluster_state@g" "${temp_file}" # Replace the volume host path. sed -i -e "s@/mnt/master-pd/var/etcd@/mnt/disks/master-pd/var/etcd@g" "${temp_file}" diff --git a/cluster/gce/trusty/configure-helper.sh b/cluster/gce/trusty/configure-helper.sh index f2115727c8c..944740217e8 100644 --- a/cluster/gce/trusty/configure-helper.sh +++ b/cluster/gce/trusty/configure-helper.sh @@ -429,6 +429,7 @@ prepare_etcd_manifest() { sed -i -e "s@{{ *cpulimit *}}@\"$4\"@g" "${etcd_temp_file}" sed -i -e "s@{{ *hostname *}}@$host_name@g" "${etcd_temp_file}" sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${etcd_temp_file}" + sed -i -e "s@{{ *storage_backend *}}@${STORAGE_BACKEND:-}@g" "${temp_file}" sed -i -e "s@{{ *cluster_state *}}@$cluster_state@g" "${etcd_temp_file}" # Replace the volume host path sed -i -e "s@/mnt/master-pd/var/etcd@/mnt/disks/master-pd/var/etcd@g" "${etcd_temp_file}" diff --git a/cluster/images/etcd/Dockerfile b/cluster/images/etcd/Dockerfile index 6199aef8625..ef2a8254d0f 100644 --- a/cluster/images/etcd/Dockerfile +++ b/cluster/images/etcd/Dockerfile @@ -16,4 +16,4 @@ FROM BASEIMAGE MAINTAINER Dawn Chen EXPOSE 2379 2380 4001 7001 -COPY etcd etcdctl /usr/local/bin/ +COPY etcd etcdctl migrate-if-needed.sh /usr/local/bin/ diff --git a/cluster/images/etcd/migrate-if-needed.sh b/cluster/images/etcd/migrate-if-needed.sh new file mode 100755 index 00000000000..83cdca9436b --- /dev/null +++ b/cluster/images/etcd/migrate-if-needed.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script performs data migration between etcd2 and etcd3 versions +# if needed. +# Expected usage of it is: +# ./migrate_if_needed +# It will look into the contents of file /version.txt to +# determine the current storage version (no file means etcd2). + +set -o errexit +set -o nounset +set -o pipefail + +if [ -z "${TARGET_STORAGE:-}" ]; then + echo "TARGET_USAGE variable unset - skipping migration" + exit 0 +fi + +if [ -z "${DATA_DIRECTORY:-}" ]; then + echo "DATA_DIRECTORY variable unset - skipping migration" + exit 0 +fi + +VERSION_FILE="version.txt" +CURRENT_STORAGE='etcd2' +if [ -e "${DATA_DIRECTORY}/${VERSION_FILE}" ]; then + CURRENT_STORAGE="$(cat ${DATA_DIRECTORY}/${VERSION_FILE})" +fi + +if [ "${CURRENT_STORAGE}" = "etcd2" -a "${TARGET_STORAGE}" = "etcd3" ]; then + # If directory doesn't exist or is empty, this means that there aren't any + # data for migration, which means we can skip this step. + if [ -d "${DATA_DIRECTORY}" ]; then + if [ "$(ls -A ${DATA_DIRECTORY})" ]; then + echo "Performing etcd2 -> etcd3 migration" + # TODO: Pass a correct transformer to handle TTLs. + ETCDCTL_API=3 /usr/local/bin/etcdctl migrate --data-dir=${DATA_DIRECTORY} + fi + fi +fi + +if [ "${CURRENT_STORAGE}" = "etcd3" -a "${TARGET_STORAGE}" = "etcd2" ]; then + echo "Performing etcd3 -> etcd2 migration" + # TODO: Implement rollback once this will be supported. + echo "etcd3 -> etcd2 downgrade is NOT supported." +fi + +# Write current storage version to avoid future migrations. +# If directory doesn't exist, we need to create it first. +mkdir -p "${DATA_DIRECTORY}" +echo "${TARGET_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}" diff --git a/cluster/saltbase/salt/etcd/etcd.manifest b/cluster/saltbase/salt/etcd/etcd.manifest index e3078e200b2..9838ed45c81 100644 --- a/cluster/saltbase/salt/etcd/etcd.manifest +++ b/cluster/saltbase/salt/etcd/etcd.manifest @@ -14,6 +14,7 @@ {% endfor -%} {% set etcd_cluster = vars.etcd_cluster -%} {% set cluster_state = vars.cluster_state -%} +{% set storage_backend = pillar.get('storage_backend', 'etcd2') -%} { "apiVersion": "v1", @@ -38,6 +39,14 @@ "-c", "/usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls http://{{ hostname }}:{{ server_port }} --initial-advertise-peer-urls http://{{ hostname }}:{{ server_port }} --advertise-client-urls http://127.0.0.1:{{ port }} --listen-client-urls http://127.0.0.1:{{ port }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} 1>>/var/log/etcd{{ suffix }}.log 2>&1" ], + "env": [ + { "name": "TARGET_STORAGE", + "value": "{{ storage_backend }}" + }, + { "name": "DATA_DIRECTORY", + "value": "/var/etcd/data{{ suffix }}" + } + ], "livenessProbe": { "httpGet": { "host": "127.0.0.1", @@ -47,26 +56,26 @@ "initialDelaySeconds": 15, "timeoutSeconds": 15 }, - "ports":[ + "ports": [ { "name": "serverport", "containerPort": {{ server_port }}, "hostPort": {{ server_port }} - },{ - "name": "clientport", + }, + { "name": "clientport", "containerPort": {{ port }}, "hostPort": {{ port }} } ], "volumeMounts": [ - {"name": "varetcd", - "mountPath": "/var/etcd", - "readOnly": false + { "name": "varetcd", + "mountPath": "/var/etcd", + "readOnly": false }, - {"name": "varlogetcd", - "mountPath": "/var/log/etcd{{ suffix }}.log", - "readOnly": false + { "name": "varlogetcd", + "mountPath": "/var/log/etcd{{ suffix }}.log", + "readOnly": false } - ] + ] } ], "volumes":[ diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index c90ce6f2398..0ce17a2654c 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -16,7 +16,9 @@ cluster/gce/configure-vm.sh: kubelet_api_servers: '${KUBELET_APISERVER}' cluster/gce/gci/configure-helper.sh: reconcile_cidr="false" cluster/gce/gci/configure-helper.sh: local api_servers="--master=https://${KUBERNETES_MASTER_NAME}" cluster/gce/gci/configure-helper.sh: local reconcile_cidr="true" +cluster/gce/gci/configure-helper.sh: sed -i -e "s@{{ *storage_backend *}}@${STORAGE_BACKEND:-}@g" "${temp_file}" cluster/gce/gci/configure-helper.sh: sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" +cluster/gce/trusty/configure-helper.sh: sed -i -e "s@{{ *storage_backend *}}@${STORAGE_BACKEND:-}@g" "${temp_file}" cluster/gce/trusty/configure-helper.sh: sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" cluster/gce/util.sh: local node_ip=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \ cluster/juju/layers/kubernetes/reactive/k8s.py: check_call(split(cmd.format(kubeconfig, cluster_name, server, ca))) @@ -37,10 +39,10 @@ cluster/photon-controller/util.sh: local cert_dir="/srv/kubernetes" cluster/photon-controller/util.sh: node_name=${1} cluster/rackspace/util.sh: local node_ip=$(nova show --minimal ${NODE_NAMES[$i]} \ cluster/saltbase/salt/cluster-autoscaler/cluster-autoscaler.manifest:{% set params = pillar['autoscaler_mig_config'] + " " + cloud_config -%} +cluster/saltbase/salt/etcd/etcd.manifest: "value": "{{ storage_backend }}" +cluster/saltbase/salt/etcd/etcd.manifest:{% set storage_backend = pillar.get('storage_backend', 'etcd2') -%} cluster/saltbase/salt/kube-admission-controls/init.sls:{% if 'LimitRanger' in pillar.get('admission_control', '') %} -cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set enable_garbage_collector = pillar['enable_garbage_collector'] -%} cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = address + " " + storage_backend + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + admission_control + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector -%} -cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set enable_garbage_collector = pillar['enable_garbage_collector'] -%} cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + service_cluster_ip_range + " " + terminated_pod_gc + " " + enable_garbage_collector + " " + cloud_provider + " " + cloud_config + " " + service_account_key + " " + log_level + " " + root_ca_file -%} cluster/saltbase/salt/kube-proxy/kube-proxy.manifest: {% set api_servers_with_port = api_servers + ":6443" -%} cluster/saltbase/salt/kube-proxy/kube-proxy.manifest: {% set api_servers_with_port = api_servers -%} @@ -80,8 +82,9 @@ hack/local-up-cluster.sh: advertise_address="--advertise_address=${API_HO hack/local-up-cluster.sh: runtime_config="--runtime-config=${RUNTIME_CONFIG}" hack/local-up-cluster.sh: advertise_address="" hack/local-up-cluster.sh: runtime_config="" -hack/test-update-storage-objects.sh: local storage_media_type=${2:-""} -hack/test-update-storage-objects.sh: local storage_versions=${1:-""} +hack/test-update-storage-objects.sh: local storage_backend=${1:-"${STORAGE_BACKEND_ETCD2}"} +hack/test-update-storage-objects.sh: local storage_media_type=${3:-""} +hack/test-update-storage-objects.sh: local storage_versions=${2:-""} hack/test-update-storage-objects.sh: source_file=${test_data[0]} hack/test-update-storage-objects.sh:# source_file,resource,namespace,name,old_version,new_version pkg/kubelet/api/v1alpha1/runtime/api.pb.go: ContainerPort *int32 `protobuf:"varint,3,opt,name=container_port,json=containerPort" json:"container_port,omitempty"` @@ -98,14 +101,13 @@ pkg/util/oom/oom_linux.go:// Writes 'value' to /proc//oom_score_adj for all pkg/util/oom/oom_linux.go:// Writes 'value' to /proc//oom_score_adj. PID = 0 means self test/e2e/common/configmap.go: Command: []string{"/mt", "--break_on_expected_content=false", "--retry_time=120", "--file_content_in_loop=/etc/configmap-volume/data-1"}, test/e2e/common/downwardapi_volume.go: Command: []string{"/mt", "--break_on_expected_content=false", "--retry_time=120", "--file_content_in_loop=" + filePath}, -test/e2e/es_cluster_logging.go: framework.Failf("No cluster_name field in Elasticsearch response: %v", esResponse) -test/e2e/es_cluster_logging.go: // Check to see if have a cluster_name field. -test/e2e/es_cluster_logging.go: clusterName, ok := esResponse["cluster_name"] test/e2e/common/host_path.go: fmt.Sprintf("--file_content_in_loop=%v", filePath), test/e2e/common/host_path.go: fmt.Sprintf("--file_content_in_loop=%v", filePathInReader), test/e2e/common/host_path.go: fmt.Sprintf("--retry_time=%d", retryDuration), test/e2e/common/host_path.go: fmt.Sprintf("--retry_time=%d", retryDuration), -test/e2e_node/configmap_test.go: Command: []string{"/mt", "--break_on_expected_content=false", "--retry_time=120", "--file_content_in_loop=/etc/configmap-volume/data-1"}, +test/e2e/es_cluster_logging.go: framework.Failf("No cluster_name field in Elasticsearch response: %v", esResponse) +test/e2e/es_cluster_logging.go: // Check to see if have a cluster_name field. +test/e2e/es_cluster_logging.go: clusterName, ok := esResponse["cluster_name"] test/images/mount-tester/mt.go: flag.BoolVar(&breakOnExpectedContent, "break_on_expected_content", true, "Break out of loop on expected content, (use with --file_content_in_loop flag only)") test/images/mount-tester/mt.go: flag.IntVar(&retryDuration, "retry_time", 180, "Retry time during the loop") test/images/mount-tester/mt.go: flag.StringVar(&readFileContentInLoopPath, "file_content_in_loop", "", "Path to read the file content in loop from")