From 793d3acd8b1e07b44c0992d15f6d6d0bd68f5a63 Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Tue, 23 Aug 2016 12:02:41 +0200 Subject: [PATCH] Support for TTLs in etcd migration. --- cluster/images/etcd/Dockerfile | 2 +- cluster/images/etcd/Makefile | 8 ++- .../images/etcd/attachlease/attachlease.go | 70 +++++++++++++++++++ cluster/images/etcd/migrate-if-needed.sh | 46 +++++++++++- hack/.linted_packages | 1 + hack/test-update-storage-objects.sh | 3 + hack/verify-flags/known-flags.txt | 3 + 7 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 cluster/images/etcd/attachlease/attachlease.go diff --git a/cluster/images/etcd/Dockerfile b/cluster/images/etcd/Dockerfile index ef2a8254d0f..a72c303edfa 100644 --- a/cluster/images/etcd/Dockerfile +++ b/cluster/images/etcd/Dockerfile @@ -16,4 +16,4 @@ FROM BASEIMAGE MAINTAINER Dawn Chen EXPOSE 2379 2380 4001 7001 -COPY etcd etcdctl migrate-if-needed.sh /usr/local/bin/ +COPY etcd etcdctl migrate-if-needed.sh attachlease /usr/local/bin/ diff --git a/cluster/images/etcd/Makefile b/cluster/images/etcd/Makefile index a423b605e56..09ec25fd7bb 100644 --- a/cluster/images/etcd/Makefile +++ b/cluster/images/etcd/Makefile @@ -38,8 +38,12 @@ ifeq ($(ARCH),ppc64le) endif build: - # Copy the content in this dir to the temp dir - cp ./* $(TEMP_DIR) + # Copy the content in this dir to the temp dir, + # without copying the subdirectories. + find ./ -maxdepth 1 -type f | xargs cp -t $(TEMP_DIR) + + make -C ../../../ WHAT=cluster/images/etcd/attachlease KUBE_STATIC_OVERRIDES="etcd"; + cp "../../../_output/bin/attachlease" $(TEMP_DIR) ifeq ($(ARCH),amd64) diff --git a/cluster/images/etcd/attachlease/attachlease.go b/cluster/images/etcd/attachlease/attachlease.go new file mode 100644 index 00000000000..1c7d6b81473 --- /dev/null +++ b/cluster/images/etcd/attachlease/attachlease.go @@ -0,0 +1,70 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "strings" + "time" + + "github.com/coreos/etcd/clientv3" + "github.com/golang/glog" + "golang.org/x/net/context" +) + +var ( + etcdAddress = flag.String("etcd-address", "", "Etcd address") + ttlKeysPrefix = flag.String("ttl-keys-prefix", "", "Prefix for TTL keys") + leaseDuration = flag.Duration("lease-duration", time.Hour, "Lease duration (seconds granularity)") +) + +func main() { + flag.Parse() + + if *etcdAddress == "" { + glog.Fatalf("--etcd-address flag is required") + } + client, err := clientv3.New(clientv3.Config{Endpoints: []string{*etcdAddress}}) + if err != nil { + glog.Fatalf("Error while creating etcd client: %v", err) + } + + // Make sure that ttlKeysPrefix is ended with "/" so that we only get children "directories". + if !strings.HasSuffix(*ttlKeysPrefix, "/") { + *ttlKeysPrefix += "/" + } + ctx := context.Background() + + objectsResp, err := client.KV.Get(ctx, *ttlKeysPrefix, clientv3.WithPrefix()) + if err != nil { + glog.Fatalf("Error while getting objects to attach to the lease") + } + + lease, err := client.Lease.Grant(ctx, int64(*leaseDuration/time.Second)) + if err != nil { + glog.Fatalf("Error while creating lease: %v", err) + } + glog.Infof("Lease with TTL: %v created", lease.TTL) + + glog.Infof("Attaching lease to %d entries", len(objectsResp.Kvs)) + for _, kv := range objectsResp.Kvs { + _, err := client.KV.Put(ctx, string(kv.Key), string(kv.Value), clientv3.WithLease(lease.ID)) + if err != nil { + glog.Errorf("Error while attaching lease to: %s", string(kv.Key)) + } + } +} diff --git a/cluster/images/etcd/migrate-if-needed.sh b/cluster/images/etcd/migrate-if-needed.sh index 36ad4190af1..4b4abe1701a 100755 --- a/cluster/images/etcd/migrate-if-needed.sh +++ b/cluster/images/etcd/migrate-if-needed.sh @@ -34,22 +34,62 @@ if [ -z "${DATA_DIRECTORY:-}" ]; then exit 0 fi +ETCD="${ETCD:-/usr/local/bin/etcd}" ETCDCTL="${ETCDCTL:-/usr/local/bin/etcdctl}" +ATTACHLEASE="${ATTACHLEASE:-/usr/local/bin/attachlease}" VERSION_FILE="version.txt" CURRENT_STORAGE='etcd2' if [ -e "${DATA_DIRECTORY}/${VERSION_FILE}" ]; then CURRENT_STORAGE="$(cat ${DATA_DIRECTORY}/${VERSION_FILE})" fi +start_etcd() { + ETCD_PORT=18629 + ETCD_PEER_PORT=18630 + ${ETCD} --data-dir=${DATA_DIRECTORY} \ + --listen-client-urls http://127.0.0.1:${ETCD_PORT} \ + --advertise-client-urls http://127.0.0.1:${ETCD_PORT} \ + --listen-peer-urls http://127.0.0.1:${ETCD_PEER_PORT} \ + --initial-advertise-peer-urls http://127.0.01:${ETCD_PEER_PORT} \ + 1>>/dev/null 2>&1 & + ETCD_PID=$! + # Wait until etcd is up. + for i in $(seq 30); do + local out + if out=$(wget -q --timeout=1 http://127.0.0.1:${ETCD_PORT}/v2/machines 2> /dev/null); then + echo "Etcd on port ${ETCD_PORT} is up." + return 0 + fi + sleep 0.5 + done + echo "Timeout while waiting for etcd on port ${ETCD_PORT}" + return 1 +} + +stop_etcd() { + kill "${ETCD_PID-}" >/dev/null 2>&1 || : + wait "${ETCD_PID-}" >/dev/null 2>&1 || : +} + if [ "${CURRENT_STORAGE}" = "etcd2" -a "${TARGET_STORAGE}" = "etcd3" ]; then # If directory doesn't exist or is empty, this means that there aren't any # data for migration, which means we can skip this step. if [ -d "${DATA_DIRECTORY}" ]; then if [ "$(ls -A ${DATA_DIRECTORY})" ]; then echo "Performing etcd2 -> etcd3 migration" - # TODO: Pass a correct transformer to handle TTLs. - echo "ETCDCTL_API=3 ${ETCDCTL} migrate --data-dir=${DATA_DIRECTORY}" ETCDCTL_API=3 ${ETCDCTL} migrate --data-dir=${DATA_DIRECTORY} + echo "Attaching leases to TTL entries" + # Now attach lease to all keys. + # To do it, we temporarily start etcd on a random port (so that + # apiserver actually cannot access it). + start_etcd + # Create a lease and attach all keys to it. + ${ATTACHLEASE} \ + --etcd-address http://127.0.0.1:${ETCD_PORT} \ + --ttl-keys-prefix "${TTL_KEYS_DIRECTORY:-/registry/events}" \ + --lease-duration 1h + # Kill etcd and wait until this is down. + stop_etcd fi fi fi @@ -58,6 +98,8 @@ if [ "${CURRENT_STORAGE}" = "etcd3" -a "${TARGET_STORAGE}" = "etcd2" ]; then echo "Performing etcd3 -> etcd2 migration" # TODO: Implement rollback once this will be supported. echo "etcd3 -> etcd2 downgrade is NOT supported." + # FIXME: On rollback, we will not support TTLs - we will simply clear + # all events. fi # Write current storage version to avoid future migrations. diff --git a/hack/.linted_packages b/hack/.linted_packages index 1345da825c3..2a607915fba 100644 --- a/hack/.linted_packages +++ b/hack/.linted_packages @@ -1,4 +1,5 @@ cluster/addons/fluentd-elasticsearch/es-image +cluster/images/etcd/attachlease cmd/gendocs cmd/genkubedocs cmd/genman diff --git a/hack/test-update-storage-objects.sh b/hack/test-update-storage-objects.sh index 25cb74d32c9..dbaf173fc66 100755 --- a/hack/test-update-storage-objects.sh +++ b/hack/test-update-storage-objects.sh @@ -100,6 +100,7 @@ function cleanup() { trap cleanup EXIT SIGINT make -C "${KUBE_ROOT}" WHAT=cmd/kube-apiserver +make -C "${KUBE_ROOT}" WHAT=cluster/images/etcd/attachlease kube::etcd::start @@ -159,7 +160,9 @@ killApiServer kube::etcd::stop TARGET_STORAGE="etcd3" \ DATA_DIRECTORY="${ETCD_DIR}" \ + ETCD=$(which etcd) \ ETCDCTL=$(which etcdctl) \ + ATTACHLEASE="${KUBE_OUTPUT_HOSTBIN}/attachlease" \ ${KUBE_ROOT}/cluster/images/etcd/migrate-if-needed.sh kube::etcd::start diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 87f8c817304..20bfa5069a6 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -140,6 +140,7 @@ enable-hostpath-provisioner enable-server enable-swagger-ui enable-garbage-collector +etcd-address etcd-cafile etcd-certfile etcd-config @@ -289,6 +290,7 @@ leader-elect leader-elect-lease-duration leader-elect-renew-deadline leader-elect-retry-period +lease-duration leave-stdin-open limit-bytes load-balancer-ip @@ -501,6 +503,7 @@ tls-cert-file tls-private-key-file to-version token-auth-file +ttl-keys-prefix ttl-secs type-src udp-port