Merge pull request #31242 from wojtek-t/attach_ttls

Automatic merge from submit-queue

Support for TTLs in etcd migration.

Ref #20504

cluster/images/etcd/Dockerfile

@@ -16,4 +16,4 @@ FROM BASEIMAGE
 MAINTAINER Dawn Chen <dawnchen@google.com>
 
 EXPOSE 2379 2380 4001 7001
-COPY etcd etcdctl migrate-if-needed.sh /usr/local/bin/
+COPY etcd etcdctl migrate-if-needed.sh attachlease /usr/local/bin/

cluster/images/etcd/Makefile

@@ -38,8 +38,12 @@ ifeq ($(ARCH),ppc64le)
 endif
 
 build:
-	# Copy the content in this dir to the temp dir
+	# Copy the content in this dir to the temp dir,
-	cp ./* $(TEMP_DIR)
+	# without copying the subdirectories.
+	find ./ -maxdepth 1 -type f | xargs cp -t $(TEMP_DIR)
+
+	make -C ../../../ WHAT=cluster/images/etcd/attachlease KUBE_STATIC_OVERRIDES="etcd";
+	cp "../../../_output/bin/attachlease" $(TEMP_DIR)
 
 ifeq ($(ARCH),amd64)
 

cluster/images/etcd/attachlease/attachlease.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"strings"
+	"time"
+
+	"github.com/coreos/etcd/clientv3"
+	"github.com/golang/glog"
+	"golang.org/x/net/context"
+)
+
+var (
+	etcdAddress   = flag.String("etcd-address", "", "Etcd address")
+	ttlKeysPrefix = flag.String("ttl-keys-prefix", "", "Prefix for TTL keys")
+	leaseDuration = flag.Duration("lease-duration", time.Hour, "Lease duration (seconds granularity)")
+)
+
+func main() {
+	flag.Parse()
+
+	if *etcdAddress == "" {
+		glog.Fatalf("--etcd-address flag is required")
+	}
+	client, err := clientv3.New(clientv3.Config{Endpoints: []string{*etcdAddress}})
+	if err != nil {
+		glog.Fatalf("Error while creating etcd client: %v", err)
+	}
+
+	// Make sure that ttlKeysPrefix is ended with "/" so that we only get children "directories".
+	if !strings.HasSuffix(*ttlKeysPrefix, "/") {
+		*ttlKeysPrefix += "/"
+	}
+	ctx := context.Background()
+
+	objectsResp, err := client.KV.Get(ctx, *ttlKeysPrefix, clientv3.WithPrefix())
+	if err != nil {
+		glog.Fatalf("Error while getting objects to attach to the lease")
+	}
+
+	lease, err := client.Lease.Grant(ctx, int64(*leaseDuration/time.Second))
+	if err != nil {
+		glog.Fatalf("Error while creating lease: %v", err)
+	}
+	glog.Infof("Lease with TTL: %v created", lease.TTL)
+
+	glog.Infof("Attaching lease to %d entries", len(objectsResp.Kvs))
+	for _, kv := range objectsResp.Kvs {
+		_, err := client.KV.Put(ctx, string(kv.Key), string(kv.Value), clientv3.WithLease(lease.ID))
+		if err != nil {
+			glog.Errorf("Error while attaching lease to: %s", string(kv.Key))
+		}
+	}
+}

cluster/images/etcd/migrate-if-needed.sh

@@ -34,22 +34,62 @@ if [ -z "${DATA_DIRECTORY:-}" ]; then
   exit 0
 fi
 
+ETCD="${ETCD:-/usr/local/bin/etcd}"
 ETCDCTL="${ETCDCTL:-/usr/local/bin/etcdctl}"
+ATTACHLEASE="${ATTACHLEASE:-/usr/local/bin/attachlease}"
 VERSION_FILE="version.txt"
 CURRENT_STORAGE='etcd2'
 if [ -e "${DATA_DIRECTORY}/${VERSION_FILE}" ]; then
   CURRENT_STORAGE="$(cat ${DATA_DIRECTORY}/${VERSION_FILE})"
 fi
 
+start_etcd() {
+  ETCD_PORT=18629
+  ETCD_PEER_PORT=18630
+  ${ETCD} --data-dir=${DATA_DIRECTORY} \
+    --listen-client-urls http://127.0.0.1:${ETCD_PORT} \
+    --advertise-client-urls http://127.0.0.1:${ETCD_PORT} \
+    --listen-peer-urls http://127.0.0.1:${ETCD_PEER_PORT} \
+    --initial-advertise-peer-urls http://127.0.01:${ETCD_PEER_PORT} \
+    1>>/dev/null 2>&1 &
+  ETCD_PID=$!
+  # Wait until etcd is up.
+  for i in $(seq 30); do
+    local out
+    if out=$(wget -q --timeout=1 http://127.0.0.1:${ETCD_PORT}/v2/machines 2> /dev/null); then
+      echo "Etcd on port ${ETCD_PORT} is up."
+      return 0
+    fi
+    sleep 0.5
+  done
+  echo "Timeout while waiting for etcd on port ${ETCD_PORT}"
+  return 1
+}
+
+stop_etcd() {
+  kill "${ETCD_PID-}" >/dev/null 2>&1 || :
+  wait "${ETCD_PID-}" >/dev/null 2>&1 || :
+}
+
 if [ "${CURRENT_STORAGE}" = "etcd2" -a "${TARGET_STORAGE}" = "etcd3" ]; then
   # If directory doesn't exist or is empty, this means that there aren't any
   # data for migration, which means we can skip this step.
   if [ -d "${DATA_DIRECTORY}" ]; then
     if [ "$(ls -A ${DATA_DIRECTORY})" ]; then
       echo "Performing etcd2 -> etcd3 migration"
-      # TODO: Pass a correct transformer to handle TTLs.
-      echo "ETCDCTL_API=3 ${ETCDCTL} migrate --data-dir=${DATA_DIRECTORY}"
       ETCDCTL_API=3 ${ETCDCTL} migrate --data-dir=${DATA_DIRECTORY}
+      echo "Attaching leases to TTL entries"
+      # Now attach lease to all keys.
+      # To do it, we temporarily start etcd on a random port (so that
+      # apiserver actually cannot access it).
+      start_etcd
+      # Create a lease and attach all keys to it.
+      ${ATTACHLEASE} \
+        --etcd-address http://127.0.0.1:${ETCD_PORT} \
+        --ttl-keys-prefix "${TTL_KEYS_DIRECTORY:-/registry/events}" \
+        --lease-duration 1h
+      # Kill etcd and wait until this is down.
+      stop_etcd
     fi
   fi
 fi
@@ -58,6 +98,8 @@ if [ "${CURRENT_STORAGE}" = "etcd3" -a "${TARGET_STORAGE}" = "etcd2" ]; then
   echo "Performing etcd3 -> etcd2 migration"
   # TODO: Implement rollback once this will be supported.
   echo "etcd3 -> etcd2 downgrade is NOT supported."
+  # FIXME: On rollback, we will not support TTLs - we will simply clear
+  # all events.
 fi
 
 # Write current storage version to avoid future migrations.

hack/.linted_packages

@@ -1,4 +1,5 @@
 cluster/addons/fluentd-elasticsearch/es-image
+cluster/images/etcd/attachlease
 cmd/gendocs
 cmd/genkubedocs
 cmd/genman

hack/test-update-storage-objects.sh

@@ -100,6 +100,7 @@ function cleanup() {
 trap cleanup EXIT SIGINT
 
 make -C "${KUBE_ROOT}" WHAT=cmd/kube-apiserver
+make -C "${KUBE_ROOT}" WHAT=cluster/images/etcd/attachlease
 
 kube::etcd::start
 
@@ -159,7 +160,9 @@ killApiServer
 kube::etcd::stop
 TARGET_STORAGE="etcd3" \
   DATA_DIRECTORY="${ETCD_DIR}" \
+  ETCD=$(which etcd) \
   ETCDCTL=$(which etcdctl) \
+  ATTACHLEASE="${KUBE_OUTPUT_HOSTBIN}/attachlease" \
   ${KUBE_ROOT}/cluster/images/etcd/migrate-if-needed.sh
 kube::etcd::start
 

hack/verify-flags/known-flags.txt

@@ -140,6 +140,7 @@ enable-hostpath-provisioner
 enable-server
 enable-swagger-ui
 enable-garbage-collector
+etcd-address
 etcd-cafile
 etcd-certfile
 etcd-config
@@ -289,6 +290,7 @@ leader-elect
 leader-elect-lease-duration
 leader-elect-renew-deadline
 leader-elect-retry-period
+lease-duration
 leave-stdin-open
 limit-bytes
 load-balancer-ip
@@ -501,6 +503,7 @@ tls-cert-file
 tls-private-key-file
 to-version
 token-auth-file
+ttl-keys-prefix
 ttl-secs
 type-src
 udp-port