Merge pull request #15428 from mikedanese/gc

Add e2e test for pod garbage collection

cluster/gce/config-test.sh

@@ -47,6 +47,7 @@ MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
 MINION_SCOPES="${MINION_SCOPES:-compute-rw,monitoring,logging-write,storage-ro}"
 RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}"
 ENABLE_EXPERIMENTAL_API="${KUBE_ENABLE_EXPERIMENTAL_API:-false}"
+TERMINATED_POD_GC_THRESHOLD=${TERMINATED_POD_GC_THRESHOLD:-100}
 
 # Increase the sleep interval value if concerned about API rate limits. 3, in seconds, is the default.
 POLL_SLEEP_INTERVAL=3

cluster/gce/configure-vm.sh

@@ -322,6 +322,11 @@ EOF
     if [ -n "${ENABLE_EXPERIMENTAL_API:-}" ]; then
       cat <<EOF >>/srv/salt-overlay/pillar/cluster-params.sls
 enable_experimental_api: '$(echo "$ENABLE_EXPERIMENTAL_API" | sed -e "s/'/''/g")'
+EOF
+    fi
+    if [ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]; then
+      cat <<EOF >>/srv/salt-overlay/pillar/cluster-params.sls
+terminated_pod_gc_threshold: '$(echo "${TERMINATED_POD_GC_THRESHOLD}" | sed -e "s/'/''/g")'
 EOF
     fi
 }

cluster/gce/debian/helper.sh

@@ -64,6 +64,11 @@ EOF
   if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then
     cat >>$file <<EOF
 KUBE_APISERVER_REQUEST_TIMEOUT: $(yaml-quote ${KUBE_APISERVER_REQUEST_TIMEOUT})
+EOF
+  fi
+  if [ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]; then
+    cat >>$file <<EOF
+TERMINATED_POD_GC_THRESHOLD: $(yaml-quote ${TERMINATED_POD_GC_THRESHOLD})
 EOF
   fi
   if [ -n "${TEST_CLUSTER:-}" ]; then

cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest

@@ -1,6 +1,8 @@
 {% set cluster_name = "" -%}
 {% set cluster_cidr = "" -%}
 {% set allocate_node_cidrs = "" -%}
+{% set terminated_pod_gc = "" -%}
+
 
 {% if pillar['instance_prefix'] is defined -%}
   {% set cluster_name = "--cluster-name=" + pillar['instance_prefix'] -%}
@@ -11,6 +13,9 @@
 {% if pillar['allocate_node_cidrs'] is defined -%}
   {% set allocate_node_cidrs = "--allocate-node-cidrs=" + pillar['allocate_node_cidrs'] -%}
 {% endif -%}
+{% if pillar['terminated_pod_gc_threshold'] is defined -%}
+  {% set terminated_pod_gc = "--terminated-pod-gc-threshold=" + pillar['terminated_pod_gc_threshold'] -%}
+{% endif -%}
 
 {% set cloud_provider = "" -%}
 {% set cloud_config = "" -%}
@@ -34,7 +39,8 @@
    {% set root_ca_file = "--root-ca-file=/srv/kubernetes/ca.crt" -%}
 {% endif -%}
 
-{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + cloud_provider  + " " + cloud_config + service_account_key + pillar['log_level'] + " " + root_ca_file -%}
+{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + terminated_pod_gc + " " + cloud_provider  + " " + cloud_config + service_account_key + pillar['log_level'] + " " + root_ca_file -%}
+
 
 # test_args has to be kept at the end, so they'll overwrite any prior configuration
 {% if pillar['controller_manager_test_args'] is defined -%}

hack/jenkins/e2e.sh

@@ -158,6 +158,7 @@ GCE_SLOW_TESTS=(
     "Nodes\sResize"                                   # 3 min 30 sec, file: resize_nodes.go,         issue: #13323
     "resource\susage\stracking"                       # 1 hour,       file: kubelet_perf.go,         slow by design
     "monotonically\sincreasing\srestart\scount"       # 1.5 to 5 min, file: pods.go,                 slow by design
+    "Garbage\scollector\sshould"                      # 7 min,        file: garbage_collector.go,    slow by design
     "KubeProxy\sshould\stest\skube-proxy"             # 9 min 30 sec, file: kubeproxy.go,            issue: #14204
     )
 

test/e2e/garbage_collector.go

@@ -0,0 +1,82 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/fields"
+	"k8s.io/kubernetes/pkg/labels"
+	"k8s.io/kubernetes/pkg/util"
+)
+
+// This test requires that --terminated-pod-gc-threshold=100 be set on the controller manager
+var _ = Describe("Garbage collector", func() {
+	f := NewFramework("garbage-collector")
+	It("should handle the creation of 1000 pods", func() {
+		SkipUnlessProviderIs("gce")
+
+		var count int
+		for count < 1000 {
+			pod, err := createTerminatingPod(f)
+			pod.ResourceVersion = ""
+			pod.Status.Phase = api.PodFailed
+			pod, err = f.Client.Pods(f.Namespace.Name).UpdateStatus(pod)
+			if err != nil {
+				Failf("err failing pod: %v", err)
+			}
+
+			count++
+			if count%50 == 0 {
+				Logf("count: %v", count)
+			}
+		}
+
+		Logf("created: %v", count)
+		// This sleep has to be longer than the gcCheckPeriod defined
+		// in pkg/controller/gc/gc_controller.go which is currently
+		// 20 seconds.
+		time.Sleep(30 * time.Second)
+
+		pods, err := f.Client.Pods(f.Namespace.Name).List(labels.Everything(), fields.Everything())
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(pods.Items)).To(BeNumerically("==", 100))
+	})
+})
+
+func createTerminatingPod(f *Framework) (*api.Pod, error) {
+	uuid := util.NewUUID()
+	pod := &api.Pod{
+		ObjectMeta: api.ObjectMeta{
+			Name: string(uuid),
+		},
+		Spec: api.PodSpec{
+			NodeName: "nonexistant-node",
+			Containers: []api.Container{
+				{
+					Name:  string(uuid),
+					Image: "beta.gcr.io/google_containers/busybox",
+				},
+			},
+		},
+	}
+	return f.Client.Pods(f.Namespace.Name).Create(pod)
+}