diff --git a/test/e2e_node/BUILD b/test/e2e_node/BUILD index 18841759ef4..b8b5ef4f743 100644 --- a/test/e2e_node/BUILD +++ b/test/e2e_node/BUILD @@ -41,6 +41,7 @@ go_library( "//test/e2e/perftype:go_default_library", "//test/e2e_node/perftype:go_default_library", "//vendor/github.com/blang/semver:go_default_library", + "//vendor/github.com/coreos/go-systemd/util:go_default_library", "//vendor/github.com/docker/docker/client:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/client/v2:go_default_library", diff --git a/test/e2e_node/docker_test.go b/test/e2e_node/docker_test.go index 8c79604fb14..fa5ed68e621 100644 --- a/test/e2e_node/docker_test.go +++ b/test/e2e_node/docker_test.go @@ -17,11 +17,16 @@ limitations under the License. package e2e_node import ( + "fmt" + "strings" + "time" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/test/e2e/framework" . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) var _ = framework.KubeDescribe("Docker features [Feature:Docker]", func() { @@ -70,4 +75,103 @@ var _ = framework.KubeDescribe("Docker features [Feature:Docker]", func() { } }) }) + + Context("when live-restore is enabled [Serial] [Slow] [Disruptive]", func() { + It("containers should not be disrupted when the daemon shuts down and restarts", func() { + const ( + podName = "live-restore-test-pod" + containerName = "live-restore-test-container" + ) + + isSupported, err := isDockerLiveRestoreSupported() + framework.ExpectNoError(err) + if !isSupported { + framework.Skipf("Docker live-restore is not supported.") + } + isEnabled, err := isDockerLiveRestoreEnabled() + framework.ExpectNoError(err) + if !isEnabled { + framework.Skipf("Docker live-restore is not enabled.") + } + + By("Create the test pod.") + pod := f.PodClient().CreateSync(&v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: podName}, + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Name: containerName, + Image: "gcr.io/google_containers/nginx-slim:0.7", + }}, + }, + }) + + By("Ensure that the container is running before Docker is down.") + Eventually(func() bool { + return isContainerRunning(pod.Status.PodIP) + }).Should(BeTrue()) + + startTime1, err := getContainerStartTime(f, podName, containerName) + framework.ExpectNoError(err) + + By("Stop Docker daemon.") + framework.ExpectNoError(stopDockerDaemon()) + isDockerDown := true + defer func() { + if isDockerDown { + By("Start Docker daemon.") + framework.ExpectNoError(startDockerDaemon()) + } + }() + + By("Ensure that the container is running after Docker is down.") + Consistently(func() bool { + return isContainerRunning(pod.Status.PodIP) + }).Should(BeTrue()) + + By("Start Docker daemon.") + framework.ExpectNoError(startDockerDaemon()) + isDockerDown = false + + By("Ensure that the container is running after Docker has restarted.") + Consistently(func() bool { + return isContainerRunning(pod.Status.PodIP) + }).Should(BeTrue()) + + By("Ensure that the container has not been restarted after Docker is restarted.") + Consistently(func() bool { + startTime2, err := getContainerStartTime(f, podName, containerName) + framework.ExpectNoError(err) + return startTime1 == startTime2 + }, 3*time.Second, time.Second).Should(BeTrue()) + }) + }) }) + +// isContainerRunning returns true if the container is running by checking +// whether the server is responding, and false otherwise. +func isContainerRunning(podIP string) bool { + output, err := runCommand("curl", podIP) + if err != nil { + return false + } + return strings.Contains(output, "Welcome to nginx!") +} + +// getContainerStartTime returns the start time of the container with the +// containerName of the pod having the podName. +func getContainerStartTime(f *framework.Framework, podName, containerName string) (time.Time, error) { + pod, err := f.PodClient().Get(podName, metav1.GetOptions{}) + if err != nil { + return time.Time{}, fmt.Errorf("failed to get pod %q: %v", podName, err) + } + for _, status := range pod.Status.ContainerStatuses { + if status.Name != containerName { + continue + } + if status.State.Running == nil { + return time.Time{}, fmt.Errorf("%v/%v is not running", podName, containerName) + } + return status.State.Running.StartedAt.Time, nil + } + return time.Time{}, fmt.Errorf("failed to find %v/%v", podName, containerName) +} diff --git a/test/e2e_node/docker_util.go b/test/e2e_node/docker_util.go index b8bc2497083..625733daf5a 100644 --- a/test/e2e_node/docker_util.go +++ b/test/e2e_node/docker_util.go @@ -21,11 +21,13 @@ import ( "fmt" "github.com/blang/semver" + systemdutil "github.com/coreos/go-systemd/util" "github.com/docker/docker/client" ) const ( - defaultDockerEndpoint = "unix:///var/run/docker.sock" + defaultDockerEndpoint = "unix:///var/run/docker.sock" + dockerDaemonConfigName = "/etc/docker/daemon.json" ) // getDockerAPIVersion returns the Docker's API version. @@ -36,7 +38,7 @@ func getDockerAPIVersion() (semver.Version, error) { } version, err := c.ServerVersion(context.Background()) if err != nil { - return semver.Version{}, fmt.Errorf("failed to get docker info: %v", err) + return semver.Version{}, fmt.Errorf("failed to get docker server version: %v", err) } return semver.MustParse(version.APIVersion + ".0"), nil } @@ -60,3 +62,51 @@ func isDockerNoNewPrivilegesSupported() (bool, error) { } return version.GTE(semver.MustParse("1.23.0")), nil } + +// isDockerLiveRestoreSupported returns true if live-restore is supported in +// the current Docker version. +func isDockerLiveRestoreSupported() (bool, error) { + version, err := getDockerAPIVersion() + if err != nil { + return false, err + } + return version.GTE(semver.MustParse("1.26.0")), nil +} + +// isDockerLiveRestoreEnabled returns true if live-restore is enabled in the +// Docker. +func isDockerLiveRestoreEnabled() (bool, error) { + c, err := client.NewClient(defaultDockerEndpoint, "", nil, nil) + if err != nil { + return false, fmt.Errorf("failed to create docker client: %v", err) + } + info, err := c.Info(context.Background()) + if err != nil { + return false, fmt.Errorf("failed to get docker info: %v", err) + } + return info.LiveRestoreEnabled, nil +} + +// stopDockerDaemon starts the Docker daemon. +func startDockerDaemon() error { + switch { + case systemdutil.IsRunningSystemd(): + _, err := runCommand("systemctl", "start", "docker") + return err + default: + _, err := runCommand("service", "docker", "start") + return err + } +} + +// stopDockerDaemon stops the Docker daemon. +func stopDockerDaemon() error { + switch { + case systemdutil.IsRunningSystemd(): + _, err := runCommand("systemctl", "stop", "docker") + return err + default: + _, err := runCommand("service", "docker", "stop") + return err + } +} diff --git a/test/e2e_node/gke_environment_test.go b/test/e2e_node/gke_environment_test.go index 133f7647c11..1b48614a628 100644 --- a/test/e2e_node/gke_environment_test.go +++ b/test/e2e_node/gke_environment_test.go @@ -342,16 +342,6 @@ var _ = framework.KubeDescribe("GKE system requirements [Conformance] [Feature:G }) }) -// runCommand runs the cmd and returns the combined stdout and stderr, or an -// error if the command failed. -func runCommand(cmd ...string) (string, error) { - output, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput() - if err != nil { - return "", fmt.Errorf("failed to run %q: %s (%s)", strings.Join(cmd, " "), err, output) - } - return string(output), nil -} - // getPPID returns the PPID for the pid. func getPPID(pid int) (int, error) { statusFile := "/proc/" + strconv.Itoa(pid) + "/status" diff --git a/test/e2e_node/jenkins/cos-init-live-restore.yaml b/test/e2e_node/jenkins/cos-init-live-restore.yaml new file mode 100644 index 00000000000..ccc5688679c --- /dev/null +++ b/test/e2e_node/jenkins/cos-init-live-restore.yaml @@ -0,0 +1,19 @@ +#cloud-config + +runcmd: + - echo '{"live-restore":true}' > /etc/docker/daemon.json + - systemctl restart docker + - mount /tmp /tmp -o remount,exec,suid + - usermod -a -G docker jenkins + - mkdir -p /var/lib/kubelet + - mkdir -p /home/kubernetes/containerized_mounter/rootfs + - mount --bind /home/kubernetes/containerized_mounter/ /home/kubernetes/containerized_mounter/ + - mount -o remount, exec /home/kubernetes/containerized_mounter/ + - wget https://storage.googleapis.com/kubernetes-release/gci-mounter/mounter.tar -O /tmp/mounter.tar + - tar xvf /tmp/mounter.tar -C /home/kubernetes/containerized_mounter/rootfs + - mkdir -p /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet + - mount --rbind /var/lib/kubelet /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet + - mount --make-rshared /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet + - mount --bind /proc /home/kubernetes/containerized_mounter/rootfs/proc + - mount --bind /dev /home/kubernetes/containerized_mounter/rootfs/dev + - rm /tmp/mounter.tar diff --git a/test/e2e_node/jenkins/image-config.yaml b/test/e2e_node/jenkins/image-config.yaml index cea41804203..471212d8047 100644 --- a/test/e2e_node/jenkins/image-config.yaml +++ b/test/e2e_node/jenkins/image-config.yaml @@ -19,4 +19,4 @@ images: cos-beta: image_regex: cos-beta-60-9592-70-0 # docker 1.13.1 project: cos-cloud - metadata: "user-data