mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-09-17 15:13:08 +00:00
Clean up dockershim in tests
Signed-off-by: Ciprian Hacman <ciprian@hakman.dev>
This commit is contained in:
@@ -24,9 +24,7 @@ import (
|
||||
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
|
||||
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
||||
kubelogs "k8s.io/kubernetes/pkg/kubelet/logs"
|
||||
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
|
||||
|
||||
"github.com/onsi/ginkgo"
|
||||
"github.com/onsi/gomega"
|
||||
@@ -43,12 +41,6 @@ const (
|
||||
var _ = SIGDescribe("ContainerLogRotation [Slow] [Serial] [Disruptive]", func() {
|
||||
f := framework.NewDefaultFramework("container-log-rotation-test")
|
||||
ginkgo.Context("when a container generates a lot of log", func() {
|
||||
ginkgo.BeforeEach(func() {
|
||||
if framework.TestContext.ContainerRuntime != kubetypes.RemoteContainerRuntime {
|
||||
e2eskipper.Skipf("Skipping ContainerLogRotation test since the container runtime is not remote")
|
||||
}
|
||||
})
|
||||
|
||||
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
|
||||
initialConfig.ContainerLogMaxFiles = testContainerLogMaxFiles
|
||||
initialConfig.ContainerLogMaxSize = testContainerLogMaxSize
|
||||
|
@@ -1,140 +0,0 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2enode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
|
||||
imageutils "k8s.io/kubernetes/test/utils/image"
|
||||
|
||||
"github.com/onsi/ginkgo"
|
||||
"github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = SIGDescribe("Docker features [Feature:Docker][Legacy:Docker]", func() {
|
||||
f := framework.NewDefaultFramework("docker-feature-test")
|
||||
|
||||
ginkgo.BeforeEach(func() {
|
||||
e2eskipper.RunIfContainerRuntimeIs("docker")
|
||||
})
|
||||
|
||||
ginkgo.Context("when live-restore is enabled [Serial] [Slow] [Disruptive]", func() {
|
||||
ginkgo.It("containers should not be disrupted when the daemon shuts down and restarts", func() {
|
||||
const (
|
||||
podName = "live-restore-test-pod"
|
||||
containerName = "live-restore-test-container"
|
||||
)
|
||||
|
||||
isSupported, err := isDockerLiveRestoreSupported()
|
||||
framework.ExpectNoError(err)
|
||||
if !isSupported {
|
||||
e2eskipper.Skipf("Docker live-restore is not supported.")
|
||||
}
|
||||
isEnabled, err := isDockerLiveRestoreEnabled()
|
||||
framework.ExpectNoError(err)
|
||||
if !isEnabled {
|
||||
e2eskipper.Skipf("Docker live-restore is not enabled.")
|
||||
}
|
||||
|
||||
ginkgo.By("Create the test pod.")
|
||||
pod := f.PodClient().CreateSync(&v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: podName},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{{
|
||||
Name: containerName,
|
||||
Image: imageutils.GetE2EImage(imageutils.Nginx),
|
||||
}},
|
||||
},
|
||||
})
|
||||
|
||||
ginkgo.By("Ensure that the container is running before Docker is down.")
|
||||
gomega.Eventually(func() bool {
|
||||
return isContainerRunning(pod.Status.PodIP)
|
||||
}).Should(gomega.BeTrue())
|
||||
|
||||
startTime1, err := getContainerStartTime(f, podName, containerName)
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
ginkgo.By("Stop Docker daemon.")
|
||||
framework.ExpectNoError(stopDockerDaemon())
|
||||
isDockerDown := true
|
||||
defer func() {
|
||||
if isDockerDown {
|
||||
ginkgo.By("Start Docker daemon.")
|
||||
framework.ExpectNoError(startDockerDaemon())
|
||||
}
|
||||
}()
|
||||
|
||||
ginkgo.By("Ensure that the container is running after Docker is down.")
|
||||
gomega.Consistently(func() bool {
|
||||
return isContainerRunning(pod.Status.PodIP)
|
||||
}).Should(gomega.BeTrue())
|
||||
|
||||
ginkgo.By("Start Docker daemon.")
|
||||
framework.ExpectNoError(startDockerDaemon())
|
||||
isDockerDown = false
|
||||
|
||||
ginkgo.By("Ensure that the container is running after Docker has restarted.")
|
||||
gomega.Consistently(func() bool {
|
||||
return isContainerRunning(pod.Status.PodIP)
|
||||
}).Should(gomega.BeTrue())
|
||||
|
||||
ginkgo.By("Ensure that the container has not been restarted after Docker is restarted.")
|
||||
gomega.Consistently(func() bool {
|
||||
startTime2, err := getContainerStartTime(f, podName, containerName)
|
||||
framework.ExpectNoError(err)
|
||||
return startTime1 == startTime2
|
||||
}, 3*time.Second, time.Second).Should(gomega.BeTrue())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
// isContainerRunning returns true if the container is running by checking
|
||||
// whether the server is responding, and false otherwise.
|
||||
func isContainerRunning(podIP string) bool {
|
||||
output, err := runCommand("curl", podIP)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(output, "Welcome to nginx!")
|
||||
}
|
||||
|
||||
// getContainerStartTime returns the start time of the container with the
|
||||
// containerName of the pod having the podName.
|
||||
func getContainerStartTime(f *framework.Framework, podName, containerName string) (time.Time, error) {
|
||||
pod, err := f.PodClient().Get(context.TODO(), podName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("failed to get pod %q: %v", podName, err)
|
||||
}
|
||||
for _, status := range pod.Status.ContainerStatuses {
|
||||
if status.Name != containerName {
|
||||
continue
|
||||
}
|
||||
if status.State.Running == nil {
|
||||
return time.Time{}, fmt.Errorf("%v/%v is not running", podName, containerName)
|
||||
}
|
||||
return status.State.Running.StartedAt.Time, nil
|
||||
}
|
||||
return time.Time{}, fmt.Errorf("failed to find %v/%v", podName, containerName)
|
||||
}
|
@@ -1,116 +0,0 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2enode
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/blang/semver"
|
||||
systemdutil "github.com/coreos/go-systemd/v22/util"
|
||||
)
|
||||
|
||||
// getDockerAPIVersion returns the Docker's API version.
|
||||
func getDockerAPIVersion() (semver.Version, error) {
|
||||
output, err := runCommand("docker", "version", "-f", "{{.Server.APIVersion}}")
|
||||
if err != nil {
|
||||
return semver.Version{}, fmt.Errorf("failed to get docker server version: %v", err)
|
||||
}
|
||||
return semver.MustParse(strings.TrimSpace(output) + ".0"), nil
|
||||
}
|
||||
|
||||
// isSharedPIDNamespaceSupported returns true if the Docker version is 1.13.1+
|
||||
// (API version 1.26+), and false otherwise.
|
||||
func isSharedPIDNamespaceSupported() (bool, error) {
|
||||
version, err := getDockerAPIVersion()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return version.GTE(semver.MustParse("1.26.0")), nil
|
||||
}
|
||||
|
||||
// isDockerLiveRestoreSupported returns true if live-restore is supported in
|
||||
// the current Docker version.
|
||||
func isDockerLiveRestoreSupported() (bool, error) {
|
||||
version, err := getDockerAPIVersion()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return version.GTE(semver.MustParse("1.26.0")), nil
|
||||
}
|
||||
|
||||
// getDockerInfo returns the Info struct for the running Docker daemon.
|
||||
func getDockerInfo(key string) (string, error) {
|
||||
output, err := runCommand("docker", "info", "-f", "{{."+key+"}}")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get docker info: %v", err)
|
||||
}
|
||||
return strings.TrimSpace(output), nil
|
||||
}
|
||||
|
||||
// isDockerLiveRestoreEnabled returns true if live-restore is enabled in the
|
||||
// Docker.
|
||||
func isDockerLiveRestoreEnabled() (bool, error) {
|
||||
info, err := getDockerInfo("LiveRestoreEnabled")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return info == "true", nil
|
||||
}
|
||||
|
||||
// getDockerLoggingDriver returns the name of the logging driver.
|
||||
func getDockerLoggingDriver() (string, error) {
|
||||
info, err := getDockerInfo("LoggingDriver")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// isDockerSELinuxSupportEnabled checks whether the Docker daemon was started
|
||||
// with SELinux support enabled.
|
||||
func isDockerSELinuxSupportEnabled() (bool, error) {
|
||||
info, err := getDockerInfo("SecurityOptions")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return strings.Contains(info, "name=selinux"), nil
|
||||
}
|
||||
|
||||
// startDockerDaemon starts the Docker daemon.
|
||||
func startDockerDaemon() error {
|
||||
switch {
|
||||
case systemdutil.IsRunningSystemd():
|
||||
_, err := runCommand("systemctl", "start", "docker")
|
||||
return err
|
||||
default:
|
||||
_, err := runCommand("service", "docker", "start")
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// stopDockerDaemon stops the Docker daemon.
|
||||
func stopDockerDaemon() error {
|
||||
switch {
|
||||
case systemdutil.IsRunningSystemd():
|
||||
_, err := runCommand("systemctl", "stop", "docker")
|
||||
return err
|
||||
default:
|
||||
_, err := runCommand("service", "docker", "stop")
|
||||
return err
|
||||
}
|
||||
}
|
@@ -1,231 +0,0 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2enode
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/uuid"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
|
||||
imageutils "k8s.io/kubernetes/test/utils/image"
|
||||
|
||||
"github.com/onsi/ginkgo"
|
||||
"github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
const (
|
||||
testCheckpoint = "checkpoint-test"
|
||||
// Container GC Period is 1 minute
|
||||
gcTimeout = 3 * time.Minute
|
||||
testCheckpointContent = `{"version":"v1","name":"fluentd-gcp-v2.0-vmnqx","namespace":"kube-system","data":{},"checksum":1799154314}`
|
||||
)
|
||||
|
||||
var _ = SIGDescribe("Dockershim [Serial] [Disruptive] [Feature:Docker][Legacy:Docker]", func() {
|
||||
f := framework.NewDefaultFramework("dockerhism-checkpoint-test")
|
||||
|
||||
ginkgo.BeforeEach(func() {
|
||||
e2eskipper.RunIfContainerRuntimeIs("docker")
|
||||
})
|
||||
|
||||
ginkgo.It("should clean up pod sandbox checkpoint after pod deletion", func() {
|
||||
podName := "pod-checkpoint-no-disrupt"
|
||||
runPodCheckpointTest(f, podName, func() {
|
||||
checkpoints := findCheckpoints(podName)
|
||||
if len(checkpoints) == 0 {
|
||||
framework.Failf("No checkpoint for the pod was found")
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
ginkgo.It("should remove dangling checkpoint file", func() {
|
||||
filename := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%s/%s", testCheckpoint, f.Namespace.Name))))
|
||||
fullpath := path.Join(framework.TestContext.DockershimCheckpointDir, filename)
|
||||
|
||||
ginkgo.By(fmt.Sprintf("Write a file at %q", fullpath))
|
||||
err := writeFileAndSync(fullpath, []byte(testCheckpointContent))
|
||||
framework.ExpectNoError(err, "Failed to create file %q", fullpath)
|
||||
|
||||
ginkgo.By("Check if file is removed")
|
||||
gomega.Eventually(func() bool {
|
||||
if _, err := os.Stat(fullpath); os.IsNotExist(err) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}, gcTimeout, 10*time.Second).Should(gomega.BeTrue())
|
||||
|
||||
})
|
||||
|
||||
ginkgo.Context("When pod sandbox checkpoint is missing", func() {
|
||||
ginkgo.It("should complete pod sandbox clean up", func() {
|
||||
podName := "pod-checkpoint-missing"
|
||||
runPodCheckpointTest(f, podName, func() {
|
||||
checkpoints := findCheckpoints(podName)
|
||||
if len(checkpoints) == 0 {
|
||||
framework.Failf("No checkpoint for the pod was found")
|
||||
}
|
||||
ginkgo.By("Removing checkpoint of test pod")
|
||||
for _, filename := range checkpoints {
|
||||
if len(filename) == 0 {
|
||||
continue
|
||||
}
|
||||
framework.Logf("Removing checkpoint %q", filename)
|
||||
_, err := exec.Command("sudo", "rm", filename).CombinedOutput()
|
||||
framework.ExpectNoError(err, "Failed to remove checkpoint file %q: %v", string(filename), err)
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
ginkgo.Context("When all containers in pod are missing", func() {
|
||||
ginkgo.It("should complete pod sandbox clean up based on the information in sandbox checkpoint", func() {
|
||||
runPodCheckpointTest(f, "pod-containers-missing", func() {
|
||||
ginkgo.By("Gathering pod container ids")
|
||||
stdout, err := exec.Command("sudo", "docker", "ps", "-q", "-f",
|
||||
fmt.Sprintf("name=%s", f.Namespace.Name)).CombinedOutput()
|
||||
framework.ExpectNoError(err, "Failed to run docker ps: %v", err)
|
||||
lines := strings.Split(string(stdout), "\n")
|
||||
ids := []string{}
|
||||
for _, id := range lines {
|
||||
id = cleanString(id)
|
||||
if len(id) > 0 {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
|
||||
ginkgo.By("Stop and remove pod containers")
|
||||
dockerStopCmd := append([]string{"docker", "stop"}, ids...)
|
||||
_, err = exec.Command("sudo", dockerStopCmd...).CombinedOutput()
|
||||
framework.ExpectNoError(err, "Failed to run command %v: %v", dockerStopCmd, err)
|
||||
dockerRmCmd := append([]string{"docker", "rm"}, ids...)
|
||||
_, err = exec.Command("sudo", dockerRmCmd...).CombinedOutput()
|
||||
framework.ExpectNoError(err, "Failed to run command %v: %v", dockerRmCmd, err)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
ginkgo.Context("When checkpoint file is corrupted", func() {
|
||||
ginkgo.It("should complete pod sandbox clean up", func() {
|
||||
podName := "pod-checkpoint-corrupted"
|
||||
runPodCheckpointTest(f, podName, func() {
|
||||
ginkgo.By("Corrupt checkpoint file")
|
||||
checkpoints := findCheckpoints(podName)
|
||||
if len(checkpoints) == 0 {
|
||||
framework.Failf("No checkpoint for the pod was found")
|
||||
}
|
||||
for _, file := range checkpoints {
|
||||
f, err := os.OpenFile(file, os.O_WRONLY|os.O_APPEND, 0644)
|
||||
framework.ExpectNoError(err, "Failed to open file %q", file)
|
||||
_, err = f.WriteString("blabblab")
|
||||
framework.ExpectNoError(err, "Failed to write to file %q", file)
|
||||
f.Sync()
|
||||
f.Close()
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
func runPodCheckpointTest(f *framework.Framework, podName string, twist func()) {
|
||||
podName = podName + string(uuid.NewUUID())
|
||||
ginkgo.By(fmt.Sprintf("Creating test pod: %s", podName))
|
||||
f.PodClient().CreateSync(&v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: podName},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Image: imageutils.GetPauseImageName(),
|
||||
Name: "pause-container",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
ginkgo.By("Performing disruptive operations")
|
||||
twist()
|
||||
|
||||
ginkgo.By("Remove test pod")
|
||||
f.PodClient().DeleteSync(podName, metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
|
||||
|
||||
ginkgo.By("Waiting for checkpoint to be removed")
|
||||
if err := wait.PollImmediate(10*time.Second, gcTimeout, func() (bool, error) {
|
||||
checkpoints := findCheckpoints(podName)
|
||||
if len(checkpoints) == 0 {
|
||||
return true, nil
|
||||
}
|
||||
framework.Logf("Checkpoint of %q still exists: %v", podName, checkpoints)
|
||||
return false, nil
|
||||
}); err != nil {
|
||||
framework.Failf("Failed to observe checkpoint being removed within timeout: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// cleanString cleans up any trailing spaces and new line character for the input string
|
||||
func cleanString(output string) string {
|
||||
processed := strings.TrimSpace(string(output))
|
||||
regex := regexp.MustCompile(`\r?\n`)
|
||||
processed = regex.ReplaceAllString(processed, "")
|
||||
return processed
|
||||
}
|
||||
|
||||
func writeFileAndSync(path string, data []byte) error {
|
||||
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = f.Write(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.Sync()
|
||||
if err1 := f.Close(); err == nil {
|
||||
err = err1
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// findCheckpoints returns all checkpoint files containing input string
|
||||
func findCheckpoints(match string) []string {
|
||||
ginkgo.By(fmt.Sprintf("Search checkpoints containing %q", match))
|
||||
checkpoints := []string{}
|
||||
stdout, err := exec.Command("sudo", "grep", "-rl", match, framework.TestContext.DockershimCheckpointDir).CombinedOutput()
|
||||
if err != nil {
|
||||
framework.Logf("grep from dockershim checkpoint directory returns error: %v", err)
|
||||
}
|
||||
if stdout == nil {
|
||||
return checkpoints
|
||||
}
|
||||
files := strings.Split(string(stdout), "\n")
|
||||
for _, file := range files {
|
||||
cleaned := cleanString(file)
|
||||
if len(cleaned) == 0 {
|
||||
continue
|
||||
}
|
||||
checkpoints = append(checkpoints, cleaned)
|
||||
}
|
||||
return checkpoints
|
||||
}
|
@@ -29,8 +29,6 @@ import (
|
||||
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
|
||||
)
|
||||
|
||||
const success = "\033[0;32mSUCESS\033[0m"
|
||||
@@ -66,12 +64,9 @@ func check(options ...string) []error {
|
||||
switch c {
|
||||
case "all":
|
||||
errs = appendNotNil(errs, kernel())
|
||||
errs = appendNotNil(errs, containerRuntime())
|
||||
errs = appendNotNil(errs, daemons())
|
||||
errs = appendNotNil(errs, firewall())
|
||||
errs = appendNotNil(errs, dns())
|
||||
case "containerruntime":
|
||||
errs = appendNotNil(errs, containerRuntime())
|
||||
case "daemons":
|
||||
errs = appendNotNil(errs, daemons())
|
||||
case "dns":
|
||||
@@ -88,37 +83,6 @@ func check(options ...string) []error {
|
||||
return errs
|
||||
}
|
||||
|
||||
const dockerVersionRegex = `1\.[7-9]\.[0-9]+`
|
||||
|
||||
// containerRuntime checks that a suitable container runtime is installed and recognized by cadvisor: docker 1.7-1.9
|
||||
func containerRuntime() error {
|
||||
dockerRegex, err := regexp.Compile(dockerVersionRegex)
|
||||
if err != nil {
|
||||
// This should never happen and can only be fixed by changing the code
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Setup cadvisor to check the container environment
|
||||
c, err := cadvisor.New(cadvisor.NewImageFsInfoProvider("docker", ""), "/var/lib/kubelet", []string{"/"}, false)
|
||||
if err != nil {
|
||||
return printError("Container Runtime Check: %s Could not start cadvisor %v", failed, err)
|
||||
}
|
||||
|
||||
vi, err := c.VersionInfo()
|
||||
if err != nil {
|
||||
return printError("Container Runtime Check: %s Could not get VersionInfo %v", failed, err)
|
||||
}
|
||||
|
||||
d := vi.DockerVersion
|
||||
if !dockerRegex.Match([]byte(d)) {
|
||||
return printError(
|
||||
"Container Runtime Check: %s Docker version %s does not matching %s. You may need to run as root or the "+
|
||||
"user the kubelet will run under.", failed, d, dockerVersionRegex)
|
||||
}
|
||||
|
||||
return printSuccess("Container Runtime Check: %s", success)
|
||||
}
|
||||
|
||||
const kubeletClusterDNSRegexStr = `\/kubelet.*--cluster-dns=(\S+) `
|
||||
const kubeletClusterDomainRegexStr = `\/kubelet.*--cluster-domain=(\S+)`
|
||||
|
||||
|
@@ -22,16 +22,12 @@ import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/onsi/ginkgo"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
|
||||
imageutils "k8s.io/kubernetes/test/utils/image"
|
||||
|
||||
"github.com/blang/semver"
|
||||
"github.com/onsi/ginkgo"
|
||||
)
|
||||
|
||||
// checkProcess checks whether there's a process whose command line contains
|
||||
@@ -111,215 +107,6 @@ func checkPublicGCR() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkDockerConfig runs docker's check-config.sh script and ensures that all
|
||||
// expected kernel configs are enabled.
|
||||
func checkDockerConfig() error {
|
||||
var (
|
||||
re = regexp.MustCompile("\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[mGK]")
|
||||
bins = []string{
|
||||
"/usr/share/docker.io/contrib/check-config.sh",
|
||||
"/usr/share/docker/contrib/check-config.sh",
|
||||
}
|
||||
allowlist = map[string]bool{
|
||||
"CONFIG_MEMCG_SWAP_ENABLED": true,
|
||||
"CONFIG_RT_GROUP_SCHED": true,
|
||||
"CONFIG_EXT3_FS": true,
|
||||
"CONFIG_EXT3_FS_XATTR": true,
|
||||
"CONFIG_EXT3_FS_POSIX_ACL": true,
|
||||
"CONFIG_EXT3_FS_SECURITY": true,
|
||||
"/dev/zfs": true,
|
||||
"zfs command": true,
|
||||
"zpool command": true,
|
||||
}
|
||||
missing = map[string]bool{}
|
||||
)
|
||||
|
||||
// Allowlists CONFIG_DEVPTS_MULTIPLE_INSTANCES (meaning allowing it to be
|
||||
// absent) if the kernel version is >= 4.8, because this option has been
|
||||
// removed from the 4.8 kernel.
|
||||
kernelVersion, err := getKernelVersion()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if kernelVersion.GTE(semver.MustParse("4.8.0")) {
|
||||
allowlist["CONFIG_DEVPTS_MULTIPLE_INSTANCES"] = true
|
||||
}
|
||||
|
||||
for _, bin := range bins {
|
||||
if _, err := os.Stat(bin); os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
// We don't check the return code because it's OK if the script returns
|
||||
// a non-zero exit code just because the configs in the allowlist are
|
||||
// missing.
|
||||
output, _ := runCommand(bin)
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
if !strings.Contains(line, "missing") {
|
||||
continue
|
||||
}
|
||||
line = re.ReplaceAllString(line, "")
|
||||
fields := strings.Split(line, ":")
|
||||
if len(fields) != 2 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimFunc(fields[0], func(c rune) bool {
|
||||
return c == ' ' || c == '-'
|
||||
})
|
||||
if _, found := allowlist[key]; !found {
|
||||
missing[key] = true
|
||||
}
|
||||
}
|
||||
if len(missing) != 0 {
|
||||
return fmt.Errorf("missing docker config: %v", missing)
|
||||
}
|
||||
break
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkDockerNetworkClient checks client networking by pinging an external IP
|
||||
// address from a container.
|
||||
func checkDockerNetworkClient() error {
|
||||
imageName := imageutils.GetE2EImage(imageutils.BusyBox)
|
||||
output, err := runCommand("docker", "run", "--rm", imageName, "sh", "-c", "ping -w 5 -q google.com")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !strings.Contains(output, `0% packet loss`) {
|
||||
return fmt.Errorf("failed to ping from container: %s", output)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkDockerNetworkServer checks server networking by running an echo server
|
||||
// within a container and accessing it from outside.
|
||||
func checkDockerNetworkServer() error {
|
||||
const (
|
||||
imageName = "k8s.gcr.io/nginx:1.7.9"
|
||||
hostAddr = "127.0.0.1"
|
||||
hostPort = "8088"
|
||||
containerPort = "80"
|
||||
containerID = "nginx"
|
||||
message = "Welcome to nginx!"
|
||||
)
|
||||
var (
|
||||
portMapping = fmt.Sprintf("%s:%s", hostPort, containerPort)
|
||||
host = fmt.Sprintf("http://%s:%s", hostAddr, hostPort)
|
||||
)
|
||||
runCommand("docker", "rm", "-f", containerID)
|
||||
if _, err := runCommand("docker", "run", "-d", "--name", containerID, "-p", portMapping, imageName); err != nil {
|
||||
return err
|
||||
}
|
||||
output, err := runCommand("curl", host)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !strings.Contains(output, message) {
|
||||
return fmt.Errorf("failed to connect to container")
|
||||
}
|
||||
// Clean up
|
||||
if _, err = runCommand("docker", "rm", "-f", containerID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = runCommand("docker", "rmi", imageName); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkDockerAppArmor checks whether AppArmor is enabled and has the
|
||||
// "docker-default" profile.
|
||||
func checkDockerAppArmor() error {
|
||||
buf, err := ioutil.ReadFile("/sys/module/apparmor/parameters/enabled")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if string(buf) != "Y\n" {
|
||||
return fmt.Errorf("apparmor module is not loaded")
|
||||
}
|
||||
|
||||
// Checks that the "docker-default" profile is loaded and enforced.
|
||||
buf, err = ioutil.ReadFile("/sys/kernel/security/apparmor/profiles")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !strings.Contains(string(buf), "docker-default (enforce)") {
|
||||
return fmt.Errorf("'docker-default' profile is not loaded and enforced")
|
||||
}
|
||||
|
||||
// Checks that the `apparmor_parser` binary is present.
|
||||
_, err = exec.LookPath("apparmor_parser")
|
||||
if err != nil {
|
||||
return fmt.Errorf("'apparmor_parser' is not in directories named by the PATH env")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkDockerSeccomp checks whether the Docker supports seccomp.
|
||||
func checkDockerSeccomp() error {
|
||||
const (
|
||||
seccompProfileFileName = "/tmp/no_mkdir.json"
|
||||
seccompProfile = `{
|
||||
"defaultAction": "SCMP_ACT_ALLOW",
|
||||
"syscalls": [
|
||||
{
|
||||
"name": "mkdir",
|
||||
"action": "SCMP_ACT_ERRNO"
|
||||
}
|
||||
]}`
|
||||
image = "gcr.io/google-appengine/debian8:2017-06-07-171918"
|
||||
)
|
||||
if err := ioutil.WriteFile(seccompProfileFileName, []byte(seccompProfile), 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
// Starts a container with no seccomp profile and ensures that unshare
|
||||
// succeeds.
|
||||
_, err := runCommand("docker", "run", "--rm", "-i", "--security-opt", "seccomp=unconfined", image, "unshare", "-r", "whoami")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Starts a container with the default seccomp profile and ensures that
|
||||
// unshare (a denylisted system call in the default profile) fails.
|
||||
cmd := []string{"docker", "run", "--rm", "-i", image, "unshare", "-r", "whoami"}
|
||||
_, err = runCommand(cmd...)
|
||||
if err == nil {
|
||||
return fmt.Errorf("%q did not fail as expected", strings.Join(cmd, " "))
|
||||
}
|
||||
// Starts a container with a custom seccomp profile that denylists mkdir
|
||||
// and ensures that unshare succeeds.
|
||||
_, err = runCommand("docker", "run", "--rm", "-i", "--security-opt", fmt.Sprintf("seccomp=%s", seccompProfileFileName), image, "unshare", "-r", "whoami")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Starts a container with a custom seccomp profile that denylists mkdir
|
||||
// and ensures that mkdir fails.
|
||||
cmd = []string{"docker", "run", "--rm", "-i", "--security-opt", fmt.Sprintf("seccomp=%s", seccompProfileFileName), image, "mkdir", "-p", "/tmp/foo"}
|
||||
_, err = runCommand(cmd...)
|
||||
if err == nil {
|
||||
return fmt.Errorf("%q did not fail as expected", strings.Join(cmd, " "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkDockerStorageDriver checks whether the current storage driver used by
|
||||
// Docker is overlay.
|
||||
func checkDockerStorageDriver() error {
|
||||
output, err := runCommand("docker", "info")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, line := range strings.Split(string(output), "\n") {
|
||||
if !strings.Contains(line, "Storage Driver:") {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(line, "overlay") {
|
||||
return fmt.Errorf("storage driver is not 'overlay': %s", line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("failed to find storage driver")
|
||||
}
|
||||
|
||||
var _ = SIGDescribe("GKE system requirements [NodeConformance][Feature:GKEEnv][NodeFeature:GKEEnv]", func() {
|
||||
ginkgo.BeforeEach(func() {
|
||||
e2eskipper.RunIfSystemSpecNameIs("gke")
|
||||
@@ -345,23 +132,6 @@ var _ = SIGDescribe("GKE system requirements [NodeConformance][Feature:GKEEnv][N
|
||||
ginkgo.It("The GCR is accessible", func() {
|
||||
framework.ExpectNoError(checkPublicGCR())
|
||||
})
|
||||
ginkgo.It("The docker configuration validation should pass", func() {
|
||||
e2eskipper.RunIfContainerRuntimeIs("docker")
|
||||
framework.ExpectNoError(checkDockerConfig())
|
||||
})
|
||||
ginkgo.It("The docker container network should work", func() {
|
||||
e2eskipper.RunIfContainerRuntimeIs("docker")
|
||||
framework.ExpectNoError(checkDockerNetworkServer())
|
||||
framework.ExpectNoError(checkDockerNetworkClient())
|
||||
})
|
||||
ginkgo.It("The docker daemon should support AppArmor and seccomp", func() {
|
||||
e2eskipper.RunIfContainerRuntimeIs("docker")
|
||||
framework.ExpectNoError(checkDockerAppArmor())
|
||||
framework.ExpectNoError(checkDockerSeccomp())
|
||||
})
|
||||
ginkgo.It("The docker storage driver should work", func() {
|
||||
framework.ExpectNoError(checkDockerStorageDriver())
|
||||
})
|
||||
})
|
||||
|
||||
// getPPID returns the PPID for the pid.
|
||||
@@ -423,21 +193,6 @@ func getCmdToProcessMap() (map[string][]process, error) {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// getKernelVersion returns the kernel version in the semantic version format.
|
||||
func getKernelVersion() (*semver.Version, error) {
|
||||
output, err := runCommand("uname", "-r")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// An example 'output' could be "4.13.0-1001-gke".
|
||||
v := strings.TrimSpace(strings.Split(output, "-")[0])
|
||||
kernelVersion, err := semver.Make(v)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to convert %q to semantic version: %s", v, err)
|
||||
}
|
||||
return &kernelVersion, nil
|
||||
}
|
||||
|
||||
// runCommand runs the cmd and returns the combined stdout and stderr, or an
|
||||
// error if the command failed.
|
||||
func runCommand(cmd ...string) (string, error) {
|
||||
|
@@ -20,7 +20,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/user"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -117,21 +116,6 @@ type puller interface {
|
||||
Name() string
|
||||
}
|
||||
|
||||
type dockerPuller struct {
|
||||
}
|
||||
|
||||
func (dp *dockerPuller) Name() string {
|
||||
return "docker"
|
||||
}
|
||||
|
||||
func (dp *dockerPuller) Pull(image string) ([]byte, error) {
|
||||
// TODO(random-liu): Use docker client to get rid of docker binary dependency.
|
||||
if exec.Command("docker", "inspect", "--type=image", image).Run() != nil {
|
||||
return exec.Command("docker", "pull", image).CombinedOutput()
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type remotePuller struct {
|
||||
imageService internalapi.ImageManagerService
|
||||
}
|
||||
@@ -150,20 +134,13 @@ func (rp *remotePuller) Pull(image string) ([]byte, error) {
|
||||
}
|
||||
|
||||
func getPuller() (puller, error) {
|
||||
runtime := framework.TestContext.ContainerRuntime
|
||||
switch runtime {
|
||||
case "docker":
|
||||
return &dockerPuller{}, nil
|
||||
case "remote":
|
||||
_, is, err := getCRIClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &remotePuller{
|
||||
imageService: is,
|
||||
}, nil
|
||||
_, is, err := getCRIClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, fmt.Errorf("can't prepull images, unknown container runtime %q", runtime)
|
||||
return &remotePuller{
|
||||
imageService: is,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// PrePullAllImages pre-fetches all images tests depend on so that we don't fail in an actual test.
|
||||
|
@@ -26,7 +26,6 @@ import (
|
||||
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
||||
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
|
||||
|
||||
"github.com/onsi/ginkgo"
|
||||
)
|
||||
@@ -116,29 +115,6 @@ var _ = SIGDescribe("ContainerLogPath [NodeConformance]", func() {
|
||||
|
||||
var logPodName string
|
||||
ginkgo.BeforeEach(func() {
|
||||
if framework.TestContext.ContainerRuntime == "docker" {
|
||||
// Container Log Path support requires JSON logging driver.
|
||||
// It does not work when Docker daemon is logging to journald.
|
||||
d, err := getDockerLoggingDriver()
|
||||
framework.ExpectNoError(err)
|
||||
if d != "json-file" {
|
||||
e2eskipper.Skipf("Skipping because Docker daemon is using a logging driver other than \"json-file\": %s", d)
|
||||
}
|
||||
// Even if JSON logging is in use, this test fails if SELinux support
|
||||
// is enabled, since the isolation provided by the SELinux policy
|
||||
// prevents processes running inside Docker containers (under SELinux
|
||||
// type svirt_lxc_net_t) from accessing the log files which are owned
|
||||
// by Docker (and labeled with the container_var_lib_t type.)
|
||||
//
|
||||
// Therefore, let's also skip this test when running with SELinux
|
||||
// support enabled.
|
||||
e, err := isDockerSELinuxSupportEnabled()
|
||||
framework.ExpectNoError(err)
|
||||
if e {
|
||||
e2eskipper.Skipf("Skipping because Docker daemon is running with SELinux support enabled")
|
||||
}
|
||||
}
|
||||
|
||||
podClient = f.PodClient()
|
||||
logPodName = "log-pod-" + string(uuid.NewUUID())
|
||||
err := createAndWaitPod(makeLogPod(logPodName, logString))
|
||||
|
@@ -84,10 +84,6 @@ var _ = SIGDescribe("Restart [Serial] [Slow] [Disruptive]", func() {
|
||||
ginkgo.Context("Container Runtime", func() {
|
||||
ginkgo.Context("Network", func() {
|
||||
ginkgo.It("should recover from ip leak", func() {
|
||||
if framework.TestContext.ContainerRuntime == "docker" {
|
||||
ginkgo.Skip("Test fails with in-tree docker. Skipping test.")
|
||||
}
|
||||
|
||||
pods := newTestPods(podCount, false, imageutils.GetPauseImageName(), "restart-container-runtime-test")
|
||||
ginkgo.By(fmt.Sprintf("Trying to create %d pods on node", len(pods)))
|
||||
createBatchPodWithRateControl(f, pods, podCreationInterval)
|
||||
|
@@ -104,7 +104,7 @@ var _ = SIGDescribe("Kubelet PodOverhead handling [LinuxOnly]", func() {
|
||||
handler string
|
||||
)
|
||||
ginkgo.By("Creating a RuntimeClass with Overhead definied", func() {
|
||||
handler = e2enode.PreconfiguredRuntimeClassHandler(framework.TestContext.ContainerRuntime)
|
||||
handler = e2enode.PreconfiguredRuntimeClassHandler
|
||||
rc := &nodev1.RuntimeClass{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: handler},
|
||||
Handler: handler,
|
||||
|
@@ -28,7 +28,6 @@ import (
|
||||
"k8s.io/apimachinery/pkg/util/uuid"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
||||
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
|
||||
imageutils "k8s.io/kubernetes/test/utils/image"
|
||||
|
||||
"github.com/onsi/ginkgo"
|
||||
@@ -72,13 +71,6 @@ var _ = SIGDescribe("Security Context", func() {
|
||||
})
|
||||
|
||||
ginkgo.It("processes in containers sharing a pod namespace should be able to see each other", func() {
|
||||
ginkgo.By("Check whether shared PID namespace is supported.")
|
||||
isEnabled, err := isSharedPIDNamespaceSupported()
|
||||
framework.ExpectNoError(err)
|
||||
if !isEnabled {
|
||||
e2eskipper.Skipf("Skipped because shared PID namespace is not supported by this docker version.")
|
||||
}
|
||||
|
||||
ginkgo.By("Create a pod with shared PID namespace.")
|
||||
f.PodClient().CreateSync(&v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "shared-pid-ns-test-pod"},
|
||||
|
@@ -19,7 +19,6 @@ package e2enode
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -136,29 +135,6 @@ var _ = SIGDescribe("Summary API [NodeConformance]", func() {
|
||||
"MajorPageFaults": bounded(0, expectedMajorPageFaultsUpperBound),
|
||||
})
|
||||
runtimeContExpectations := sysContExpectations().(*gstruct.FieldsMatcher)
|
||||
if systemdutil.IsRunningSystemd() && framework.TestContext.ContainerRuntime == "docker" {
|
||||
// Some Linux distributions still ship a docker.service that is missing
|
||||
// a `Delegate=yes` setting (or equivalent CPUAccounting= and MemoryAccounting=)
|
||||
// that allows us to monitor the container runtime resource usage through
|
||||
// the "cpu" and "memory" cgroups.
|
||||
//
|
||||
// Make an exception here for those distros, only for Docker, so that they
|
||||
// can pass the full node e2e tests even in that case.
|
||||
//
|
||||
// For newer container runtimes (using CRI) and even distros that still
|
||||
// ship Docker, we should encourage them to always set `Delegate=yes` in
|
||||
// order to make monitoring of the runtime possible.
|
||||
stdout, err := exec.Command("systemctl", "show", "-p", "Delegate", "docker.service").CombinedOutput()
|
||||
if err == nil && strings.TrimSpace(string(stdout)) == "Delegate=no" {
|
||||
// Only make these optional if we can successfully confirm that
|
||||
// Delegate is set to "no" (in other words, unset.) If we fail
|
||||
// to check that, default to requiring it, which might cause
|
||||
// false positives, but that should be the safer approach.
|
||||
ginkgo.By("Making runtime container expectations optional, since systemd was not configured to Delegate=yes the cgroups")
|
||||
runtimeContExpectations.Fields["Memory"] = gomega.Or(gomega.BeNil(), runtimeContExpectations.Fields["Memory"])
|
||||
runtimeContExpectations.Fields["CPU"] = gomega.Or(gomega.BeNil(), runtimeContExpectations.Fields["CPU"])
|
||||
}
|
||||
}
|
||||
systemContainers := gstruct.Elements{
|
||||
"kubelet": sysContExpectations(),
|
||||
"runtime": runtimeContExpectations,
|
||||
|
Reference in New Issue
Block a user