Clean up dockershim in tests

Signed-off-by: Ciprian Hacman <ciprian@hakman.dev>
This commit is contained in:
Ciprian Hacman
2021-12-22 09:42:15 +02:00
parent c75d254beb
commit a0abe5aa33
24 changed files with 99 additions and 1006 deletions

View File

@@ -24,9 +24,7 @@ import (
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
kubelogs "k8s.io/kubernetes/pkg/kubelet/logs"
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
"k8s.io/kubernetes/test/e2e/framework"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
@@ -43,12 +41,6 @@ const (
var _ = SIGDescribe("ContainerLogRotation [Slow] [Serial] [Disruptive]", func() {
f := framework.NewDefaultFramework("container-log-rotation-test")
ginkgo.Context("when a container generates a lot of log", func() {
ginkgo.BeforeEach(func() {
if framework.TestContext.ContainerRuntime != kubetypes.RemoteContainerRuntime {
e2eskipper.Skipf("Skipping ContainerLogRotation test since the container runtime is not remote")
}
})
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
initialConfig.ContainerLogMaxFiles = testContainerLogMaxFiles
initialConfig.ContainerLogMaxSize = testContainerLogMaxSize

View File

@@ -1,140 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2enode
import (
"context"
"fmt"
"strings"
"time"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/framework"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
imageutils "k8s.io/kubernetes/test/utils/image"
"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
)
var _ = SIGDescribe("Docker features [Feature:Docker][Legacy:Docker]", func() {
f := framework.NewDefaultFramework("docker-feature-test")
ginkgo.BeforeEach(func() {
e2eskipper.RunIfContainerRuntimeIs("docker")
})
ginkgo.Context("when live-restore is enabled [Serial] [Slow] [Disruptive]", func() {
ginkgo.It("containers should not be disrupted when the daemon shuts down and restarts", func() {
const (
podName = "live-restore-test-pod"
containerName = "live-restore-test-container"
)
isSupported, err := isDockerLiveRestoreSupported()
framework.ExpectNoError(err)
if !isSupported {
e2eskipper.Skipf("Docker live-restore is not supported.")
}
isEnabled, err := isDockerLiveRestoreEnabled()
framework.ExpectNoError(err)
if !isEnabled {
e2eskipper.Skipf("Docker live-restore is not enabled.")
}
ginkgo.By("Create the test pod.")
pod := f.PodClient().CreateSync(&v1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: podName},
Spec: v1.PodSpec{
Containers: []v1.Container{{
Name: containerName,
Image: imageutils.GetE2EImage(imageutils.Nginx),
}},
},
})
ginkgo.By("Ensure that the container is running before Docker is down.")
gomega.Eventually(func() bool {
return isContainerRunning(pod.Status.PodIP)
}).Should(gomega.BeTrue())
startTime1, err := getContainerStartTime(f, podName, containerName)
framework.ExpectNoError(err)
ginkgo.By("Stop Docker daemon.")
framework.ExpectNoError(stopDockerDaemon())
isDockerDown := true
defer func() {
if isDockerDown {
ginkgo.By("Start Docker daemon.")
framework.ExpectNoError(startDockerDaemon())
}
}()
ginkgo.By("Ensure that the container is running after Docker is down.")
gomega.Consistently(func() bool {
return isContainerRunning(pod.Status.PodIP)
}).Should(gomega.BeTrue())
ginkgo.By("Start Docker daemon.")
framework.ExpectNoError(startDockerDaemon())
isDockerDown = false
ginkgo.By("Ensure that the container is running after Docker has restarted.")
gomega.Consistently(func() bool {
return isContainerRunning(pod.Status.PodIP)
}).Should(gomega.BeTrue())
ginkgo.By("Ensure that the container has not been restarted after Docker is restarted.")
gomega.Consistently(func() bool {
startTime2, err := getContainerStartTime(f, podName, containerName)
framework.ExpectNoError(err)
return startTime1 == startTime2
}, 3*time.Second, time.Second).Should(gomega.BeTrue())
})
})
})
// isContainerRunning returns true if the container is running by checking
// whether the server is responding, and false otherwise.
func isContainerRunning(podIP string) bool {
output, err := runCommand("curl", podIP)
if err != nil {
return false
}
return strings.Contains(output, "Welcome to nginx!")
}
// getContainerStartTime returns the start time of the container with the
// containerName of the pod having the podName.
func getContainerStartTime(f *framework.Framework, podName, containerName string) (time.Time, error) {
pod, err := f.PodClient().Get(context.TODO(), podName, metav1.GetOptions{})
if err != nil {
return time.Time{}, fmt.Errorf("failed to get pod %q: %v", podName, err)
}
for _, status := range pod.Status.ContainerStatuses {
if status.Name != containerName {
continue
}
if status.State.Running == nil {
return time.Time{}, fmt.Errorf("%v/%v is not running", podName, containerName)
}
return status.State.Running.StartedAt.Time, nil
}
return time.Time{}, fmt.Errorf("failed to find %v/%v", podName, containerName)
}

View File

@@ -1,116 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2enode
import (
"fmt"
"strings"
"github.com/blang/semver"
systemdutil "github.com/coreos/go-systemd/v22/util"
)
// getDockerAPIVersion returns the Docker's API version.
func getDockerAPIVersion() (semver.Version, error) {
output, err := runCommand("docker", "version", "-f", "{{.Server.APIVersion}}")
if err != nil {
return semver.Version{}, fmt.Errorf("failed to get docker server version: %v", err)
}
return semver.MustParse(strings.TrimSpace(output) + ".0"), nil
}
// isSharedPIDNamespaceSupported returns true if the Docker version is 1.13.1+
// (API version 1.26+), and false otherwise.
func isSharedPIDNamespaceSupported() (bool, error) {
version, err := getDockerAPIVersion()
if err != nil {
return false, err
}
return version.GTE(semver.MustParse("1.26.0")), nil
}
// isDockerLiveRestoreSupported returns true if live-restore is supported in
// the current Docker version.
func isDockerLiveRestoreSupported() (bool, error) {
version, err := getDockerAPIVersion()
if err != nil {
return false, err
}
return version.GTE(semver.MustParse("1.26.0")), nil
}
// getDockerInfo returns the Info struct for the running Docker daemon.
func getDockerInfo(key string) (string, error) {
output, err := runCommand("docker", "info", "-f", "{{."+key+"}}")
if err != nil {
return "", fmt.Errorf("failed to get docker info: %v", err)
}
return strings.TrimSpace(output), nil
}
// isDockerLiveRestoreEnabled returns true if live-restore is enabled in the
// Docker.
func isDockerLiveRestoreEnabled() (bool, error) {
info, err := getDockerInfo("LiveRestoreEnabled")
if err != nil {
return false, err
}
return info == "true", nil
}
// getDockerLoggingDriver returns the name of the logging driver.
func getDockerLoggingDriver() (string, error) {
info, err := getDockerInfo("LoggingDriver")
if err != nil {
return "", err
}
return info, nil
}
// isDockerSELinuxSupportEnabled checks whether the Docker daemon was started
// with SELinux support enabled.
func isDockerSELinuxSupportEnabled() (bool, error) {
info, err := getDockerInfo("SecurityOptions")
if err != nil {
return false, err
}
return strings.Contains(info, "name=selinux"), nil
}
// startDockerDaemon starts the Docker daemon.
func startDockerDaemon() error {
switch {
case systemdutil.IsRunningSystemd():
_, err := runCommand("systemctl", "start", "docker")
return err
default:
_, err := runCommand("service", "docker", "start")
return err
}
}
// stopDockerDaemon stops the Docker daemon.
func stopDockerDaemon() error {
switch {
case systemdutil.IsRunningSystemd():
_, err := runCommand("systemctl", "stop", "docker")
return err
default:
_, err := runCommand("service", "docker", "stop")
return err
}
}

View File

@@ -1,231 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2enode
import (
"crypto/md5"
"fmt"
"os"
"os/exec"
"path"
"regexp"
"strings"
"time"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/kubernetes/test/e2e/framework"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
imageutils "k8s.io/kubernetes/test/utils/image"
"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
)
const (
testCheckpoint = "checkpoint-test"
// Container GC Period is 1 minute
gcTimeout = 3 * time.Minute
testCheckpointContent = `{"version":"v1","name":"fluentd-gcp-v2.0-vmnqx","namespace":"kube-system","data":{},"checksum":1799154314}`
)
var _ = SIGDescribe("Dockershim [Serial] [Disruptive] [Feature:Docker][Legacy:Docker]", func() {
f := framework.NewDefaultFramework("dockerhism-checkpoint-test")
ginkgo.BeforeEach(func() {
e2eskipper.RunIfContainerRuntimeIs("docker")
})
ginkgo.It("should clean up pod sandbox checkpoint after pod deletion", func() {
podName := "pod-checkpoint-no-disrupt"
runPodCheckpointTest(f, podName, func() {
checkpoints := findCheckpoints(podName)
if len(checkpoints) == 0 {
framework.Failf("No checkpoint for the pod was found")
}
})
})
ginkgo.It("should remove dangling checkpoint file", func() {
filename := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%s/%s", testCheckpoint, f.Namespace.Name))))
fullpath := path.Join(framework.TestContext.DockershimCheckpointDir, filename)
ginkgo.By(fmt.Sprintf("Write a file at %q", fullpath))
err := writeFileAndSync(fullpath, []byte(testCheckpointContent))
framework.ExpectNoError(err, "Failed to create file %q", fullpath)
ginkgo.By("Check if file is removed")
gomega.Eventually(func() bool {
if _, err := os.Stat(fullpath); os.IsNotExist(err) {
return true
}
return false
}, gcTimeout, 10*time.Second).Should(gomega.BeTrue())
})
ginkgo.Context("When pod sandbox checkpoint is missing", func() {
ginkgo.It("should complete pod sandbox clean up", func() {
podName := "pod-checkpoint-missing"
runPodCheckpointTest(f, podName, func() {
checkpoints := findCheckpoints(podName)
if len(checkpoints) == 0 {
framework.Failf("No checkpoint for the pod was found")
}
ginkgo.By("Removing checkpoint of test pod")
for _, filename := range checkpoints {
if len(filename) == 0 {
continue
}
framework.Logf("Removing checkpoint %q", filename)
_, err := exec.Command("sudo", "rm", filename).CombinedOutput()
framework.ExpectNoError(err, "Failed to remove checkpoint file %q: %v", string(filename), err)
}
})
})
})
ginkgo.Context("When all containers in pod are missing", func() {
ginkgo.It("should complete pod sandbox clean up based on the information in sandbox checkpoint", func() {
runPodCheckpointTest(f, "pod-containers-missing", func() {
ginkgo.By("Gathering pod container ids")
stdout, err := exec.Command("sudo", "docker", "ps", "-q", "-f",
fmt.Sprintf("name=%s", f.Namespace.Name)).CombinedOutput()
framework.ExpectNoError(err, "Failed to run docker ps: %v", err)
lines := strings.Split(string(stdout), "\n")
ids := []string{}
for _, id := range lines {
id = cleanString(id)
if len(id) > 0 {
ids = append(ids, id)
}
}
ginkgo.By("Stop and remove pod containers")
dockerStopCmd := append([]string{"docker", "stop"}, ids...)
_, err = exec.Command("sudo", dockerStopCmd...).CombinedOutput()
framework.ExpectNoError(err, "Failed to run command %v: %v", dockerStopCmd, err)
dockerRmCmd := append([]string{"docker", "rm"}, ids...)
_, err = exec.Command("sudo", dockerRmCmd...).CombinedOutput()
framework.ExpectNoError(err, "Failed to run command %v: %v", dockerRmCmd, err)
})
})
})
ginkgo.Context("When checkpoint file is corrupted", func() {
ginkgo.It("should complete pod sandbox clean up", func() {
podName := "pod-checkpoint-corrupted"
runPodCheckpointTest(f, podName, func() {
ginkgo.By("Corrupt checkpoint file")
checkpoints := findCheckpoints(podName)
if len(checkpoints) == 0 {
framework.Failf("No checkpoint for the pod was found")
}
for _, file := range checkpoints {
f, err := os.OpenFile(file, os.O_WRONLY|os.O_APPEND, 0644)
framework.ExpectNoError(err, "Failed to open file %q", file)
_, err = f.WriteString("blabblab")
framework.ExpectNoError(err, "Failed to write to file %q", file)
f.Sync()
f.Close()
}
})
})
})
})
func runPodCheckpointTest(f *framework.Framework, podName string, twist func()) {
podName = podName + string(uuid.NewUUID())
ginkgo.By(fmt.Sprintf("Creating test pod: %s", podName))
f.PodClient().CreateSync(&v1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: podName},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Image: imageutils.GetPauseImageName(),
Name: "pause-container",
},
},
},
})
ginkgo.By("Performing disruptive operations")
twist()
ginkgo.By("Remove test pod")
f.PodClient().DeleteSync(podName, metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
ginkgo.By("Waiting for checkpoint to be removed")
if err := wait.PollImmediate(10*time.Second, gcTimeout, func() (bool, error) {
checkpoints := findCheckpoints(podName)
if len(checkpoints) == 0 {
return true, nil
}
framework.Logf("Checkpoint of %q still exists: %v", podName, checkpoints)
return false, nil
}); err != nil {
framework.Failf("Failed to observe checkpoint being removed within timeout: %v", err)
}
}
// cleanString cleans up any trailing spaces and new line character for the input string
func cleanString(output string) string {
processed := strings.TrimSpace(string(output))
regex := regexp.MustCompile(`\r?\n`)
processed = regex.ReplaceAllString(processed, "")
return processed
}
func writeFileAndSync(path string, data []byte) error {
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return err
}
_, err = f.Write(data)
if err != nil {
return err
}
f.Sync()
if err1 := f.Close(); err == nil {
err = err1
}
return err
}
// findCheckpoints returns all checkpoint files containing input string
func findCheckpoints(match string) []string {
ginkgo.By(fmt.Sprintf("Search checkpoints containing %q", match))
checkpoints := []string{}
stdout, err := exec.Command("sudo", "grep", "-rl", match, framework.TestContext.DockershimCheckpointDir).CombinedOutput()
if err != nil {
framework.Logf("grep from dockershim checkpoint directory returns error: %v", err)
}
if stdout == nil {
return checkpoints
}
files := strings.Split(string(stdout), "\n")
for _, file := range files {
cleaned := cleanString(file)
if len(cleaned) == 0 {
continue
}
checkpoints = append(checkpoints, cleaned)
}
return checkpoints
}

View File

@@ -29,8 +29,6 @@ import (
"errors"
"os"
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
)
const success = "\033[0;32mSUCESS\033[0m"
@@ -66,12 +64,9 @@ func check(options ...string) []error {
switch c {
case "all":
errs = appendNotNil(errs, kernel())
errs = appendNotNil(errs, containerRuntime())
errs = appendNotNil(errs, daemons())
errs = appendNotNil(errs, firewall())
errs = appendNotNil(errs, dns())
case "containerruntime":
errs = appendNotNil(errs, containerRuntime())
case "daemons":
errs = appendNotNil(errs, daemons())
case "dns":
@@ -88,37 +83,6 @@ func check(options ...string) []error {
return errs
}
const dockerVersionRegex = `1\.[7-9]\.[0-9]+`
// containerRuntime checks that a suitable container runtime is installed and recognized by cadvisor: docker 1.7-1.9
func containerRuntime() error {
dockerRegex, err := regexp.Compile(dockerVersionRegex)
if err != nil {
// This should never happen and can only be fixed by changing the code
panic(err)
}
// Setup cadvisor to check the container environment
c, err := cadvisor.New(cadvisor.NewImageFsInfoProvider("docker", ""), "/var/lib/kubelet", []string{"/"}, false)
if err != nil {
return printError("Container Runtime Check: %s Could not start cadvisor %v", failed, err)
}
vi, err := c.VersionInfo()
if err != nil {
return printError("Container Runtime Check: %s Could not get VersionInfo %v", failed, err)
}
d := vi.DockerVersion
if !dockerRegex.Match([]byte(d)) {
return printError(
"Container Runtime Check: %s Docker version %s does not matching %s. You may need to run as root or the "+
"user the kubelet will run under.", failed, d, dockerVersionRegex)
}
return printSuccess("Container Runtime Check: %s", success)
}
const kubeletClusterDNSRegexStr = `\/kubelet.*--cluster-dns=(\S+) `
const kubeletClusterDomainRegexStr = `\/kubelet.*--cluster-domain=(\S+)`

View File

@@ -22,16 +22,12 @@ import (
"io/ioutil"
"os"
"os/exec"
"regexp"
"strconv"
"strings"
"github.com/onsi/ginkgo"
"k8s.io/kubernetes/test/e2e/framework"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
imageutils "k8s.io/kubernetes/test/utils/image"
"github.com/blang/semver"
"github.com/onsi/ginkgo"
)
// checkProcess checks whether there's a process whose command line contains
@@ -111,215 +107,6 @@ func checkPublicGCR() error {
return nil
}
// checkDockerConfig runs docker's check-config.sh script and ensures that all
// expected kernel configs are enabled.
func checkDockerConfig() error {
var (
re = regexp.MustCompile("\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[mGK]")
bins = []string{
"/usr/share/docker.io/contrib/check-config.sh",
"/usr/share/docker/contrib/check-config.sh",
}
allowlist = map[string]bool{
"CONFIG_MEMCG_SWAP_ENABLED": true,
"CONFIG_RT_GROUP_SCHED": true,
"CONFIG_EXT3_FS": true,
"CONFIG_EXT3_FS_XATTR": true,
"CONFIG_EXT3_FS_POSIX_ACL": true,
"CONFIG_EXT3_FS_SECURITY": true,
"/dev/zfs": true,
"zfs command": true,
"zpool command": true,
}
missing = map[string]bool{}
)
// Allowlists CONFIG_DEVPTS_MULTIPLE_INSTANCES (meaning allowing it to be
// absent) if the kernel version is >= 4.8, because this option has been
// removed from the 4.8 kernel.
kernelVersion, err := getKernelVersion()
if err != nil {
return err
}
if kernelVersion.GTE(semver.MustParse("4.8.0")) {
allowlist["CONFIG_DEVPTS_MULTIPLE_INSTANCES"] = true
}
for _, bin := range bins {
if _, err := os.Stat(bin); os.IsNotExist(err) {
continue
}
// We don't check the return code because it's OK if the script returns
// a non-zero exit code just because the configs in the allowlist are
// missing.
output, _ := runCommand(bin)
for _, line := range strings.Split(output, "\n") {
if !strings.Contains(line, "missing") {
continue
}
line = re.ReplaceAllString(line, "")
fields := strings.Split(line, ":")
if len(fields) != 2 {
continue
}
key := strings.TrimFunc(fields[0], func(c rune) bool {
return c == ' ' || c == '-'
})
if _, found := allowlist[key]; !found {
missing[key] = true
}
}
if len(missing) != 0 {
return fmt.Errorf("missing docker config: %v", missing)
}
break
}
return nil
}
// checkDockerNetworkClient checks client networking by pinging an external IP
// address from a container.
func checkDockerNetworkClient() error {
imageName := imageutils.GetE2EImage(imageutils.BusyBox)
output, err := runCommand("docker", "run", "--rm", imageName, "sh", "-c", "ping -w 5 -q google.com")
if err != nil {
return err
}
if !strings.Contains(output, `0% packet loss`) {
return fmt.Errorf("failed to ping from container: %s", output)
}
return nil
}
// checkDockerNetworkServer checks server networking by running an echo server
// within a container and accessing it from outside.
func checkDockerNetworkServer() error {
const (
imageName = "k8s.gcr.io/nginx:1.7.9"
hostAddr = "127.0.0.1"
hostPort = "8088"
containerPort = "80"
containerID = "nginx"
message = "Welcome to nginx!"
)
var (
portMapping = fmt.Sprintf("%s:%s", hostPort, containerPort)
host = fmt.Sprintf("http://%s:%s", hostAddr, hostPort)
)
runCommand("docker", "rm", "-f", containerID)
if _, err := runCommand("docker", "run", "-d", "--name", containerID, "-p", portMapping, imageName); err != nil {
return err
}
output, err := runCommand("curl", host)
if err != nil {
return err
}
if !strings.Contains(output, message) {
return fmt.Errorf("failed to connect to container")
}
// Clean up
if _, err = runCommand("docker", "rm", "-f", containerID); err != nil {
return err
}
if _, err = runCommand("docker", "rmi", imageName); err != nil {
return err
}
return nil
}
// checkDockerAppArmor checks whether AppArmor is enabled and has the
// "docker-default" profile.
func checkDockerAppArmor() error {
buf, err := ioutil.ReadFile("/sys/module/apparmor/parameters/enabled")
if err != nil {
return err
}
if string(buf) != "Y\n" {
return fmt.Errorf("apparmor module is not loaded")
}
// Checks that the "docker-default" profile is loaded and enforced.
buf, err = ioutil.ReadFile("/sys/kernel/security/apparmor/profiles")
if err != nil {
return err
}
if !strings.Contains(string(buf), "docker-default (enforce)") {
return fmt.Errorf("'docker-default' profile is not loaded and enforced")
}
// Checks that the `apparmor_parser` binary is present.
_, err = exec.LookPath("apparmor_parser")
if err != nil {
return fmt.Errorf("'apparmor_parser' is not in directories named by the PATH env")
}
return nil
}
// checkDockerSeccomp checks whether the Docker supports seccomp.
func checkDockerSeccomp() error {
const (
seccompProfileFileName = "/tmp/no_mkdir.json"
seccompProfile = `{
"defaultAction": "SCMP_ACT_ALLOW",
"syscalls": [
{
"name": "mkdir",
"action": "SCMP_ACT_ERRNO"
}
]}`
image = "gcr.io/google-appengine/debian8:2017-06-07-171918"
)
if err := ioutil.WriteFile(seccompProfileFileName, []byte(seccompProfile), 0644); err != nil {
return err
}
// Starts a container with no seccomp profile and ensures that unshare
// succeeds.
_, err := runCommand("docker", "run", "--rm", "-i", "--security-opt", "seccomp=unconfined", image, "unshare", "-r", "whoami")
if err != nil {
return err
}
// Starts a container with the default seccomp profile and ensures that
// unshare (a denylisted system call in the default profile) fails.
cmd := []string{"docker", "run", "--rm", "-i", image, "unshare", "-r", "whoami"}
_, err = runCommand(cmd...)
if err == nil {
return fmt.Errorf("%q did not fail as expected", strings.Join(cmd, " "))
}
// Starts a container with a custom seccomp profile that denylists mkdir
// and ensures that unshare succeeds.
_, err = runCommand("docker", "run", "--rm", "-i", "--security-opt", fmt.Sprintf("seccomp=%s", seccompProfileFileName), image, "unshare", "-r", "whoami")
if err != nil {
return err
}
// Starts a container with a custom seccomp profile that denylists mkdir
// and ensures that mkdir fails.
cmd = []string{"docker", "run", "--rm", "-i", "--security-opt", fmt.Sprintf("seccomp=%s", seccompProfileFileName), image, "mkdir", "-p", "/tmp/foo"}
_, err = runCommand(cmd...)
if err == nil {
return fmt.Errorf("%q did not fail as expected", strings.Join(cmd, " "))
}
return nil
}
// checkDockerStorageDriver checks whether the current storage driver used by
// Docker is overlay.
func checkDockerStorageDriver() error {
output, err := runCommand("docker", "info")
if err != nil {
return err
}
for _, line := range strings.Split(string(output), "\n") {
if !strings.Contains(line, "Storage Driver:") {
continue
}
if !strings.Contains(line, "overlay") {
return fmt.Errorf("storage driver is not 'overlay': %s", line)
}
return nil
}
return fmt.Errorf("failed to find storage driver")
}
var _ = SIGDescribe("GKE system requirements [NodeConformance][Feature:GKEEnv][NodeFeature:GKEEnv]", func() {
ginkgo.BeforeEach(func() {
e2eskipper.RunIfSystemSpecNameIs("gke")
@@ -345,23 +132,6 @@ var _ = SIGDescribe("GKE system requirements [NodeConformance][Feature:GKEEnv][N
ginkgo.It("The GCR is accessible", func() {
framework.ExpectNoError(checkPublicGCR())
})
ginkgo.It("The docker configuration validation should pass", func() {
e2eskipper.RunIfContainerRuntimeIs("docker")
framework.ExpectNoError(checkDockerConfig())
})
ginkgo.It("The docker container network should work", func() {
e2eskipper.RunIfContainerRuntimeIs("docker")
framework.ExpectNoError(checkDockerNetworkServer())
framework.ExpectNoError(checkDockerNetworkClient())
})
ginkgo.It("The docker daemon should support AppArmor and seccomp", func() {
e2eskipper.RunIfContainerRuntimeIs("docker")
framework.ExpectNoError(checkDockerAppArmor())
framework.ExpectNoError(checkDockerSeccomp())
})
ginkgo.It("The docker storage driver should work", func() {
framework.ExpectNoError(checkDockerStorageDriver())
})
})
// getPPID returns the PPID for the pid.
@@ -423,21 +193,6 @@ func getCmdToProcessMap() (map[string][]process, error) {
return result, nil
}
// getKernelVersion returns the kernel version in the semantic version format.
func getKernelVersion() (*semver.Version, error) {
output, err := runCommand("uname", "-r")
if err != nil {
return nil, err
}
// An example 'output' could be "4.13.0-1001-gke".
v := strings.TrimSpace(strings.Split(output, "-")[0])
kernelVersion, err := semver.Make(v)
if err != nil {
return nil, fmt.Errorf("failed to convert %q to semantic version: %s", v, err)
}
return &kernelVersion, nil
}
// runCommand runs the cmd and returns the combined stdout and stderr, or an
// error if the command failed.
func runCommand(cmd ...string) (string, error) {

View File

@@ -20,7 +20,6 @@ import (
"context"
"fmt"
"os"
"os/exec"
"os/user"
"sync"
"time"
@@ -117,21 +116,6 @@ type puller interface {
Name() string
}
type dockerPuller struct {
}
func (dp *dockerPuller) Name() string {
return "docker"
}
func (dp *dockerPuller) Pull(image string) ([]byte, error) {
// TODO(random-liu): Use docker client to get rid of docker binary dependency.
if exec.Command("docker", "inspect", "--type=image", image).Run() != nil {
return exec.Command("docker", "pull", image).CombinedOutput()
}
return nil, nil
}
type remotePuller struct {
imageService internalapi.ImageManagerService
}
@@ -150,20 +134,13 @@ func (rp *remotePuller) Pull(image string) ([]byte, error) {
}
func getPuller() (puller, error) {
runtime := framework.TestContext.ContainerRuntime
switch runtime {
case "docker":
return &dockerPuller{}, nil
case "remote":
_, is, err := getCRIClient()
if err != nil {
return nil, err
}
return &remotePuller{
imageService: is,
}, nil
_, is, err := getCRIClient()
if err != nil {
return nil, err
}
return nil, fmt.Errorf("can't prepull images, unknown container runtime %q", runtime)
return &remotePuller{
imageService: is,
}, nil
}
// PrePullAllImages pre-fetches all images tests depend on so that we don't fail in an actual test.

View File

@@ -26,7 +26,6 @@ import (
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
"github.com/onsi/ginkgo"
)
@@ -116,29 +115,6 @@ var _ = SIGDescribe("ContainerLogPath [NodeConformance]", func() {
var logPodName string
ginkgo.BeforeEach(func() {
if framework.TestContext.ContainerRuntime == "docker" {
// Container Log Path support requires JSON logging driver.
// It does not work when Docker daemon is logging to journald.
d, err := getDockerLoggingDriver()
framework.ExpectNoError(err)
if d != "json-file" {
e2eskipper.Skipf("Skipping because Docker daemon is using a logging driver other than \"json-file\": %s", d)
}
// Even if JSON logging is in use, this test fails if SELinux support
// is enabled, since the isolation provided by the SELinux policy
// prevents processes running inside Docker containers (under SELinux
// type svirt_lxc_net_t) from accessing the log files which are owned
// by Docker (and labeled with the container_var_lib_t type.)
//
// Therefore, let's also skip this test when running with SELinux
// support enabled.
e, err := isDockerSELinuxSupportEnabled()
framework.ExpectNoError(err)
if e {
e2eskipper.Skipf("Skipping because Docker daemon is running with SELinux support enabled")
}
}
podClient = f.PodClient()
logPodName = "log-pod-" + string(uuid.NewUUID())
err := createAndWaitPod(makeLogPod(logPodName, logString))

View File

@@ -84,10 +84,6 @@ var _ = SIGDescribe("Restart [Serial] [Slow] [Disruptive]", func() {
ginkgo.Context("Container Runtime", func() {
ginkgo.Context("Network", func() {
ginkgo.It("should recover from ip leak", func() {
if framework.TestContext.ContainerRuntime == "docker" {
ginkgo.Skip("Test fails with in-tree docker. Skipping test.")
}
pods := newTestPods(podCount, false, imageutils.GetPauseImageName(), "restart-container-runtime-test")
ginkgo.By(fmt.Sprintf("Trying to create %d pods on node", len(pods)))
createBatchPodWithRateControl(f, pods, podCreationInterval)

View File

@@ -104,7 +104,7 @@ var _ = SIGDescribe("Kubelet PodOverhead handling [LinuxOnly]", func() {
handler string
)
ginkgo.By("Creating a RuntimeClass with Overhead definied", func() {
handler = e2enode.PreconfiguredRuntimeClassHandler(framework.TestContext.ContainerRuntime)
handler = e2enode.PreconfiguredRuntimeClassHandler
rc := &nodev1.RuntimeClass{
ObjectMeta: metav1.ObjectMeta{Name: handler},
Handler: handler,

View File

@@ -28,7 +28,6 @@ import (
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
imageutils "k8s.io/kubernetes/test/utils/image"
"github.com/onsi/ginkgo"
@@ -72,13 +71,6 @@ var _ = SIGDescribe("Security Context", func() {
})
ginkgo.It("processes in containers sharing a pod namespace should be able to see each other", func() {
ginkgo.By("Check whether shared PID namespace is supported.")
isEnabled, err := isSharedPIDNamespaceSupported()
framework.ExpectNoError(err)
if !isEnabled {
e2eskipper.Skipf("Skipped because shared PID namespace is not supported by this docker version.")
}
ginkgo.By("Create a pod with shared PID namespace.")
f.PodClient().CreateSync(&v1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "shared-pid-ns-test-pod"},

View File

@@ -19,7 +19,6 @@ package e2enode
import (
"fmt"
"io/ioutil"
"os/exec"
"strings"
"time"
@@ -136,29 +135,6 @@ var _ = SIGDescribe("Summary API [NodeConformance]", func() {
"MajorPageFaults": bounded(0, expectedMajorPageFaultsUpperBound),
})
runtimeContExpectations := sysContExpectations().(*gstruct.FieldsMatcher)
if systemdutil.IsRunningSystemd() && framework.TestContext.ContainerRuntime == "docker" {
// Some Linux distributions still ship a docker.service that is missing
// a `Delegate=yes` setting (or equivalent CPUAccounting= and MemoryAccounting=)
// that allows us to monitor the container runtime resource usage through
// the "cpu" and "memory" cgroups.
//
// Make an exception here for those distros, only for Docker, so that they
// can pass the full node e2e tests even in that case.
//
// For newer container runtimes (using CRI) and even distros that still
// ship Docker, we should encourage them to always set `Delegate=yes` in
// order to make monitoring of the runtime possible.
stdout, err := exec.Command("systemctl", "show", "-p", "Delegate", "docker.service").CombinedOutput()
if err == nil && strings.TrimSpace(string(stdout)) == "Delegate=no" {
// Only make these optional if we can successfully confirm that
// Delegate is set to "no" (in other words, unset.) If we fail
// to check that, default to requiring it, which might cause
// false positives, but that should be the safer approach.
ginkgo.By("Making runtime container expectations optional, since systemd was not configured to Delegate=yes the cgroups")
runtimeContExpectations.Fields["Memory"] = gomega.Or(gomega.BeNil(), runtimeContExpectations.Fields["Memory"])
runtimeContExpectations.Fields["CPU"] = gomega.Or(gomega.BeNil(), runtimeContExpectations.Fields["CPU"])
}
}
systemContainers := gstruct.Elements{
"kubelet": sysContExpectations(),
"runtime": runtimeContExpectations,