From 389c140eafb8f15ea306f30de9f85af47a546747 Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Wed, 3 May 2017 10:46:35 -0700 Subject: [PATCH] Move docker client code from dockertools to dockershim/dockerlib The code affected include DockerInterface (renamed to Interface), FakeDockerClient, etc. --- cmd/kubelet/app/server.go | 8 +- cmd/kubemark/hollow-node.go | 4 +- .../dockershim/cm/container_manager_linux.go | 7 +- .../cm/container_manager_unsupported.go | 4 +- pkg/kubelet/dockershim/convert.go | 16 +- pkg/kubelet/dockershim/docker_container.go | 8 +- pkg/kubelet/dockershim/docker_image.go | 7 +- pkg/kubelet/dockershim/docker_image_test.go | 13 +- pkg/kubelet/dockershim/docker_legacy.go | 4 +- pkg/kubelet/dockershim/docker_legacy_test.go | 14 +- pkg/kubelet/dockershim/docker_sandbox.go | 10 +- pkg/kubelet/dockershim/docker_sandbox_test.go | 8 +- pkg/kubelet/dockershim/docker_service.go | 25 +- pkg/kubelet/dockershim/docker_service_test.go | 8 +- pkg/kubelet/dockershim/docker_streaming.go | 13 +- pkg/kubelet/dockershim/exec.go | 11 +- pkg/kubelet/dockershim/helpers.go | 13 +- pkg/kubelet/dockershim/helpers_test.go | 5 +- pkg/kubelet/dockershim/libdocker/client.go | 120 ++++++ .../libdocker/fake_client.go} | 42 +- pkg/kubelet/dockershim/libdocker/helpers.go | 131 ++++++ .../dockershim/libdocker/helpers_test.go | 262 ++++++++++++ .../libdocker/instrumented_client.go} | 56 +-- .../libdocker}/kube_docker_client.go | 24 +- .../libdocker}/kube_docker_client_test.go | 2 +- pkg/kubelet/dockershim/libdocker/legacy.go | 92 +++++ .../dockershim/libdocker/legacy_test.go | 140 +++++++ pkg/kubelet/dockertools/docker.go | 271 +------------ pkg/kubelet/dockertools/docker_test.go | 380 +----------------- pkg/kubelet/gpu/nvidia/nvidia_gpu_manager.go | 6 +- pkg/kubelet/kubelet.go | 6 +- pkg/kubemark/hollow_kubelet.go | 4 +- test/e2e_node/garbage_collector_test.go | 8 +- 33 files changed, 923 insertions(+), 799 deletions(-) create mode 100644 pkg/kubelet/dockershim/libdocker/client.go rename pkg/kubelet/{dockertools/fake_docker_client.go => dockershim/libdocker/fake_client.go} (94%) create mode 100644 pkg/kubelet/dockershim/libdocker/helpers.go create mode 100644 pkg/kubelet/dockershim/libdocker/helpers_test.go rename pkg/kubelet/{dockertools/instrumented_docker.go => dockershim/libdocker/instrumented_client.go} (65%) rename pkg/kubelet/{dockertools => dockershim/libdocker}/kube_docker_client.go (95%) rename pkg/kubelet/{dockertools => dockershim/libdocker}/kube_docker_client_test.go (98%) create mode 100644 pkg/kubelet/dockershim/libdocker/legacy.go create mode 100644 pkg/kubelet/dockershim/libdocker/legacy_test.go diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index bb6aef10390..e242fb15d0c 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -72,8 +72,8 @@ import ( "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockershim" + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" dockerremote "k8s.io/kubernetes/pkg/kubelet/dockershim/remote" - "k8s.io/kubernetes/pkg/kubelet/dockertools" "k8s.io/kubernetes/pkg/kubelet/eviction" evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" "k8s.io/kubernetes/pkg/kubelet/server" @@ -143,9 +143,9 @@ func UnsecuredKubeletDeps(s *options.KubeletServer) (*kubelet.KubeletDeps, error writer = &kubeio.NsenterWriter{} } - var dockerClient dockertools.DockerInterface + var dockerClient libdocker.Interface if s.ContainerRuntime == "docker" { - dockerClient = dockertools.ConnectToDockerOrDie(s.DockerEndpoint, s.RuntimeRequestTimeout.Duration, + dockerClient = libdocker.ConnectToDockerOrDie(s.DockerEndpoint, s.RuntimeRequestTimeout.Duration, s.ImagePullProgressDeadline.Duration) } else { dockerClient = nil @@ -937,7 +937,7 @@ func parseResourceList(m componentconfig.ConfigurationMap) (v1.ResourceList, err // TODO(random-liu): Move this to a separate binary. func RunDockershim(c *componentconfig.KubeletConfiguration, dockershimRootDir string) error { // Create docker client. - dockerClient := dockertools.ConnectToDockerOrDie(c.DockerEndpoint, c.RuntimeRequestTimeout.Duration, + dockerClient := libdocker.ConnectToDockerOrDie(c.DockerEndpoint, c.RuntimeRequestTimeout.Duration, c.ImagePullProgressDeadline.Duration) // Initialize network plugin settings. diff --git a/cmd/kubemark/hollow-node.go b/cmd/kubemark/hollow-node.go index 1ef163dfe78..b26a846cd0a 100644 --- a/cmd/kubemark/hollow-node.go +++ b/cmd/kubemark/hollow-node.go @@ -33,7 +33,7 @@ import ( _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" "k8s.io/kubernetes/pkg/kubelet/cm" - "k8s.io/kubernetes/pkg/kubelet/dockertools" + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" "k8s.io/kubernetes/pkg/kubemark" fakeiptables "k8s.io/kubernetes/pkg/util/iptables/testing" _ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration @@ -113,7 +113,7 @@ func main() { cadvisorInterface := new(cadvisortest.Fake) containerManager := cm.NewStubContainerManager() - fakeDockerClient := dockertools.NewFakeDockerClient().WithTraceDisabled() + fakeDockerClient := libdocker.NewFakeDockerClient().WithTraceDisabled() fakeDockerClient.EnableSleep = true hollowKubelet := kubemark.NewHollowKubelet( diff --git a/pkg/kubelet/dockershim/cm/container_manager_linux.go b/pkg/kubelet/dockershim/cm/container_manager_linux.go index b9be890bff6..b820a8126a7 100644 --- a/pkg/kubelet/dockershim/cm/container_manager_linux.go +++ b/pkg/kubelet/dockershim/cm/container_manager_linux.go @@ -30,9 +30,10 @@ import ( "github.com/opencontainers/runc/libcontainer/configs" "k8s.io/apimachinery/pkg/util/wait" kubecm "k8s.io/kubernetes/pkg/kubelet/cm" - "k8s.io/kubernetes/pkg/kubelet/dockertools" "k8s.io/kubernetes/pkg/kubelet/qos" utilversion "k8s.io/kubernetes/pkg/util/version" + + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" ) const ( @@ -50,7 +51,7 @@ var ( memoryCapacityRegexp = regexp.MustCompile(`MemTotal:\s*([0-9]+) kB`) ) -func NewContainerManager(cgroupsName string, client dockertools.DockerInterface) ContainerManager { +func NewContainerManager(cgroupsName string, client libdocker.Interface) ContainerManager { return &containerManager{ cgroupsName: cgroupsName, client: client, @@ -59,7 +60,7 @@ func NewContainerManager(cgroupsName string, client dockertools.DockerInterface) type containerManager struct { // Docker client. - client dockertools.DockerInterface + client libdocker.Interface // Name of the cgroups. cgroupsName string // Manager for the cgroups. diff --git a/pkg/kubelet/dockershim/cm/container_manager_unsupported.go b/pkg/kubelet/dockershim/cm/container_manager_unsupported.go index ad0e5b80bfc..bab0d9884f7 100644 --- a/pkg/kubelet/dockershim/cm/container_manager_unsupported.go +++ b/pkg/kubelet/dockershim/cm/container_manager_unsupported.go @@ -21,13 +21,13 @@ package cm import ( "fmt" - "k8s.io/kubernetes/pkg/kubelet/dockertools" + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" ) type unsupportedContainerManager struct { } -func NewContainerManager(_ string, _ dockertools.DockerInterface) ContainerManager { +func NewContainerManager(_ string, _ libdocker.Interface) ContainerManager { return &unsupportedContainerManager{} } diff --git a/pkg/kubelet/dockershim/convert.go b/pkg/kubelet/dockershim/convert.go index f956936a5ef..cf9fb733334 100644 --- a/pkg/kubelet/dockershim/convert.go +++ b/pkg/kubelet/dockershim/convert.go @@ -24,18 +24,12 @@ import ( dockertypes "github.com/docker/engine-api/types" runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" ) // This file contains helper functions to convert docker API types to runtime // API types, or vice versa. -const ( - // Status of a container returned by docker ListContainers - statusRunningPrefix = "Up" - statusCreatedPrefix = "Created" - statusExitedPrefix = "Exited" -) - func imageToRuntimeAPIImage(image *dockertypes.Image) (*runtimeapi.Image, error) { if image == nil { return nil, fmt.Errorf("unable to convert a nil pointer to a runtime API image") @@ -126,11 +120,11 @@ func toRuntimeAPIContainerState(state string) runtimeapi.ContainerState { // Parse the state string in dockertypes.Container. This could break when // we upgrade docker. switch { - case strings.HasPrefix(state, statusRunningPrefix): + case strings.HasPrefix(state, libdocker.StatusRunningPrefix): return runtimeapi.ContainerState_CONTAINER_RUNNING - case strings.HasPrefix(state, statusExitedPrefix): + case strings.HasPrefix(state, libdocker.StatusExitedPrefix): return runtimeapi.ContainerState_CONTAINER_EXITED - case strings.HasPrefix(state, statusCreatedPrefix): + case strings.HasPrefix(state, libdocker.StatusCreatedPrefix): return runtimeapi.ContainerState_CONTAINER_CREATED default: return runtimeapi.ContainerState_CONTAINER_UNKNOWN @@ -141,7 +135,7 @@ func toRuntimeAPISandboxState(state string) runtimeapi.PodSandboxState { // Parse the state string in dockertypes.Container. This could break when // we upgrade docker. switch { - case strings.HasPrefix(state, statusRunningPrefix): + case strings.HasPrefix(state, libdocker.StatusRunningPrefix): return runtimeapi.PodSandboxState_SANDBOX_READY default: return runtimeapi.PodSandboxState_SANDBOX_NOTREADY diff --git a/pkg/kubelet/dockershim/docker_container.go b/pkg/kubelet/dockershim/docker_container.go index aaf59f42070..d63b1ce27b6 100644 --- a/pkg/kubelet/dockershim/docker_container.go +++ b/pkg/kubelet/dockershim/docker_container.go @@ -29,7 +29,7 @@ import ( "github.com/golang/glog" runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" - "k8s.io/kubernetes/pkg/kubelet/dockertools" + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" ) const ( @@ -314,15 +314,15 @@ func getContainerTimestamps(r *dockertypes.ContainerJSON) (time.Time, time.Time, var createdAt, startedAt, finishedAt time.Time var err error - createdAt, err = dockertools.ParseDockerTimestamp(r.Created) + createdAt, err = libdocker.ParseDockerTimestamp(r.Created) if err != nil { return createdAt, startedAt, finishedAt, err } - startedAt, err = dockertools.ParseDockerTimestamp(r.State.StartedAt) + startedAt, err = libdocker.ParseDockerTimestamp(r.State.StartedAt) if err != nil { return createdAt, startedAt, finishedAt, err } - finishedAt, err = dockertools.ParseDockerTimestamp(r.State.FinishedAt) + finishedAt, err = libdocker.ParseDockerTimestamp(r.State.FinishedAt) if err != nil { return createdAt, startedAt, finishedAt, err } diff --git a/pkg/kubelet/dockershim/docker_image.go b/pkg/kubelet/dockershim/docker_image.go index df59a7df28f..70bca7fa5e2 100644 --- a/pkg/kubelet/dockershim/docker_image.go +++ b/pkg/kubelet/dockershim/docker_image.go @@ -21,7 +21,8 @@ import ( dockertypes "github.com/docker/engine-api/types" runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" - "k8s.io/kubernetes/pkg/kubelet/dockertools" + + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" ) // This file implements methods in ImageManagerService. @@ -56,7 +57,7 @@ func (ds *dockerService) ListImages(filter *runtimeapi.ImageFilter) ([]*runtimea func (ds *dockerService) ImageStatus(image *runtimeapi.ImageSpec) (*runtimeapi.Image, error) { imageInspect, err := ds.client.InspectImageByRef(image.Image) if err != nil { - if dockertools.IsImageNotFoundError(err) { + if libdocker.IsImageNotFoundError(err) { return nil, nil } return nil, err @@ -105,7 +106,7 @@ func (ds *dockerService) RemoveImage(image *runtimeapi.ImageSpec) error { } // getImageRef returns the image digest if exists, or else returns the image ID. -func getImageRef(client dockertools.DockerInterface, image string) (string, error) { +func getImageRef(client libdocker.Interface, image string) (string, error) { img, err := client.InspectImageByRef(image) if err != nil { return "", err diff --git a/pkg/kubelet/dockershim/docker_image_test.go b/pkg/kubelet/dockershim/docker_image_test.go index 8096f2ca66b..2aeb5175a14 100644 --- a/pkg/kubelet/dockershim/docker_image_test.go +++ b/pkg/kubelet/dockershim/docker_image_test.go @@ -22,7 +22,8 @@ import ( dockertypes "github.com/docker/engine-api/types" runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" - "k8s.io/kubernetes/pkg/kubelet/dockertools" + + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" ) func TestRemoveImage(t *testing.T) { @@ -30,8 +31,8 @@ func TestRemoveImage(t *testing.T) { id := "1111" fakeDocker.InjectImageInspects([]dockertypes.ImageInspect{{ID: id, RepoTags: []string{"foo"}}}) ds.RemoveImage(&runtimeapi.ImageSpec{Image: id}) - fakeDocker.AssertCallDetails(dockertools.NewCalledDetail("inspect_image", nil), - dockertools.NewCalledDetail("remove_image", []interface{}{id, dockertypes.ImageRemoveOptions{PruneChildren: true}})) + fakeDocker.AssertCallDetails(libdocker.NewCalledDetail("inspect_image", nil), + libdocker.NewCalledDetail("remove_image", []interface{}{id, dockertypes.ImageRemoveOptions{PruneChildren: true}})) } func TestRemoveImageWithMultipleTags(t *testing.T) { @@ -39,7 +40,7 @@ func TestRemoveImageWithMultipleTags(t *testing.T) { id := "1111" fakeDocker.InjectImageInspects([]dockertypes.ImageInspect{{ID: id, RepoTags: []string{"foo", "bar"}}}) ds.RemoveImage(&runtimeapi.ImageSpec{Image: id}) - fakeDocker.AssertCallDetails(dockertools.NewCalledDetail("inspect_image", nil), - dockertools.NewCalledDetail("remove_image", []interface{}{"foo", dockertypes.ImageRemoveOptions{PruneChildren: true}}), - dockertools.NewCalledDetail("remove_image", []interface{}{"bar", dockertypes.ImageRemoveOptions{PruneChildren: true}})) + fakeDocker.AssertCallDetails(libdocker.NewCalledDetail("inspect_image", nil), + libdocker.NewCalledDetail("remove_image", []interface{}{"foo", dockertypes.ImageRemoveOptions{PruneChildren: true}}), + libdocker.NewCalledDetail("remove_image", []interface{}{"bar", dockertypes.ImageRemoveOptions{PruneChildren: true}})) } diff --git a/pkg/kubelet/dockershim/docker_legacy.go b/pkg/kubelet/dockershim/docker_legacy.go index c39890bcb5a..59488ed3aec 100644 --- a/pkg/kubelet/dockershim/docker_legacy.go +++ b/pkg/kubelet/dockershim/docker_legacy.go @@ -30,7 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" - "k8s.io/kubernetes/pkg/kubelet/dockertools" + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" "k8s.io/kubernetes/pkg/kubelet/leaky" ) @@ -70,7 +70,7 @@ func convertLegacyNameAndLabels(names []string, labels map[string]string) ([]str } // Generate new dockershim name. - m, _, err := dockertools.ParseDockerName(names[0]) + m, _, err := libdocker.ParseDockerName(names[0]) if err != nil { return nil, nil, err } diff --git a/pkg/kubelet/dockershim/docker_legacy_test.go b/pkg/kubelet/dockershim/docker_legacy_test.go index d0efb1da5ad..0b4551b60ba 100644 --- a/pkg/kubelet/dockershim/docker_legacy_test.go +++ b/pkg/kubelet/dockershim/docker_legacy_test.go @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "k8s.io/kubernetes/pkg/kubelet/dockertools" + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" "k8s.io/kubernetes/pkg/kubelet/types" ) @@ -105,8 +105,8 @@ func TestConvertLegacyNameAndLabels(t *testing.T) { } // getFakeLegacyContainers returns a list of fake legacy containers. -func getFakeLegacyContainers() []*dockertools.FakeContainer { - return []*dockertools.FakeContainer{ +func getFakeLegacyContainers() []*libdocker.FakeContainer { + return []*libdocker.FakeContainer{ { ID: "12", Name: "k8s_POD.hash1_podname_podnamespace_poduid_randomid", @@ -139,8 +139,8 @@ func getFakeLegacyContainers() []*dockertools.FakeContainer { } // getFakeNewContainers returns a list of fake new containers. -func getFakeNewContainers() []*dockertools.FakeContainer { - return []*dockertools.FakeContainer{ +func getFakeNewContainers() []*libdocker.FakeContainer { + return []*libdocker.FakeContainer{ { ID: "56", Name: "k8s_POD_podname_podnamespace_poduid_0", @@ -233,11 +233,11 @@ func TestListLegacyPodSandbox(t *testing.T) { func TestCheckLegacyCleanup(t *testing.T) { for desc, test := range map[string]struct { - containers []*dockertools.FakeContainer + containers []*libdocker.FakeContainer done bool }{ "no containers": { - containers: []*dockertools.FakeContainer{}, + containers: []*libdocker.FakeContainer{}, done: true, }, "only new containers": { diff --git a/pkg/kubelet/dockershim/docker_sandbox.go b/pkg/kubelet/dockershim/docker_sandbox.go index 3d56fb20c9f..b9cf1226b36 100644 --- a/pkg/kubelet/dockershim/docker_sandbox.go +++ b/pkg/kubelet/dockershim/docker_sandbox.go @@ -30,7 +30,7 @@ import ( runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockershim/errors" - "k8s.io/kubernetes/pkg/kubelet/dockertools" + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" "k8s.io/kubernetes/pkg/kubelet/qos" "k8s.io/kubernetes/pkg/kubelet/types" ) @@ -161,7 +161,7 @@ func (ds *dockerService) StopPodSandbox(podSandboxID string) error { // actions will only have sandbox ID and not have pod namespace and name information. // Return error if encounter any unexpected error. if checkpointErr != nil { - if dockertools.IsContainerNotFoundError(statusErr) && checkpointErr == errors.CheckpointNotFoundError { + if libdocker.IsContainerNotFoundError(statusErr) && checkpointErr == errors.CheckpointNotFoundError { glog.Warningf("Both sandbox container and checkpoint for id %q could not be found. "+ "Proceed without further sandbox information.", podSandboxID) } else { @@ -206,7 +206,7 @@ func (ds *dockerService) StopPodSandbox(podSandboxID string) error { if err := ds.client.StopContainer(podSandboxID, defaultSandboxGracePeriod); err != nil { glog.Errorf("Failed to stop sandbox %q: %v", podSandboxID, err) // Do not return error if the container does not exist - if !dockertools.IsContainerNotFoundError(err) { + if !libdocker.IsContainerNotFoundError(err) { errList = append(errList, err) } } @@ -231,13 +231,13 @@ func (ds *dockerService) RemovePodSandbox(podSandboxID string) error { // Remove all containers in the sandbox. for i := range containers { - if err := ds.RemoveContainer(containers[i].ID); err != nil && !dockertools.IsContainerNotFoundError(err) { + if err := ds.RemoveContainer(containers[i].ID); err != nil && !libdocker.IsContainerNotFoundError(err) { errs = append(errs, err) } } // Remove the sandbox container. - if err := ds.client.RemoveContainer(podSandboxID, dockertypes.ContainerRemoveOptions{RemoveVolumes: true}); err != nil && !dockertools.IsContainerNotFoundError(err) { + if err := ds.client.RemoveContainer(podSandboxID, dockertypes.ContainerRemoveOptions{RemoveVolumes: true}); err != nil && !libdocker.IsContainerNotFoundError(err) { errs = append(errs, err) } diff --git a/pkg/kubelet/dockershim/docker_sandbox_test.go b/pkg/kubelet/dockershim/docker_sandbox_test.go index 61bff303868..4f55d639cdc 100644 --- a/pkg/kubelet/dockershim/docker_sandbox_test.go +++ b/pkg/kubelet/dockershim/docker_sandbox_test.go @@ -27,7 +27,7 @@ import ( runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" - "k8s.io/kubernetes/pkg/kubelet/dockertools" + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" "k8s.io/kubernetes/pkg/kubelet/network" "k8s.io/kubernetes/pkg/kubelet/types" ) @@ -158,7 +158,7 @@ func TestNetworkPluginInvocation(t *testing.T) { map[string]string{"label": name}, map[string]string{"annotation": ns}, ) - cID := kubecontainer.ContainerID{Type: runtimeName, ID: dockertools.GetFakeContainerID(fmt.Sprintf("/%v", makeSandboxName(c)))} + cID := kubecontainer.ContainerID{Type: runtimeName, ID: libdocker.GetFakeContainerID(fmt.Sprintf("/%v", makeSandboxName(c)))} mockPlugin.EXPECT().Name().Return("mockNetworkPlugin").AnyTimes() setup := mockPlugin.EXPECT().SetUpPod(ns, name, cID) @@ -196,7 +196,7 @@ func TestHostNetworkPluginInvocation(t *testing.T) { }, }, } - cID := kubecontainer.ContainerID{Type: runtimeName, ID: dockertools.GetFakeContainerID(fmt.Sprintf("/%v", makeSandboxName(c)))} + cID := kubecontainer.ContainerID{Type: runtimeName, ID: libdocker.GetFakeContainerID(fmt.Sprintf("/%v", makeSandboxName(c)))} // No calls to network plugin are expected _, err := ds.RunPodSandbox(c) @@ -219,7 +219,7 @@ func TestSetUpPodFailure(t *testing.T) { map[string]string{"label": name}, map[string]string{"annotation": ns}, ) - cID := kubecontainer.ContainerID{Type: runtimeName, ID: dockertools.GetFakeContainerID(fmt.Sprintf("/%v", makeSandboxName(c)))} + cID := kubecontainer.ContainerID{Type: runtimeName, ID: libdocker.GetFakeContainerID(fmt.Sprintf("/%v", makeSandboxName(c)))} mockPlugin.EXPECT().Name().Return("mockNetworkPlugin").AnyTimes() mockPlugin.EXPECT().SetUpPod(ns, name, cID).Return(errors.New("setup pod error")).AnyTimes() // Assume network plugin doesn't return error, dockershim should still be able to return not ready correctly. diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index 7ea60931f96..f7b84bf29c1 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -36,23 +36,20 @@ import ( kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockershim/cm" "k8s.io/kubernetes/pkg/kubelet/dockershim/errors" - "k8s.io/kubernetes/pkg/kubelet/dockertools" "k8s.io/kubernetes/pkg/kubelet/network" "k8s.io/kubernetes/pkg/kubelet/network/cni" "k8s.io/kubernetes/pkg/kubelet/network/hostport" "k8s.io/kubernetes/pkg/kubelet/network/kubenet" "k8s.io/kubernetes/pkg/kubelet/server/streaming" "k8s.io/kubernetes/pkg/kubelet/util/cache" + + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" ) const ( dockerRuntimeName = "docker" kubeAPIVersion = "0.1.0" - // https://docs.docker.com/engine/reference/api/docker_remote_api/ - // docker version should be at least 1.10.x - minimumDockerAPIVersion = "1.22.0" - // String used to detect docker host mode for various namespaces (e.g. // networking). Must match the value returned by docker inspect -f // '{{.HostConfig.NetworkMode}}'. @@ -148,9 +145,9 @@ type dockerNetworkHost struct { var internalLabelKeys []string = []string{containerTypeLabelKey, containerLogPathLabelKey, sandboxIDLabelKey} // NOTE: Anything passed to DockerService should be eventually handled in another way when we switch to running the shim as a different process. -func NewDockerService(client dockertools.DockerInterface, seccompProfileRoot string, podSandboxImage string, streamingConfig *streaming.Config, +func NewDockerService(client libdocker.Interface, seccompProfileRoot string, podSandboxImage string, streamingConfig *streaming.Config, pluginSettings *NetworkPluginSettings, cgroupsName string, kubeCgroupDriver string, execHandlerName, dockershimRootDir string, disableSharedPID bool) (DockerService, error) { - c := dockertools.NewInstrumentedDockerInterface(client) + c := libdocker.NewInstrumentedInterface(client) checkpointHandler, err := NewPersistentCheckpointHandler(dockershimRootDir) if err != nil { return nil, err @@ -246,7 +243,7 @@ type DockerService interface { type dockerService struct { seccompProfileRoot string - client dockertools.DockerInterface + client libdocker.Interface os kubecontainer.OSInterface podSandboxImage string streamingRuntime *streamingRuntime @@ -413,7 +410,7 @@ func (ds *dockerService) checkVersionCompatibility() error { return err } - minAPIVersion, err := semver.Parse(minimumDockerAPIVersion) + minAPIVersion, err := semver.Parse(libdocker.MinimumDockerAPIVersion) if err != nil { return err } @@ -421,7 +418,7 @@ func (ds *dockerService) checkVersionCompatibility() error { // Verify the docker version. result := apiVersion.Compare(minAPIVersion) if result < 0 { - return fmt.Errorf("docker API version is older than %s", minimumDockerAPIVersion) + return fmt.Errorf("docker API version is older than %s", libdocker.MinimumDockerAPIVersion) } return nil @@ -478,10 +475,10 @@ type DockerLegacyService interface { // dockerLegacyService implements the DockerLegacyService. We add this for non json-log driver // support. (See #41996) type dockerLegacyService struct { - client dockertools.DockerInterface + client libdocker.Interface } -func NewDockerLegacyService(client dockertools.DockerInterface) DockerLegacyService { +func NewDockerLegacyService(client libdocker.Interface) DockerLegacyService { return &dockerLegacyService{client: client} } @@ -511,7 +508,7 @@ func (d *dockerLegacyService) GetContainerLogs(pod *v1.Pod, containerID kubecont opts.Tail = strconv.FormatInt(*logOptions.TailLines, 10) } - sopts := dockertools.StreamOptions{ + sopts := libdocker.StreamOptions{ OutputStream: stdout, ErrorStream: stderr, RawTerminal: container.Config.Tty, @@ -524,7 +521,7 @@ var criSupportedLogDrivers = []string{"json-file"} // IsCRISupportedLogDriver checks whether the logging driver used by docker is // suppoted by native CRI integration. -func IsCRISupportedLogDriver(client dockertools.DockerInterface) (bool, error) { +func IsCRISupportedLogDriver(client libdocker.Interface) (bool, error) { info, err := client.Info() if err != nil { return false, fmt.Errorf("failed to get docker info: %v", err) diff --git a/pkg/kubelet/dockershim/docker_service_test.go b/pkg/kubelet/dockershim/docker_service_test.go index 5bc92925d78..c5672092f18 100644 --- a/pkg/kubelet/dockershim/docker_service_test.go +++ b/pkg/kubelet/dockershim/docker_service_test.go @@ -30,7 +30,7 @@ import ( "k8s.io/client-go/util/clock" runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" - "k8s.io/kubernetes/pkg/kubelet/dockertools" + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" "k8s.io/kubernetes/pkg/kubelet/network" nettest "k8s.io/kubernetes/pkg/kubelet/network/testing" "k8s.io/kubernetes/pkg/kubelet/util/cache" @@ -42,15 +42,15 @@ func newTestNetworkPlugin(t *testing.T) *nettest.MockNetworkPlugin { return nettest.NewMockNetworkPlugin(ctrl) } -func newTestDockerService() (*dockerService, *dockertools.FakeDockerClient, *clock.FakeClock) { +func newTestDockerService() (*dockerService, *libdocker.FakeDockerClient, *clock.FakeClock) { fakeClock := clock.NewFakeClock(time.Time{}) - c := dockertools.NewFakeDockerClient().WithClock(fakeClock).WithVersion("1.11.2", "1.23") + c := libdocker.NewFakeDockerClient().WithClock(fakeClock).WithVersion("1.11.2", "1.23") pm := network.NewPluginManager(&network.NoopNetworkPlugin{}) return &dockerService{client: c, os: &containertest.FakeOS{}, network: pm, legacyCleanup: legacyCleanupFlag{done: 1}, checkpointHandler: NewTestPersistentCheckpointHandler()}, c, fakeClock } -func newTestDockerServiceWithVersionCache() (*dockerService, *dockertools.FakeDockerClient, *clock.FakeClock) { +func newTestDockerServiceWithVersionCache() (*dockerService, *libdocker.FakeDockerClient, *clock.FakeClock) { ds, c, fakeClock := newTestDockerService() ds.versionCache = cache.NewObjectCache( func() (interface{}, error) { diff --git a/pkg/kubelet/dockershim/docker_streaming.go b/pkg/kubelet/dockershim/docker_streaming.go index 262cadfee0a..a1a06606c76 100644 --- a/pkg/kubelet/dockershim/docker_streaming.go +++ b/pkg/kubelet/dockershim/docker_streaming.go @@ -30,13 +30,14 @@ import ( "k8s.io/kubernetes/pkg/client/unversioned/remotecommand" runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" - "k8s.io/kubernetes/pkg/kubelet/dockertools" "k8s.io/kubernetes/pkg/kubelet/server/streaming" "k8s.io/kubernetes/pkg/kubelet/util/ioutils" + + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" ) type streamingRuntime struct { - client dockertools.DockerInterface + client libdocker.Interface execHandler ExecHandler } @@ -122,7 +123,7 @@ func (ds *dockerService) PortForward(req *runtimeapi.PortForwardRequest) (*runti return ds.streamingServer.GetPortForward(req) } -func checkContainerStatus(client dockertools.DockerInterface, containerID string) (*dockertypes.ContainerJSON, error) { +func checkContainerStatus(client libdocker.Interface, containerID string) (*dockertypes.ContainerJSON, error) { container, err := client.InspectContainer(containerID) if err != nil { return nil, err @@ -133,7 +134,7 @@ func checkContainerStatus(client dockertools.DockerInterface, containerID string return container, nil } -func attachContainer(client dockertools.DockerInterface, containerID string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error { +func attachContainer(client libdocker.Interface, containerID string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error { // Have to start this before the call to client.AttachToContainer because client.AttachToContainer is a blocking // call :-( Otherwise, resize events don't get processed and the terminal never resizes. kubecontainer.HandleResizing(resize, func(size remotecommand.TerminalSize) { @@ -147,7 +148,7 @@ func attachContainer(client dockertools.DockerInterface, containerID string, std Stdout: stdout != nil, Stderr: stderr != nil, } - sopts := dockertools.StreamOptions{ + sopts := libdocker.StreamOptions{ InputStream: stdin, OutputStream: stdout, ErrorStream: stderr, @@ -156,7 +157,7 @@ func attachContainer(client dockertools.DockerInterface, containerID string, std return client.AttachToContainer(containerID, opts, sopts) } -func portForward(client dockertools.DockerInterface, podInfraContainerID string, port int32, stream io.ReadWriteCloser) error { +func portForward(client libdocker.Interface, podInfraContainerID string, port int32, stream io.ReadWriteCloser) error { container, err := client.InspectContainer(podInfraContainerID) if err != nil { return err diff --git a/pkg/kubelet/dockershim/exec.go b/pkg/kubelet/dockershim/exec.go index aba59ae25f7..3aee5e045fb 100644 --- a/pkg/kubelet/dockershim/exec.go +++ b/pkg/kubelet/dockershim/exec.go @@ -28,14 +28,15 @@ import ( "k8s.io/kubernetes/pkg/client/unversioned/remotecommand" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" - "k8s.io/kubernetes/pkg/kubelet/dockertools" utilexec "k8s.io/kubernetes/pkg/util/exec" "k8s.io/kubernetes/pkg/util/term" + + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" ) // ExecHandler knows how to execute a command in a running Docker container. type ExecHandler interface { - ExecInContainer(client dockertools.DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error + ExecInContainer(client libdocker.Interface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error } // NsenterExecHandler executes commands in Docker containers using nsenter. @@ -62,7 +63,7 @@ func (d *dockerExitError) ExitStatus() int { } // TODO should we support nsenter in a container, running with elevated privs and --pid=host? -func (*NsenterExecHandler) ExecInContainer(client dockertools.DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error { +func (*NsenterExecHandler) ExecInContainer(client libdocker.Interface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error { nsenter, err := exec.LookPath("nsenter") if err != nil { return fmt.Errorf("exec unavailable - unable to locate nsenter") @@ -133,7 +134,7 @@ func (*NsenterExecHandler) ExecInContainer(client dockertools.DockerInterface, c // NativeExecHandler executes commands in Docker containers using Docker's exec API. type NativeExecHandler struct{} -func (*NativeExecHandler) ExecInContainer(client dockertools.DockerInterface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error { +func (*NativeExecHandler) ExecInContainer(client libdocker.Interface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error { createOpts := dockertypes.ExecConfig{ Cmd: cmd, AttachStdin: stdin != nil, @@ -153,7 +154,7 @@ func (*NativeExecHandler) ExecInContainer(client dockertools.DockerInterface, co }) startOpts := dockertypes.ExecStartCheck{Detach: false, Tty: tty} - streamOpts := dockertools.StreamOptions{ + streamOpts := libdocker.StreamOptions{ InputStream: stdin, OutputStream: stdout, ErrorStream: stderr, diff --git a/pkg/kubelet/dockershim/helpers.go b/pkg/kubelet/dockershim/helpers.go index 0927ad35b45..d1918c9489f 100644 --- a/pkg/kubelet/dockershim/helpers.go +++ b/pkg/kubelet/dockershim/helpers.go @@ -36,9 +36,10 @@ import ( "k8s.io/kubernetes/pkg/api/v1" v1helper "k8s.io/kubernetes/pkg/api/v1/helper" runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" - "k8s.io/kubernetes/pkg/kubelet/dockertools" "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/security/apparmor" + + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" ) const ( @@ -354,7 +355,7 @@ func getUserFromImageUser(imageUser string) (*int64, string) { // In that case we have to create the container with a randomized name. // TODO(random-liu): Remove this work around after docker 1.11 is deprecated. // TODO(#33189): Monitor the tests to see if the fix is sufficient. -func recoverFromCreationConflictIfNeeded(client dockertools.DockerInterface, createConfig dockertypes.ContainerCreateConfig, err error) (*dockertypes.ContainerCreateResponse, error) { +func recoverFromCreationConflictIfNeeded(client libdocker.Interface, createConfig dockertypes.ContainerCreateConfig, err error) (*dockertypes.ContainerCreateResponse, error) { matches := conflictRE.FindStringSubmatch(err.Error()) if len(matches) != 2 { return nil, err @@ -368,7 +369,7 @@ func recoverFromCreationConflictIfNeeded(client dockertools.DockerInterface, cre } else { glog.Errorf("Failed to remove the conflicting container %q: %v", id, rmErr) // Return if the error is not container not found error. - if !dockertools.IsContainerNotFoundError(rmErr) { + if !libdocker.IsContainerNotFoundError(rmErr) { return nil, err } } @@ -395,12 +396,12 @@ func getSecurityOptSeparator(v *semver.Version) rune { } // ensureSandboxImageExists pulls the sandbox image when it's not present. -func ensureSandboxImageExists(client dockertools.DockerInterface, image string) error { +func ensureSandboxImageExists(client libdocker.Interface, image string) error { _, err := client.InspectImageByRef(image) if err == nil { return nil } - if !dockertools.IsImageNotFoundError(err) { + if !libdocker.IsImageNotFoundError(err) { return fmt.Errorf("failed to inspect sandbox image %q: %v", image, err) } err = client.PullImage(image, dockertypes.AuthConfig{}, dockertypes.ImagePullOptions{}) @@ -437,7 +438,7 @@ type dockerOpt struct { msg string } -// Expose key/value from dockertools +// Expose key/value from dockerOpt. func (d dockerOpt) GetKV() (string, string) { return d.key, d.value } diff --git a/pkg/kubelet/dockershim/helpers_test.go b/pkg/kubelet/dockershim/helpers_test.go index 12cda3e413c..a5ea088e296 100644 --- a/pkg/kubelet/dockershim/helpers_test.go +++ b/pkg/kubelet/dockershim/helpers_test.go @@ -29,8 +29,9 @@ import ( "k8s.io/kubernetes/pkg/api/v1" runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" - "k8s.io/kubernetes/pkg/kubelet/dockertools" "k8s.io/kubernetes/pkg/security/apparmor" + + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" ) func TestLabelsAndAnnotationsRoundTrip(t *testing.T) { @@ -302,7 +303,7 @@ func TestEnsureSandboxImageExists(t *testing.T) { }, "should pull image when it doesn't exist": { injectImage: false, - injectErr: dockertools.ImageNotFoundError{ID: "image_id"}, + injectErr: libdocker.ImageNotFoundError{ID: "image_id"}, calls: []string{"inspect_image", "pull"}, }, "should return error when inspect image fails": { diff --git a/pkg/kubelet/dockershim/libdocker/client.go b/pkg/kubelet/dockershim/libdocker/client.go new file mode 100644 index 00000000000..cc71a68247d --- /dev/null +++ b/pkg/kubelet/dockershim/libdocker/client.go @@ -0,0 +1,120 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package libdocker + +import ( + "strings" + "time" + + dockerapi "github.com/docker/engine-api/client" + dockertypes "github.com/docker/engine-api/types" + "github.com/golang/glog" +) + +const ( + // https://docs.docker.com/engine/reference/api/docker_remote_api/ + // docker version should be at least 1.10.x + MinimumDockerAPIVersion = "1.22.0" + + // Status of a container returned by ListContainers. + StatusRunningPrefix = "Up" + StatusCreatedPrefix = "Created" + StatusExitedPrefix = "Exited" + + // This is only used by GetKubeletDockerContainers(), and should be removed + // along with the function. + containerNamePrefix = "k8s" +) + +// Interface is an abstract interface for testability. It abstracts the interface of docker client. +type Interface interface { + ListContainers(options dockertypes.ContainerListOptions) ([]dockertypes.Container, error) + InspectContainer(id string) (*dockertypes.ContainerJSON, error) + CreateContainer(dockertypes.ContainerCreateConfig) (*dockertypes.ContainerCreateResponse, error) + StartContainer(id string) error + StopContainer(id string, timeout int) error + RemoveContainer(id string, opts dockertypes.ContainerRemoveOptions) error + InspectImageByRef(imageRef string) (*dockertypes.ImageInspect, error) + InspectImageByID(imageID string) (*dockertypes.ImageInspect, error) + ListImages(opts dockertypes.ImageListOptions) ([]dockertypes.Image, error) + PullImage(image string, auth dockertypes.AuthConfig, opts dockertypes.ImagePullOptions) error + RemoveImage(image string, opts dockertypes.ImageRemoveOptions) ([]dockertypes.ImageDelete, error) + ImageHistory(id string) ([]dockertypes.ImageHistory, error) + Logs(string, dockertypes.ContainerLogsOptions, StreamOptions) error + Version() (*dockertypes.Version, error) + Info() (*dockertypes.Info, error) + CreateExec(string, dockertypes.ExecConfig) (*dockertypes.ContainerExecCreateResponse, error) + StartExec(string, dockertypes.ExecStartCheck, StreamOptions) error + InspectExec(id string) (*dockertypes.ContainerExecInspect, error) + AttachToContainer(string, dockertypes.ContainerAttachOptions, StreamOptions) error + ResizeContainerTTY(id string, height, width int) error + ResizeExecTTY(id string, height, width int) error +} + +// Get a *dockerapi.Client, either using the endpoint passed in, or using +// DOCKER_HOST, DOCKER_TLS_VERIFY, and DOCKER_CERT path per their spec +func getDockerClient(dockerEndpoint string) (*dockerapi.Client, error) { + if len(dockerEndpoint) > 0 { + glog.Infof("Connecting to docker on %s", dockerEndpoint) + return dockerapi.NewClient(dockerEndpoint, "", nil, nil) + } + return dockerapi.NewEnvClient() +} + +// ConnectToDockerOrDie creates docker client connecting to docker daemon. +// If the endpoint passed in is "fake://", a fake docker client +// will be returned. The program exits if error occurs. The requestTimeout +// is the timeout for docker requests. If timeout is exceeded, the request +// will be cancelled and throw out an error. If requestTimeout is 0, a default +// value will be applied. +func ConnectToDockerOrDie(dockerEndpoint string, requestTimeout, imagePullProgressDeadline time.Duration) Interface { + if dockerEndpoint == "fake://" { + return NewFakeDockerClient() + } + client, err := getDockerClient(dockerEndpoint) + if err != nil { + glog.Fatalf("Couldn't connect to docker: %v", err) + } + glog.Infof("Start docker client with request timeout=%v", requestTimeout) + return newKubeDockerClient(client, requestTimeout, imagePullProgressDeadline) +} + +// GetKubeletDockerContainers lists all container or just the running ones. +// Returns a list of docker containers that we manage +// TODO: This function should be deleted after migrating +// test/e2e_node/garbage_collector_test.go off of it. +func GetKubeletDockerContainers(client Interface, allContainers bool) ([]*dockertypes.Container, error) { + result := []*dockertypes.Container{} + containers, err := client.ListContainers(dockertypes.ContainerListOptions{All: allContainers}) + if err != nil { + return nil, err + } + for i := range containers { + container := &containers[i] + if len(container.Names) == 0 { + continue + } + // Skip containers that we didn't create to allow users to manually + // spin up their own containers if they want. + if !strings.HasPrefix(container.Names[0], "/"+containerNamePrefix+"_") { + glog.V(5).Infof("Docker Container: %s is not managed by kubelet.", container.Names[0]) + continue + } + result = append(result, container) + } + return result, nil +} diff --git a/pkg/kubelet/dockertools/fake_docker_client.go b/pkg/kubelet/dockershim/libdocker/fake_client.go similarity index 94% rename from pkg/kubelet/dockertools/fake_docker_client.go rename to pkg/kubelet/dockershim/libdocker/fake_client.go index 5830cb4fa19..b85b416beef 100644 --- a/pkg/kubelet/dockertools/fake_docker_client.go +++ b/pkg/kubelet/dockershim/libdocker/fake_client.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package dockertools +package libdocker import ( "encoding/json" @@ -77,16 +77,16 @@ type FakeDockerClient struct { } const ( - // We don't check docker version now, just set the docker version of fake docker client to 1.8.1. // Notice that if someday we also have minimum docker version requirement, this should also be updated. - fakeDockerVersion = "1.8.1" + fakeDockerVersion = "1.11.2" fakeImageSize = 1024 ) func NewFakeDockerClient() *FakeDockerClient { return &FakeDockerClient{ - VersionInfo: dockertypes.Version{Version: fakeDockerVersion, APIVersion: minimumDockerAPIVersion}, + // Docker's API version does not include the patch number. + VersionInfo: dockertypes.Version{Version: fakeDockerVersion, APIVersion: strings.TrimSuffix(MinimumDockerAPIVersion, ".0")}, Errors: make(map[string]error), ContainerMap: make(map[string]*dockertypes.ContainerJSON), Clock: clock.RealClock{}, @@ -345,6 +345,14 @@ func (f *FakeDockerClient) AssertImagesPulled(pulled []string) error { return sortedStringSlicesEqual(pulled, actualPulled) } +func (f *FakeDockerClient) AssertImagesPulledMsgs(expected []string) error { + f.Lock() + defer f.Unlock() + // Copy pulled to avoid modifying it. + actual := append([]string{}, f.pulled...) + return sortedStringSlicesEqual(expected, actual) +} + func sortedStringSlicesEqual(expected, actual []string) error { sort.StringSlice(expected).Sort() sort.StringSlice(actual).Sort() @@ -367,7 +375,7 @@ func (f *FakeDockerClient) popError(op string) error { } } -// ListContainers is a test-spy implementation of DockerInterface.ListContainers. +// ListContainers is a test-spy implementation of Interface.ListContainers. // It adds an entry "list" to the internal method call record. func (f *FakeDockerClient) ListContainers(options dockertypes.ContainerListOptions) ([]dockertypes.Container, error) { f.Lock() @@ -434,7 +442,7 @@ func (f *FakeDockerClient) ListContainers(options dockertypes.ContainerListOptio return containerList, err } -// InspectContainer is a test-spy implementation of DockerInterface.InspectContainer. +// InspectContainer is a test-spy implementation of Interface.InspectContainer. // It adds an entry "inspect" to the internal method call record. func (f *FakeDockerClient) InspectContainer(id string) (*dockertypes.ContainerJSON, error) { f.Lock() @@ -451,7 +459,7 @@ func (f *FakeDockerClient) InspectContainer(id string) (*dockertypes.ContainerJS return nil, fmt.Errorf("container %q not found", id) } -// InspectImageByRef is a test-spy implementation of DockerInterface.InspectImageByRef. +// InspectImageByRef is a test-spy implementation of Interface.InspectImageByRef. // It adds an entry "inspect" to the internal method call record. func (f *FakeDockerClient) InspectImageByRef(name string) (*dockertypes.ImageInspect, error) { f.Lock() @@ -466,7 +474,7 @@ func (f *FakeDockerClient) InspectImageByRef(name string) (*dockertypes.ImageIns return nil, ImageNotFoundError{name} } -// InspectImageByID is a test-spy implementation of DockerInterface.InspectImageByID. +// InspectImageByID is a test-spy implementation of Interface.InspectImageByID. // It adds an entry "inspect" to the internal method call record. func (f *FakeDockerClient) InspectImageByID(name string) (*dockertypes.ImageInspect, error) { f.Lock() @@ -502,7 +510,7 @@ func GetFakeContainerID(name string) string { return strconv.FormatUint(hash.Sum64(), 16) } -// CreateContainer is a test-spy implementation of DockerInterface.CreateContainer. +// CreateContainer is a test-spy implementation of Interface.CreateContainer. // It adds an entry "create" to the internal method call record. func (f *FakeDockerClient) CreateContainer(c dockertypes.ContainerCreateConfig) (*dockertypes.ContainerCreateResponse, error) { f.Lock() @@ -519,7 +527,7 @@ func (f *FakeDockerClient) CreateContainer(c dockertypes.ContainerCreateConfig) timestamp := f.Clock.Now() // The newest container should be in front, because we assume so in GetPodStatus() f.RunningContainerList = append([]dockertypes.Container{ - {ID: id, Names: []string{name}, Image: c.Config.Image, Created: timestamp.Unix(), State: statusCreatedPrefix, Labels: c.Config.Labels}, + {ID: id, Names: []string{name}, Image: c.Config.Image, Created: timestamp.Unix(), State: StatusCreatedPrefix, Labels: c.Config.Labels}, }, f.RunningContainerList...) f.ContainerMap[id] = convertFakeContainer(&FakeContainer{ ID: id, Name: name, Config: c.Config, HostConfig: c.HostConfig, CreatedAt: timestamp}) @@ -529,7 +537,7 @@ func (f *FakeDockerClient) CreateContainer(c dockertypes.ContainerCreateConfig) return &dockertypes.ContainerCreateResponse{ID: id}, nil } -// StartContainer is a test-spy implementation of DockerInterface.StartContainer. +// StartContainer is a test-spy implementation of Interface.StartContainer. // It adds an entry "start" to the internal method call record. func (f *FakeDockerClient) StartContainer(id string) error { f.Lock() @@ -549,12 +557,12 @@ func (f *FakeDockerClient) StartContainer(id string) error { container.State.StartedAt = dockerTimestampToString(timestamp) container.NetworkSettings.IPAddress = "2.3.4.5" f.ContainerMap[id] = container - f.updateContainerStatus(id, statusRunningPrefix) + f.updateContainerStatus(id, StatusRunningPrefix) f.normalSleep(200, 50, 50) return nil } -// StopContainer is a test-spy implementation of DockerInterface.StopContainer. +// StopContainer is a test-spy implementation of Interface.StopContainer. // It adds an entry "stop" to the internal method call record. func (f *FakeDockerClient) StopContainer(id string, timeout int) error { f.Lock() @@ -565,7 +573,7 @@ func (f *FakeDockerClient) StopContainer(id string, timeout int) error { } f.appendContainerTrace("Stopped", id) // Container status should be Updated before container moved to ExitedContainerList - f.updateContainerStatus(id, statusExitedPrefix) + f.updateContainerStatus(id, StatusExitedPrefix) var newList []dockertypes.Container for _, container := range f.RunningContainerList { if container.ID == id { @@ -615,7 +623,7 @@ func (f *FakeDockerClient) RemoveContainer(id string, opts dockertypes.Container return fmt.Errorf("container not stopped") } -// Logs is a test-spy implementation of DockerInterface.Logs. +// Logs is a test-spy implementation of Interface.Logs. // It adds an entry "logs" to the internal method call record. func (f *FakeDockerClient) Logs(id string, opts dockertypes.ContainerLogsOptions, sopts StreamOptions) error { f.Lock() @@ -624,7 +632,7 @@ func (f *FakeDockerClient) Logs(id string, opts dockertypes.ContainerLogsOptions return f.popError("logs") } -// PullImage is a test-spy implementation of DockerInterface.PullImage. +// PullImage is a test-spy implementation of Interface.PullImage. // It adds an entry "pull" to the internal method call record. func (f *FakeDockerClient) PullImage(image string, auth dockertypes.AuthConfig, opts dockertypes.ImagePullOptions) error { f.Lock() @@ -804,7 +812,7 @@ func (f *FakeDockerClient) InjectImageHistory(data map[string][]dockertypes.Imag // FakeDockerPuller is meant to be a simple wrapper around FakeDockerClient. // Please do not add more functionalities to it. type FakeDockerPuller struct { - client DockerInterface + client Interface } func (f *FakeDockerPuller) Pull(image string, _ []v1.Secret) error { diff --git a/pkg/kubelet/dockershim/libdocker/helpers.go b/pkg/kubelet/dockershim/libdocker/helpers.go new file mode 100644 index 00000000000..28c69a22ace --- /dev/null +++ b/pkg/kubelet/dockershim/libdocker/helpers.go @@ -0,0 +1,131 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package libdocker + +import ( + "strings" + "time" + + dockerdigest "github.com/docker/distribution/digest" + dockerref "github.com/docker/distribution/reference" + dockertypes "github.com/docker/engine-api/types" + "github.com/golang/glog" +) + +// ParseDockerTimestamp parses the timestamp returned by Interface from string to time.Time +func ParseDockerTimestamp(s string) (time.Time, error) { + // Timestamp returned by Docker is in time.RFC3339Nano format. + return time.Parse(time.RFC3339Nano, s) +} + +// matchImageTagOrSHA checks if the given image specifier is a valid image ref, +// and that it matches the given image. It should fail on things like image IDs +// (config digests) and other digest-only references, but succeed on image names +// (`foo`), tag references (`foo:bar`), and manifest digest references +// (`foo@sha256:xyz`). +func matchImageTagOrSHA(inspected dockertypes.ImageInspect, image string) bool { + // The image string follows the grammar specified here + // https://github.com/docker/distribution/blob/master/reference/reference.go#L4 + named, err := dockerref.ParseNamed(image) + if err != nil { + glog.V(4).Infof("couldn't parse image reference %q: %v", image, err) + return false + } + _, isTagged := named.(dockerref.Tagged) + digest, isDigested := named.(dockerref.Digested) + if !isTagged && !isDigested { + // No Tag or SHA specified, so just return what we have + return true + } + + if isTagged { + // Check the RepoTags for a match. + for _, tag := range inspected.RepoTags { + // An image name (without the tag/digest) can be [hostname '/'] component ['/' component]* + // Because either the RepoTag or the name *may* contain the + // hostname or not, we only check for the suffix match. + if strings.HasSuffix(image, tag) || strings.HasSuffix(tag, image) { + return true + } + } + } + + if isDigested { + for _, repoDigest := range inspected.RepoDigests { + named, err := dockerref.ParseNamed(repoDigest) + if err != nil { + glog.V(4).Infof("couldn't parse image RepoDigest reference %q: %v", repoDigest, err) + continue + } + if d, isDigested := named.(dockerref.Digested); isDigested { + if digest.Digest().Algorithm().String() == d.Digest().Algorithm().String() && + digest.Digest().Hex() == d.Digest().Hex() { + return true + } + } + } + + // process the ID as a digest + id, err := dockerdigest.ParseDigest(inspected.ID) + if err != nil { + glog.V(4).Infof("couldn't parse image ID reference %q: %v", id, err) + return false + } + if digest.Digest().Algorithm().String() == id.Algorithm().String() && digest.Digest().Hex() == id.Hex() { + return true + } + } + glog.V(4).Infof("Inspected image (%q) does not match %s", inspected.ID, image) + return false +} + +// matchImageIDOnly checks that the given image specifier is a digest-only +// reference, and that it matches the given image. +func matchImageIDOnly(inspected dockertypes.ImageInspect, image string) bool { + // If the image ref is literally equal to the inspected image's ID, + // just return true here (this might be the case for Docker 1.9, + // where we won't have a digest for the ID) + if inspected.ID == image { + return true + } + + // Otherwise, we should try actual parsing to be more correct + ref, err := dockerref.Parse(image) + if err != nil { + glog.V(4).Infof("couldn't parse image reference %q: %v", image, err) + return false + } + + digest, isDigested := ref.(dockerref.Digested) + if !isDigested { + glog.V(4).Infof("the image reference %q was not a digest reference") + return false + } + + id, err := dockerdigest.ParseDigest(inspected.ID) + if err != nil { + glog.V(4).Infof("couldn't parse image ID reference %q: %v", id, err) + return false + } + + if digest.Digest().Algorithm().String() == id.Algorithm().String() && digest.Digest().Hex() == id.Hex() { + return true + } + + glog.V(4).Infof("The reference %s does not directly refer to the given image's ID (%q)", image, inspected.ID) + return false +} diff --git a/pkg/kubelet/dockershim/libdocker/helpers_test.go b/pkg/kubelet/dockershim/libdocker/helpers_test.go new file mode 100644 index 00000000000..430db01f9f1 --- /dev/null +++ b/pkg/kubelet/dockershim/libdocker/helpers_test.go @@ -0,0 +1,262 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package libdocker + +import ( + "fmt" + "testing" + + dockertypes "github.com/docker/engine-api/types" + "github.com/stretchr/testify/assert" +) + +func TestMatchImageTagOrSHA(t *testing.T) { + for i, testCase := range []struct { + Inspected dockertypes.ImageInspect + Image string + Output bool + }{ + { + Inspected: dockertypes.ImageInspect{RepoTags: []string{"ubuntu:latest"}}, + Image: "ubuntu", + Output: true, + }, + { + Inspected: dockertypes.ImageInspect{RepoTags: []string{"ubuntu:14.04"}}, + Image: "ubuntu:latest", + Output: false, + }, + { + Inspected: dockertypes.ImageInspect{RepoTags: []string{"colemickens/hyperkube-amd64:217.9beff63"}}, + Image: "colemickens/hyperkube-amd64:217.9beff63", + Output: true, + }, + { + Inspected: dockertypes.ImageInspect{RepoTags: []string{"colemickens/hyperkube-amd64:217.9beff63"}}, + Image: "docker.io/colemickens/hyperkube-amd64:217.9beff63", + Output: true, + }, + { + Inspected: dockertypes.ImageInspect{RepoTags: []string{"docker.io/kubernetes/pause:latest"}}, + Image: "kubernetes/pause:latest", + Output: true, + }, + { + Inspected: dockertypes.ImageInspect{ + ID: "sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", + }, + Image: "myimage@sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", + Output: true, + }, + { + Inspected: dockertypes.ImageInspect{ + ID: "sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", + }, + Image: "myimage@sha256:2208f7a29005", + Output: false, + }, + { + Inspected: dockertypes.ImageInspect{ + ID: "sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", + }, + Image: "myimage@sha256:2208", + Output: false, + }, + { + // mismatched ID is ignored + Inspected: dockertypes.ImageInspect{ + ID: "sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", + }, + Image: "myimage@sha256:0000f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", + Output: false, + }, + { + // invalid digest is ignored + Inspected: dockertypes.ImageInspect{ + ID: "sha256:unparseable", + }, + Image: "myimage@sha256:unparseable", + Output: false, + }, + { + // v1 schema images can be pulled in one format and returned in another + Inspected: dockertypes.ImageInspect{ + ID: "sha256:9bbdf247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227", + RepoDigests: []string{"centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf"}, + }, + Image: "centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf", + Output: true, + }, + { + // RepoDigest match is is required + Inspected: dockertypes.ImageInspect{ + ID: "", + RepoDigests: []string{"docker.io/centos/ruby-23-centos7@sha256:000084acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf"}, + }, + Image: "centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf", + Output: false, + }, + { + // RepoDigest match is allowed + Inspected: dockertypes.ImageInspect{ + ID: "sha256:9bbdf247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227", + RepoDigests: []string{"docker.io/centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf"}, + }, + Image: "centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf", + Output: true, + }, + { + // RepoDigest and ID are checked + Inspected: dockertypes.ImageInspect{ + ID: "sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf", + RepoDigests: []string{"docker.io/centos/ruby-23-centos7@sha256:9bbdf247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227"}, + }, + Image: "centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf", + Output: true, + }, + { + // unparseable RepoDigests are skipped + Inspected: dockertypes.ImageInspect{ + ID: "sha256:9bbdf247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227", + RepoDigests: []string{ + "centos/ruby-23-centos7@sha256:unparseable", + "docker.io/centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf", + }, + }, + Image: "centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf", + Output: true, + }, + { + // unparseable RepoDigest is ignored + Inspected: dockertypes.ImageInspect{ + ID: "sha256:9bbdf247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227", + RepoDigests: []string{"docker.io/centos/ruby-23-centos7@sha256:unparseable"}, + }, + Image: "centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf", + Output: false, + }, + { + // unparseable image digest is ignored + Inspected: dockertypes.ImageInspect{ + ID: "sha256:9bbdf247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227", + RepoDigests: []string{"docker.io/centos/ruby-23-centos7@sha256:unparseable"}, + }, + Image: "centos/ruby-23-centos7@sha256:unparseable", + Output: false, + }, + { + // prefix match is rejected for ID and RepoDigest + Inspected: dockertypes.ImageInspect{ + ID: "sha256:unparseable", + RepoDigests: []string{"docker.io/centos/ruby-23-centos7@sha256:unparseable"}, + }, + Image: "sha256:unparseable", + Output: false, + }, + { + // possible SHA prefix match is rejected for ID and RepoDigest because it is not in the named format + Inspected: dockertypes.ImageInspect{ + ID: "sha256:0000f247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227", + RepoDigests: []string{"docker.io/centos/ruby-23-centos7@sha256:0000f247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227"}, + }, + Image: "sha256:0000", + Output: false, + }, + } { + match := matchImageTagOrSHA(testCase.Inspected, testCase.Image) + assert.Equal(t, testCase.Output, match, testCase.Image+fmt.Sprintf(" is not a match (%d)", i)) + } +} + +func TestMatchImageIDOnly(t *testing.T) { + for i, testCase := range []struct { + Inspected dockertypes.ImageInspect + Image string + Output bool + }{ + // shouldn't match names or tagged names + { + Inspected: dockertypes.ImageInspect{RepoTags: []string{"ubuntu:latest"}}, + Image: "ubuntu", + Output: false, + }, + { + Inspected: dockertypes.ImageInspect{RepoTags: []string{"colemickens/hyperkube-amd64:217.9beff63"}}, + Image: "colemickens/hyperkube-amd64:217.9beff63", + Output: false, + }, + // should match name@digest refs if they refer to the image ID (but only the full ID) + { + Inspected: dockertypes.ImageInspect{ + ID: "sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", + }, + Image: "myimage@sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", + Output: true, + }, + { + Inspected: dockertypes.ImageInspect{ + ID: "sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", + }, + Image: "myimage@sha256:2208f7a29005", + Output: false, + }, + { + Inspected: dockertypes.ImageInspect{ + ID: "sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", + }, + Image: "myimage@sha256:2208", + Output: false, + }, + // should match when the IDs are literally the same + { + Inspected: dockertypes.ImageInspect{ + ID: "foobar", + }, + Image: "foobar", + Output: true, + }, + // shouldn't match mismatched IDs + { + Inspected: dockertypes.ImageInspect{ + ID: "sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", + }, + Image: "myimage@sha256:0000f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", + Output: false, + }, + // shouldn't match invalid IDs or refs + { + Inspected: dockertypes.ImageInspect{ + ID: "sha256:unparseable", + }, + Image: "myimage@sha256:unparseable", + Output: false, + }, + // shouldn't match against repo digests + { + Inspected: dockertypes.ImageInspect{ + ID: "sha256:9bbdf247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227", + RepoDigests: []string{"centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf"}, + }, + Image: "centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf", + Output: false, + }, + } { + match := matchImageIDOnly(testCase.Inspected, testCase.Image) + assert.Equal(t, testCase.Output, match, fmt.Sprintf("%s is not a match (%d)", testCase.Image, i)) + } + +} diff --git a/pkg/kubelet/dockertools/instrumented_docker.go b/pkg/kubelet/dockershim/libdocker/instrumented_client.go similarity index 65% rename from pkg/kubelet/dockertools/instrumented_docker.go rename to pkg/kubelet/dockershim/libdocker/instrumented_client.go index 66de3aecfea..9c2b78d4f10 100644 --- a/pkg/kubelet/dockertools/instrumented_docker.go +++ b/pkg/kubelet/dockershim/libdocker/instrumented_client.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package dockertools +package libdocker import ( "time" @@ -23,15 +23,15 @@ import ( "k8s.io/kubernetes/pkg/kubelet/metrics" ) -// instrumentedDockerInterface wraps the DockerInterface and records the operations +// instrumentedInterface wraps the Interface and records the operations // and errors metrics. -type instrumentedDockerInterface struct { - client DockerInterface +type instrumentedInterface struct { + client Interface } -// Creates an instrumented DockerInterface from an existing DockerInterface. -func NewInstrumentedDockerInterface(dockerClient DockerInterface) DockerInterface { - return instrumentedDockerInterface{ +// Creates an instrumented Interface from an existing Interface. +func NewInstrumentedInterface(dockerClient Interface) Interface { + return instrumentedInterface{ client: dockerClient, } } @@ -53,7 +53,7 @@ func recordError(operation string, err error) { } } -func (in instrumentedDockerInterface) ListContainers(options dockertypes.ContainerListOptions) ([]dockertypes.Container, error) { +func (in instrumentedInterface) ListContainers(options dockertypes.ContainerListOptions) ([]dockertypes.Container, error) { const operation = "list_containers" defer recordOperation(operation, time.Now()) @@ -62,7 +62,7 @@ func (in instrumentedDockerInterface) ListContainers(options dockertypes.Contain return out, err } -func (in instrumentedDockerInterface) InspectContainer(id string) (*dockertypes.ContainerJSON, error) { +func (in instrumentedInterface) InspectContainer(id string) (*dockertypes.ContainerJSON, error) { const operation = "inspect_container" defer recordOperation(operation, time.Now()) @@ -71,7 +71,7 @@ func (in instrumentedDockerInterface) InspectContainer(id string) (*dockertypes. return out, err } -func (in instrumentedDockerInterface) CreateContainer(opts dockertypes.ContainerCreateConfig) (*dockertypes.ContainerCreateResponse, error) { +func (in instrumentedInterface) CreateContainer(opts dockertypes.ContainerCreateConfig) (*dockertypes.ContainerCreateResponse, error) { const operation = "create_container" defer recordOperation(operation, time.Now()) @@ -80,7 +80,7 @@ func (in instrumentedDockerInterface) CreateContainer(opts dockertypes.Container return out, err } -func (in instrumentedDockerInterface) StartContainer(id string) error { +func (in instrumentedInterface) StartContainer(id string) error { const operation = "start_container" defer recordOperation(operation, time.Now()) @@ -89,7 +89,7 @@ func (in instrumentedDockerInterface) StartContainer(id string) error { return err } -func (in instrumentedDockerInterface) StopContainer(id string, timeout int) error { +func (in instrumentedInterface) StopContainer(id string, timeout int) error { const operation = "stop_container" defer recordOperation(operation, time.Now()) @@ -98,7 +98,7 @@ func (in instrumentedDockerInterface) StopContainer(id string, timeout int) erro return err } -func (in instrumentedDockerInterface) RemoveContainer(id string, opts dockertypes.ContainerRemoveOptions) error { +func (in instrumentedInterface) RemoveContainer(id string, opts dockertypes.ContainerRemoveOptions) error { const operation = "remove_container" defer recordOperation(operation, time.Now()) @@ -107,7 +107,7 @@ func (in instrumentedDockerInterface) RemoveContainer(id string, opts dockertype return err } -func (in instrumentedDockerInterface) InspectImageByRef(image string) (*dockertypes.ImageInspect, error) { +func (in instrumentedInterface) InspectImageByRef(image string) (*dockertypes.ImageInspect, error) { const operation = "inspect_image" defer recordOperation(operation, time.Now()) @@ -116,7 +116,7 @@ func (in instrumentedDockerInterface) InspectImageByRef(image string) (*dockerty return out, err } -func (in instrumentedDockerInterface) InspectImageByID(image string) (*dockertypes.ImageInspect, error) { +func (in instrumentedInterface) InspectImageByID(image string) (*dockertypes.ImageInspect, error) { const operation = "inspect_image" defer recordOperation(operation, time.Now()) @@ -125,7 +125,7 @@ func (in instrumentedDockerInterface) InspectImageByID(image string) (*dockertyp return out, err } -func (in instrumentedDockerInterface) ListImages(opts dockertypes.ImageListOptions) ([]dockertypes.Image, error) { +func (in instrumentedInterface) ListImages(opts dockertypes.ImageListOptions) ([]dockertypes.Image, error) { const operation = "list_images" defer recordOperation(operation, time.Now()) @@ -134,7 +134,7 @@ func (in instrumentedDockerInterface) ListImages(opts dockertypes.ImageListOptio return out, err } -func (in instrumentedDockerInterface) PullImage(imageID string, auth dockertypes.AuthConfig, opts dockertypes.ImagePullOptions) error { +func (in instrumentedInterface) PullImage(imageID string, auth dockertypes.AuthConfig, opts dockertypes.ImagePullOptions) error { const operation = "pull_image" defer recordOperation(operation, time.Now()) err := in.client.PullImage(imageID, auth, opts) @@ -142,7 +142,7 @@ func (in instrumentedDockerInterface) PullImage(imageID string, auth dockertypes return err } -func (in instrumentedDockerInterface) RemoveImage(image string, opts dockertypes.ImageRemoveOptions) ([]dockertypes.ImageDelete, error) { +func (in instrumentedInterface) RemoveImage(image string, opts dockertypes.ImageRemoveOptions) ([]dockertypes.ImageDelete, error) { const operation = "remove_image" defer recordOperation(operation, time.Now()) @@ -151,7 +151,7 @@ func (in instrumentedDockerInterface) RemoveImage(image string, opts dockertypes return imageDelete, err } -func (in instrumentedDockerInterface) Logs(id string, opts dockertypes.ContainerLogsOptions, sopts StreamOptions) error { +func (in instrumentedInterface) Logs(id string, opts dockertypes.ContainerLogsOptions, sopts StreamOptions) error { const operation = "logs" defer recordOperation(operation, time.Now()) @@ -160,7 +160,7 @@ func (in instrumentedDockerInterface) Logs(id string, opts dockertypes.Container return err } -func (in instrumentedDockerInterface) Version() (*dockertypes.Version, error) { +func (in instrumentedInterface) Version() (*dockertypes.Version, error) { const operation = "version" defer recordOperation(operation, time.Now()) @@ -169,7 +169,7 @@ func (in instrumentedDockerInterface) Version() (*dockertypes.Version, error) { return out, err } -func (in instrumentedDockerInterface) Info() (*dockertypes.Info, error) { +func (in instrumentedInterface) Info() (*dockertypes.Info, error) { const operation = "info" defer recordOperation(operation, time.Now()) @@ -178,7 +178,7 @@ func (in instrumentedDockerInterface) Info() (*dockertypes.Info, error) { return out, err } -func (in instrumentedDockerInterface) CreateExec(id string, opts dockertypes.ExecConfig) (*dockertypes.ContainerExecCreateResponse, error) { +func (in instrumentedInterface) CreateExec(id string, opts dockertypes.ExecConfig) (*dockertypes.ContainerExecCreateResponse, error) { const operation = "create_exec" defer recordOperation(operation, time.Now()) @@ -187,7 +187,7 @@ func (in instrumentedDockerInterface) CreateExec(id string, opts dockertypes.Exe return out, err } -func (in instrumentedDockerInterface) StartExec(startExec string, opts dockertypes.ExecStartCheck, sopts StreamOptions) error { +func (in instrumentedInterface) StartExec(startExec string, opts dockertypes.ExecStartCheck, sopts StreamOptions) error { const operation = "start_exec" defer recordOperation(operation, time.Now()) @@ -196,7 +196,7 @@ func (in instrumentedDockerInterface) StartExec(startExec string, opts dockertyp return err } -func (in instrumentedDockerInterface) InspectExec(id string) (*dockertypes.ContainerExecInspect, error) { +func (in instrumentedInterface) InspectExec(id string) (*dockertypes.ContainerExecInspect, error) { const operation = "inspect_exec" defer recordOperation(operation, time.Now()) @@ -205,7 +205,7 @@ func (in instrumentedDockerInterface) InspectExec(id string) (*dockertypes.Conta return out, err } -func (in instrumentedDockerInterface) AttachToContainer(id string, opts dockertypes.ContainerAttachOptions, sopts StreamOptions) error { +func (in instrumentedInterface) AttachToContainer(id string, opts dockertypes.ContainerAttachOptions, sopts StreamOptions) error { const operation = "attach" defer recordOperation(operation, time.Now()) @@ -214,7 +214,7 @@ func (in instrumentedDockerInterface) AttachToContainer(id string, opts dockerty return err } -func (in instrumentedDockerInterface) ImageHistory(id string) ([]dockertypes.ImageHistory, error) { +func (in instrumentedInterface) ImageHistory(id string) ([]dockertypes.ImageHistory, error) { const operation = "image_history" defer recordOperation(operation, time.Now()) @@ -223,7 +223,7 @@ func (in instrumentedDockerInterface) ImageHistory(id string) ([]dockertypes.Ima return out, err } -func (in instrumentedDockerInterface) ResizeExecTTY(id string, height, width int) error { +func (in instrumentedInterface) ResizeExecTTY(id string, height, width int) error { const operation = "resize_exec" defer recordOperation(operation, time.Now()) @@ -232,7 +232,7 @@ func (in instrumentedDockerInterface) ResizeExecTTY(id string, height, width int return err } -func (in instrumentedDockerInterface) ResizeContainerTTY(id string, height, width int) error { +func (in instrumentedInterface) ResizeContainerTTY(id string, height, width int) error { const operation = "resize_container" defer recordOperation(operation, time.Now()) diff --git a/pkg/kubelet/dockertools/kube_docker_client.go b/pkg/kubelet/dockershim/libdocker/kube_docker_client.go similarity index 95% rename from pkg/kubelet/dockertools/kube_docker_client.go rename to pkg/kubelet/dockershim/libdocker/kube_docker_client.go index c0dca47d58a..98434a1453d 100644 --- a/pkg/kubelet/dockertools/kube_docker_client.go +++ b/pkg/kubelet/dockershim/libdocker/kube_docker_client.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package dockertools +package libdocker import ( "bytes" @@ -38,16 +38,16 @@ import ( // kubeDockerClient is a wrapped layer of docker client for kubelet internal use. This layer is added to: // 1) Redirect stream for exec and attach operations. -// 2) Wrap the context in this layer to make the DockerInterface cleaner. -// 3) Stabilize the DockerInterface. The engine-api is still under active development, the interface -// is not stabilized yet. However, the DockerInterface is used in many files in Kubernetes, we may +// 2) Wrap the context in this layer to make the Interface cleaner. +// 3) Stabilize the Interface. The engine-api is still under active development, the interface +// is not stabilized yet. However, the Interface is used in many files in Kubernetes, we may // not want to change the interface frequently. With this layer, we can port the engine api to the -// DockerInterface to avoid changing DockerInterface as much as possible. +// Interface to avoid changing Interface as much as possible. // (See // * https://github.com/docker/engine-api/issues/89 // * https://github.com/docker/engine-api/issues/137 // * https://github.com/docker/engine-api/pull/140) -// TODO(random-liu): Swith to new docker interface by refactoring the functions in the old DockerInterface +// TODO(random-liu): Swith to new docker interface by refactoring the functions in the old Interface // one by one. type kubeDockerClient struct { // timeout is the timeout of short running docker operations. @@ -59,8 +59,8 @@ type kubeDockerClient struct { client *dockerapi.Client } -// Make sure that kubeDockerClient implemented the DockerInterface. -var _ DockerInterface = &kubeDockerClient{} +// Make sure that kubeDockerClient implemented the Interface. +var _ Interface = &kubeDockerClient{} // There are 2 kinds of docker operations categorized by running time: // * Long running operation: The long running operation could run for arbitrary long time, and the running time @@ -83,7 +83,7 @@ const ( // newKubeDockerClient creates an kubeDockerClient from an existing docker client. If requestTimeout is 0, // defaultTimeout will be applied. -func newKubeDockerClient(dockerClient *dockerapi.Client, requestTimeout, imagePullProgressDeadline time.Duration) DockerInterface { +func newKubeDockerClient(dockerClient *dockerapi.Client, requestTimeout, imagePullProgressDeadline time.Duration) Interface { if requestTimeout == 0 { requestTimeout = defaultTimeout } @@ -576,12 +576,6 @@ func (d *kubeDockerClient) getCustomTimeoutContext(timeout time.Duration) (conte return context.WithTimeout(context.Background(), timeout) } -// ParseDockerTimestamp parses the timestamp returned by DockerInterface from string to time.Time -func ParseDockerTimestamp(s string) (time.Time, error) { - // Timestamp returned by Docker is in time.RFC3339Nano format. - return time.Parse(time.RFC3339Nano, s) -} - // contextError checks the context, and returns error if the context is timeout. func contextError(ctx context.Context) error { if ctx.Err() == context.DeadlineExceeded { diff --git a/pkg/kubelet/dockertools/kube_docker_client_test.go b/pkg/kubelet/dockershim/libdocker/kube_docker_client_test.go similarity index 98% rename from pkg/kubelet/dockertools/kube_docker_client_test.go rename to pkg/kubelet/dockershim/libdocker/kube_docker_client_test.go index 42f1ac4fdea..dca570bbd26 100644 --- a/pkg/kubelet/dockertools/kube_docker_client_test.go +++ b/pkg/kubelet/dockershim/libdocker/kube_docker_client_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package dockertools +package libdocker import ( "fmt" diff --git a/pkg/kubelet/dockershim/libdocker/legacy.go b/pkg/kubelet/dockershim/libdocker/legacy.go new file mode 100644 index 00000000000..97e0b4d72ad --- /dev/null +++ b/pkg/kubelet/dockershim/libdocker/legacy.go @@ -0,0 +1,92 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package libdocker + +import ( + "fmt" + "math/rand" + "strconv" + "strings" + + "github.com/golang/glog" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/api/v1" + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" +) + +// This file contains functions used in the non-CRI integration (< 1.6). They +// are currently used for recoginzing containers created by pre-1.6 kubelets. +// TODO: Remove this file for kubernetes 1.8+. + +// Creates a name which can be reversed to identify both full pod name and container name. +// This function returns stable name, unique name and a unique id. +// Although rand.Uint32() is not really unique, but it's enough for us because error will +// only occur when instances of the same container in the same pod have the same UID. The +// chance is really slim. +func BuildDockerName(dockerName KubeletContainerName, container *v1.Container) (string, string, string) { + containerName := dockerName.ContainerName + "." + strconv.FormatUint(kubecontainer.HashContainerLegacy(container), 16) + stableName := fmt.Sprintf("%s_%s_%s_%s", + containerNamePrefix, + containerName, + dockerName.PodFullName, + dockerName.PodUID) + UID := fmt.Sprintf("%08x", rand.Uint32()) + return stableName, fmt.Sprintf("%s_%s", stableName, UID), UID +} + +// Unpacks a container name, returning the pod full name and container name we would have used to +// construct the docker name. If we are unable to parse the name, an error is returned. +func ParseDockerName(name string) (dockerName *KubeletContainerName, hash uint64, err error) { + // For some reason docker appears to be appending '/' to names. + // If it's there, strip it. + name = strings.TrimPrefix(name, "/") + parts := strings.Split(name, "_") + if len(parts) == 0 || parts[0] != containerNamePrefix { + err = fmt.Errorf("failed to parse Docker container name %q into parts", name) + return nil, 0, err + } + if len(parts) < 6 { + // We have at least 5 fields. We may have more in the future. + // Anything with less fields than this is not something we can + // manage. + glog.Warningf("found a container with the %q prefix, but too few fields (%d): %q", containerNamePrefix, len(parts), name) + err = fmt.Errorf("Docker container name %q has less parts than expected %v", name, parts) + return nil, 0, err + } + + nameParts := strings.Split(parts[1], ".") + containerName := nameParts[0] + if len(nameParts) > 1 { + hash, err = strconv.ParseUint(nameParts[1], 16, 32) + if err != nil { + glog.Warningf("invalid container hash %q in container %q", nameParts[1], name) + } + } + + podFullName := parts[2] + "_" + parts[3] + podUID := types.UID(parts[4]) + + return &KubeletContainerName{podFullName, podUID, containerName}, hash, nil +} + +// KubeletContainerName encapsulates a pod name and a Kubernetes container name. +type KubeletContainerName struct { + PodFullName string + PodUID types.UID + ContainerName string +} diff --git a/pkg/kubelet/dockershim/libdocker/legacy_test.go b/pkg/kubelet/dockershim/libdocker/legacy_test.go new file mode 100644 index 00000000000..ca14349a8f1 --- /dev/null +++ b/pkg/kubelet/dockershim/libdocker/legacy_test.go @@ -0,0 +1,140 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package libdocker + +import ( + "fmt" + "hash/adler32" + "testing" + + dockertypes "github.com/docker/engine-api/types" + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/api/v1" + hashutil "k8s.io/kubernetes/pkg/util/hash" +) + +func verifyCalls(t *testing.T, fakeDocker *FakeDockerClient, calls []string) { + assert.New(t).NoError(fakeDocker.AssertCalls(calls)) +} + +func verifyStringArrayEquals(t *testing.T, actual, expected []string) { + invalid := len(actual) != len(expected) + if !invalid { + for ix, value := range actual { + if expected[ix] != value { + invalid = true + } + } + } + if invalid { + t.Errorf("Expected: %#v, Actual: %#v", expected, actual) + } +} + +func findPodContainer(dockerContainers []*dockertypes.Container, podFullName string, uid types.UID, containerName string) (*dockertypes.Container, bool, uint64) { + for _, dockerContainer := range dockerContainers { + if len(dockerContainer.Names) == 0 { + continue + } + dockerName, hash, err := ParseDockerName(dockerContainer.Names[0]) + if err != nil { + continue + } + if dockerName.PodFullName == podFullName && + (uid == "" || dockerName.PodUID == uid) && + dockerName.ContainerName == containerName { + return dockerContainer, true, hash + } + } + return nil, false, 0 +} + +func TestGetContainerID(t *testing.T) { + fakeDocker := NewFakeDockerClient() + fakeDocker.SetFakeRunningContainers([]*FakeContainer{ + { + ID: "foobar", + Name: "/k8s_foo_qux_ns_1234_42", + }, + { + ID: "barbar", + Name: "/k8s_bar_qux_ns_2565_42", + }, + }) + + dockerContainers, err := GetKubeletDockerContainers(fakeDocker, false) + if err != nil { + t.Errorf("Expected no error, Got %#v", err) + } + if len(dockerContainers) != 2 { + t.Errorf("Expected %#v, Got %#v", fakeDocker.RunningContainerList, dockerContainers) + } + verifyCalls(t, fakeDocker, []string{"list"}) + + dockerContainer, found, _ := findPodContainer(dockerContainers, "qux_ns", "", "foo") + if dockerContainer == nil || !found { + t.Errorf("Failed to find container %#v", dockerContainer) + } + + fakeDocker.ClearCalls() + dockerContainer, found, _ = findPodContainer(dockerContainers, "foobar", "", "foo") + verifyCalls(t, fakeDocker, []string{}) + if dockerContainer != nil || found { + t.Errorf("Should not have found container %#v", dockerContainer) + } +} + +func verifyPackUnpack(t *testing.T, podNamespace, podUID, podName, containerName string) { + container := &v1.Container{Name: containerName} + hasher := adler32.New() + hashutil.DeepHashObject(hasher, *container) + computedHash := uint64(hasher.Sum32()) + podFullName := fmt.Sprintf("%s_%s", podName, podNamespace) + _, name, _ := BuildDockerName(KubeletContainerName{podFullName, types.UID(podUID), container.Name}, container) + returned, hash, err := ParseDockerName(name) + if err != nil { + t.Errorf("Failed to parse Docker container name %q: %v", name, err) + } + if podFullName != returned.PodFullName || podUID != string(returned.PodUID) || containerName != returned.ContainerName || computedHash != hash { + t.Errorf("For (%s, %s, %s, %d), unpacked (%s, %s, %s, %d)", podFullName, podUID, containerName, computedHash, returned.PodFullName, returned.PodUID, returned.ContainerName, hash) + } +} + +func TestContainerNaming(t *testing.T) { + podUID := "12345678" + verifyPackUnpack(t, "file", podUID, "name", "container") + verifyPackUnpack(t, "file", podUID, "name-with-dashes", "container") + // UID is same as pod name + verifyPackUnpack(t, "file", podUID, podUID, "container") + // No Container name + verifyPackUnpack(t, "other", podUID, "name", "") + + container := &v1.Container{Name: "container"} + podName := "foo" + podNamespace := "test" + name := fmt.Sprintf("k8s_%s_%s_%s_%s_42", container.Name, podName, podNamespace, podUID) + podFullName := fmt.Sprintf("%s_%s", podName, podNamespace) + + returned, hash, err := ParseDockerName(name) + if err != nil { + t.Errorf("Failed to parse Docker container name %q: %v", name, err) + } + if returned.PodFullName != podFullName || string(returned.PodUID) != podUID || returned.ContainerName != container.Name || hash != 0 { + t.Errorf("unexpected parse: %s %s %s %d", returned.PodFullName, returned.PodUID, returned.ContainerName, hash) + } +} diff --git a/pkg/kubelet/dockertools/docker.go b/pkg/kubelet/dockertools/docker.go index af19747dee1..01ba78ae26a 100644 --- a/pkg/kubelet/dockertools/docker.go +++ b/pkg/kubelet/dockertools/docker.go @@ -18,24 +18,17 @@ package dockertools import ( "fmt" - "math/rand" "net/http" "path" - "strconv" "strings" - "time" - dockerdigest "github.com/docker/distribution/digest" - dockerref "github.com/docker/distribution/reference" "github.com/docker/docker/pkg/jsonmessage" - dockerapi "github.com/docker/engine-api/client" dockertypes "github.com/docker/engine-api/types" "github.com/golang/glog" - "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/credentialprovider" - kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" "k8s.io/kubernetes/pkg/kubelet/images" ) @@ -44,60 +37,11 @@ const ( ext4MaxFileNameLen = 255 DockerType = "docker" - - // https://docs.docker.com/engine/reference/api/docker_remote_api/ - // docker version should be at least 1.10.x - minimumDockerAPIVersion = "1.22" - - statusRunningPrefix = "Up" - statusExitedPrefix = "Exited" - statusCreatedPrefix = "Created" ) -// DockerInterface is an abstract interface for testability. It abstracts the interface of docker client. -type DockerInterface interface { - ListContainers(options dockertypes.ContainerListOptions) ([]dockertypes.Container, error) - InspectContainer(id string) (*dockertypes.ContainerJSON, error) - CreateContainer(dockertypes.ContainerCreateConfig) (*dockertypes.ContainerCreateResponse, error) - StartContainer(id string) error - StopContainer(id string, timeout int) error - RemoveContainer(id string, opts dockertypes.ContainerRemoveOptions) error - InspectImageByRef(imageRef string) (*dockertypes.ImageInspect, error) - InspectImageByID(imageID string) (*dockertypes.ImageInspect, error) - ListImages(opts dockertypes.ImageListOptions) ([]dockertypes.Image, error) - PullImage(image string, auth dockertypes.AuthConfig, opts dockertypes.ImagePullOptions) error - RemoveImage(image string, opts dockertypes.ImageRemoveOptions) ([]dockertypes.ImageDelete, error) - ImageHistory(id string) ([]dockertypes.ImageHistory, error) - Logs(string, dockertypes.ContainerLogsOptions, StreamOptions) error - Version() (*dockertypes.Version, error) - Info() (*dockertypes.Info, error) - CreateExec(string, dockertypes.ExecConfig) (*dockertypes.ContainerExecCreateResponse, error) - StartExec(string, dockertypes.ExecStartCheck, StreamOptions) error - InspectExec(id string) (*dockertypes.ContainerExecInspect, error) - AttachToContainer(string, dockertypes.ContainerAttachOptions, StreamOptions) error - ResizeContainerTTY(id string, height, width int) error - ResizeExecTTY(id string, height, width int) error -} - -// KubeletContainerName encapsulates a pod name and a Kubernetes container name. -type KubeletContainerName struct { - PodFullName string - PodUID types.UID - ContainerName string -} - -// containerNamePrefix is used to identify the containers on the node managed by this -// process. -var containerNamePrefix = "k8s" - -// SetContainerNamePrefix allows the container prefix name for this process to be changed. -// This is intended to support testing and bootstrapping experimentation. It cannot be -// changed once the Kubelet starts. -func SetContainerNamePrefix(prefix string) { - containerNamePrefix = prefix -} - // DockerPuller is an abstract interface for testability. It abstracts image pull operations. +// DockerPuller is *not* in use anywhere in the codebase. +// TODO: Examine whether we can migrate the unit tests and remove the code. type DockerPuller interface { Pull(image string, secrets []v1.Secret) error GetImageRef(image string) (string, error) @@ -105,12 +49,12 @@ type DockerPuller interface { // dockerPuller is the default implementation of DockerPuller. type dockerPuller struct { - client DockerInterface + client libdocker.Interface keyring credentialprovider.DockerKeyring } // newDockerPuller creates a new instance of the default implementation of DockerPuller. -func newDockerPuller(client DockerInterface) DockerPuller { +func newDockerPuller(client libdocker.Interface) DockerPuller { return &dockerPuller{ client: client, keyring: credentialprovider.NewDockerKeyring(), @@ -133,104 +77,6 @@ func filterHTTPError(err error, image string) error { } } -// matchImageTagOrSHA checks if the given image specifier is a valid image ref, -// and that it matches the given image. It should fail on things like image IDs -// (config digests) and other digest-only references, but succeed on image names -// (`foo`), tag references (`foo:bar`), and manifest digest references -// (`foo@sha256:xyz`). -func matchImageTagOrSHA(inspected dockertypes.ImageInspect, image string) bool { - // The image string follows the grammar specified here - // https://github.com/docker/distribution/blob/master/reference/reference.go#L4 - named, err := dockerref.ParseNamed(image) - if err != nil { - glog.V(4).Infof("couldn't parse image reference %q: %v", image, err) - return false - } - _, isTagged := named.(dockerref.Tagged) - digest, isDigested := named.(dockerref.Digested) - if !isTagged && !isDigested { - // No Tag or SHA specified, so just return what we have - return true - } - - if isTagged { - // Check the RepoTags for a match. - for _, tag := range inspected.RepoTags { - // An image name (without the tag/digest) can be [hostname '/'] component ['/' component]* - // Because either the RepoTag or the name *may* contain the - // hostname or not, we only check for the suffix match. - if strings.HasSuffix(image, tag) || strings.HasSuffix(tag, image) { - return true - } - } - } - - if isDigested { - for _, repoDigest := range inspected.RepoDigests { - named, err := dockerref.ParseNamed(repoDigest) - if err != nil { - glog.V(4).Infof("couldn't parse image RepoDigest reference %q: %v", repoDigest, err) - continue - } - if d, isDigested := named.(dockerref.Digested); isDigested { - if digest.Digest().Algorithm().String() == d.Digest().Algorithm().String() && - digest.Digest().Hex() == d.Digest().Hex() { - return true - } - } - } - - // process the ID as a digest - id, err := dockerdigest.ParseDigest(inspected.ID) - if err != nil { - glog.V(4).Infof("couldn't parse image ID reference %q: %v", id, err) - return false - } - if digest.Digest().Algorithm().String() == id.Algorithm().String() && digest.Digest().Hex() == id.Hex() { - return true - } - } - glog.V(4).Infof("Inspected image (%q) does not match %s", inspected.ID, image) - return false -} - -// matchImageIDOnly checks that the given image specifier is a digest-only -// reference, and that it matches the given image. -func matchImageIDOnly(inspected dockertypes.ImageInspect, image string) bool { - // If the image ref is literally equal to the inspected image's ID, - // just return true here (this might be the case for Docker 1.9, - // where we won't have a digest for the ID) - if inspected.ID == image { - return true - } - - // Otherwise, we should try actual parsing to be more correct - ref, err := dockerref.Parse(image) - if err != nil { - glog.V(4).Infof("couldn't parse image reference %q: %v", image, err) - return false - } - - digest, isDigested := ref.(dockerref.Digested) - if !isDigested { - glog.V(4).Infof("the image reference %q was not a digest reference") - return false - } - - id, err := dockerdigest.ParseDigest(inspected.ID) - if err != nil { - glog.V(4).Infof("couldn't parse image ID reference %q: %v", id, err) - return false - } - - if digest.Digest().Algorithm().String() == id.Algorithm().String() && digest.Digest().Hex() == id.Hex() { - return true - } - - glog.V(4).Infof("The reference %s does not directly refer to the given image's ID (%q)", image, inspected.ID) - return false -} - func (p dockerPuller) Pull(image string, secrets []v1.Secret) error { keyring, err := credentialprovider.MakeDockerKeyring(secrets, p.keyring) if err != nil { @@ -296,63 +142,12 @@ func (p dockerPuller) GetImageRef(image string) (string, error) { } return imageRef, nil } - if IsImageNotFoundError(err) { + if libdocker.IsImageNotFoundError(err) { return "", nil } return "", err } -// Creates a name which can be reversed to identify both full pod name and container name. -// This function returns stable name, unique name and a unique id. -// Although rand.Uint32() is not really unique, but it's enough for us because error will -// only occur when instances of the same container in the same pod have the same UID. The -// chance is really slim. -func BuildDockerName(dockerName KubeletContainerName, container *v1.Container) (string, string, string) { - containerName := dockerName.ContainerName + "." + strconv.FormatUint(kubecontainer.HashContainerLegacy(container), 16) - stableName := fmt.Sprintf("%s_%s_%s_%s", - containerNamePrefix, - containerName, - dockerName.PodFullName, - dockerName.PodUID) - UID := fmt.Sprintf("%08x", rand.Uint32()) - return stableName, fmt.Sprintf("%s_%s", stableName, UID), UID -} - -// Unpacks a container name, returning the pod full name and container name we would have used to -// construct the docker name. If we are unable to parse the name, an error is returned. -func ParseDockerName(name string) (dockerName *KubeletContainerName, hash uint64, err error) { - // For some reason docker appears to be appending '/' to names. - // If it's there, strip it. - name = strings.TrimPrefix(name, "/") - parts := strings.Split(name, "_") - if len(parts) == 0 || parts[0] != containerNamePrefix { - err = fmt.Errorf("failed to parse Docker container name %q into parts", name) - return nil, 0, err - } - if len(parts) < 6 { - // We have at least 5 fields. We may have more in the future. - // Anything with less fields than this is not something we can - // manage. - glog.Warningf("found a container with the %q prefix, but too few fields (%d): %q", containerNamePrefix, len(parts), name) - err = fmt.Errorf("Docker container name %q has less parts than expected %v", name, parts) - return nil, 0, err - } - - nameParts := strings.Split(parts[1], ".") - containerName := nameParts[0] - if len(nameParts) > 1 { - hash, err = strconv.ParseUint(nameParts[1], 16, 32) - if err != nil { - glog.Warningf("invalid container hash %q in container %q", nameParts[1], name) - } - } - - podFullName := parts[2] + "_" + parts[3] - podUID := types.UID(parts[4]) - - return &KubeletContainerName{podFullName, podUID, containerName}, hash, nil -} - func LogSymlink(containerLogsDir, podFullName, containerName, dockerId string) string { suffix := fmt.Sprintf(".%s", LogSuffix) logPath := fmt.Sprintf("%s_%s-%s", podFullName, containerName, dockerId) @@ -362,57 +157,3 @@ func LogSymlink(containerLogsDir, podFullName, containerName, dockerId string) s } return path.Join(containerLogsDir, logPath+suffix) } - -// Get a *dockerapi.Client, either using the endpoint passed in, or using -// DOCKER_HOST, DOCKER_TLS_VERIFY, and DOCKER_CERT path per their spec -func getDockerClient(dockerEndpoint string) (*dockerapi.Client, error) { - if len(dockerEndpoint) > 0 { - glog.Infof("Connecting to docker on %s", dockerEndpoint) - return dockerapi.NewClient(dockerEndpoint, "", nil, nil) - } - return dockerapi.NewEnvClient() -} - -// ConnectToDockerOrDie creates docker client connecting to docker daemon. -// If the endpoint passed in is "fake://", a fake docker client -// will be returned. The program exits if error occurs. The requestTimeout -// is the timeout for docker requests. If timeout is exceeded, the request -// will be cancelled and throw out an error. If requestTimeout is 0, a default -// value will be applied. -func ConnectToDockerOrDie(dockerEndpoint string, requestTimeout, imagePullProgressDeadline time.Duration) DockerInterface { - if dockerEndpoint == "fake://" { - return NewFakeDockerClient() - } - client, err := getDockerClient(dockerEndpoint) - if err != nil { - glog.Fatalf("Couldn't connect to docker: %v", err) - } - glog.Infof("Start docker client with request timeout=%v", requestTimeout) - return newKubeDockerClient(client, requestTimeout, imagePullProgressDeadline) -} - -// GetKubeletDockerContainers lists all container or just the running ones. -// Returns a list of docker containers that we manage -// TODO: This function should be deleted after migrating -// test/e2e_node/garbage_collector_test.go off of it. -func GetKubeletDockerContainers(client DockerInterface, allContainers bool) ([]*dockertypes.Container, error) { - result := []*dockertypes.Container{} - containers, err := client.ListContainers(dockertypes.ContainerListOptions{All: allContainers}) - if err != nil { - return nil, err - } - for i := range containers { - container := &containers[i] - if len(container.Names) == 0 { - continue - } - // Skip containers that we didn't create to allow users to manually - // spin up their own containers if they want. - if !strings.HasPrefix(container.Names[0], "/"+containerNamePrefix+"_") { - glog.V(5).Infof("Docker Container: %s is not managed by kubelet.", container.Names[0]) - continue - } - result = append(result, container) - } - return result, nil -} diff --git a/pkg/kubelet/dockertools/docker_test.go b/pkg/kubelet/dockertools/docker_test.go index 6ea62b442d7..ed5a9e24dc3 100644 --- a/pkg/kubelet/dockertools/docker_test.go +++ b/pkg/kubelet/dockertools/docker_test.go @@ -19,371 +19,20 @@ package dockertools import ( "encoding/json" "fmt" - "hash/adler32" "math/rand" "path" - "reflect" "strings" "testing" "github.com/docker/docker/pkg/jsonmessage" - dockertypes "github.com/docker/engine-api/types" "github.com/stretchr/testify/assert" - "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/credentialprovider" + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" "k8s.io/kubernetes/pkg/kubelet/images" - hashutil "k8s.io/kubernetes/pkg/util/hash" ) -func verifyCalls(t *testing.T, fakeDocker *FakeDockerClient, calls []string) { - assert.New(t).NoError(fakeDocker.AssertCalls(calls)) -} - -func verifyStringArrayEquals(t *testing.T, actual, expected []string) { - invalid := len(actual) != len(expected) - if !invalid { - for ix, value := range actual { - if expected[ix] != value { - invalid = true - } - } - } - if invalid { - t.Errorf("Expected: %#v, Actual: %#v", expected, actual) - } -} - -func findPodContainer(dockerContainers []*dockertypes.Container, podFullName string, uid types.UID, containerName string) (*dockertypes.Container, bool, uint64) { - for _, dockerContainer := range dockerContainers { - if len(dockerContainer.Names) == 0 { - continue - } - dockerName, hash, err := ParseDockerName(dockerContainer.Names[0]) - if err != nil { - continue - } - if dockerName.PodFullName == podFullName && - (uid == "" || dockerName.PodUID == uid) && - dockerName.ContainerName == containerName { - return dockerContainer, true, hash - } - } - return nil, false, 0 -} - -func TestGetContainerID(t *testing.T) { - fakeDocker := NewFakeDockerClient() - fakeDocker.SetFakeRunningContainers([]*FakeContainer{ - { - ID: "foobar", - Name: "/k8s_foo_qux_ns_1234_42", - }, - { - ID: "barbar", - Name: "/k8s_bar_qux_ns_2565_42", - }, - }) - - dockerContainers, err := GetKubeletDockerContainers(fakeDocker, false) - if err != nil { - t.Errorf("Expected no error, Got %#v", err) - } - if len(dockerContainers) != 2 { - t.Errorf("Expected %#v, Got %#v", fakeDocker.RunningContainerList, dockerContainers) - } - verifyCalls(t, fakeDocker, []string{"list"}) - - dockerContainer, found, _ := findPodContainer(dockerContainers, "qux_ns", "", "foo") - if dockerContainer == nil || !found { - t.Errorf("Failed to find container %#v", dockerContainer) - } - - fakeDocker.ClearCalls() - dockerContainer, found, _ = findPodContainer(dockerContainers, "foobar", "", "foo") - verifyCalls(t, fakeDocker, []string{}) - if dockerContainer != nil || found { - t.Errorf("Should not have found container %#v", dockerContainer) - } -} - -func verifyPackUnpack(t *testing.T, podNamespace, podUID, podName, containerName string) { - container := &v1.Container{Name: containerName} - hasher := adler32.New() - hashutil.DeepHashObject(hasher, *container) - computedHash := uint64(hasher.Sum32()) - podFullName := fmt.Sprintf("%s_%s", podName, podNamespace) - _, name, _ := BuildDockerName(KubeletContainerName{podFullName, types.UID(podUID), container.Name}, container) - returned, hash, err := ParseDockerName(name) - if err != nil { - t.Errorf("Failed to parse Docker container name %q: %v", name, err) - } - if podFullName != returned.PodFullName || podUID != string(returned.PodUID) || containerName != returned.ContainerName || computedHash != hash { - t.Errorf("For (%s, %s, %s, %d), unpacked (%s, %s, %s, %d)", podFullName, podUID, containerName, computedHash, returned.PodFullName, returned.PodUID, returned.ContainerName, hash) - } -} - -func TestContainerNaming(t *testing.T) { - podUID := "12345678" - verifyPackUnpack(t, "file", podUID, "name", "container") - verifyPackUnpack(t, "file", podUID, "name-with-dashes", "container") - // UID is same as pod name - verifyPackUnpack(t, "file", podUID, podUID, "container") - // No Container name - verifyPackUnpack(t, "other", podUID, "name", "") - - container := &v1.Container{Name: "container"} - podName := "foo" - podNamespace := "test" - name := fmt.Sprintf("k8s_%s_%s_%s_%s_42", container.Name, podName, podNamespace, podUID) - podFullName := fmt.Sprintf("%s_%s", podName, podNamespace) - - returned, hash, err := ParseDockerName(name) - if err != nil { - t.Errorf("Failed to parse Docker container name %q: %v", name, err) - } - if returned.PodFullName != podFullName || string(returned.PodUID) != podUID || returned.ContainerName != container.Name || hash != 0 { - t.Errorf("unexpected parse: %s %s %s %d", returned.PodFullName, returned.PodUID, returned.ContainerName, hash) - } -} - -func TestMatchImageTagOrSHA(t *testing.T) { - for i, testCase := range []struct { - Inspected dockertypes.ImageInspect - Image string - Output bool - }{ - { - Inspected: dockertypes.ImageInspect{RepoTags: []string{"ubuntu:latest"}}, - Image: "ubuntu", - Output: true, - }, - { - Inspected: dockertypes.ImageInspect{RepoTags: []string{"ubuntu:14.04"}}, - Image: "ubuntu:latest", - Output: false, - }, - { - Inspected: dockertypes.ImageInspect{RepoTags: []string{"colemickens/hyperkube-amd64:217.9beff63"}}, - Image: "colemickens/hyperkube-amd64:217.9beff63", - Output: true, - }, - { - Inspected: dockertypes.ImageInspect{RepoTags: []string{"colemickens/hyperkube-amd64:217.9beff63"}}, - Image: "docker.io/colemickens/hyperkube-amd64:217.9beff63", - Output: true, - }, - { - Inspected: dockertypes.ImageInspect{RepoTags: []string{"docker.io/kubernetes/pause:latest"}}, - Image: "kubernetes/pause:latest", - Output: true, - }, - { - Inspected: dockertypes.ImageInspect{ - ID: "sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", - }, - Image: "myimage@sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", - Output: true, - }, - { - Inspected: dockertypes.ImageInspect{ - ID: "sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", - }, - Image: "myimage@sha256:2208f7a29005", - Output: false, - }, - { - Inspected: dockertypes.ImageInspect{ - ID: "sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", - }, - Image: "myimage@sha256:2208", - Output: false, - }, - { - // mismatched ID is ignored - Inspected: dockertypes.ImageInspect{ - ID: "sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", - }, - Image: "myimage@sha256:0000f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", - Output: false, - }, - { - // invalid digest is ignored - Inspected: dockertypes.ImageInspect{ - ID: "sha256:unparseable", - }, - Image: "myimage@sha256:unparseable", - Output: false, - }, - { - // v1 schema images can be pulled in one format and returned in another - Inspected: dockertypes.ImageInspect{ - ID: "sha256:9bbdf247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227", - RepoDigests: []string{"centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf"}, - }, - Image: "centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf", - Output: true, - }, - { - // RepoDigest match is is required - Inspected: dockertypes.ImageInspect{ - ID: "", - RepoDigests: []string{"docker.io/centos/ruby-23-centos7@sha256:000084acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf"}, - }, - Image: "centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf", - Output: false, - }, - { - // RepoDigest match is allowed - Inspected: dockertypes.ImageInspect{ - ID: "sha256:9bbdf247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227", - RepoDigests: []string{"docker.io/centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf"}, - }, - Image: "centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf", - Output: true, - }, - { - // RepoDigest and ID are checked - Inspected: dockertypes.ImageInspect{ - ID: "sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf", - RepoDigests: []string{"docker.io/centos/ruby-23-centos7@sha256:9bbdf247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227"}, - }, - Image: "centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf", - Output: true, - }, - { - // unparseable RepoDigests are skipped - Inspected: dockertypes.ImageInspect{ - ID: "sha256:9bbdf247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227", - RepoDigests: []string{ - "centos/ruby-23-centos7@sha256:unparseable", - "docker.io/centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf", - }, - }, - Image: "centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf", - Output: true, - }, - { - // unparseable RepoDigest is ignored - Inspected: dockertypes.ImageInspect{ - ID: "sha256:9bbdf247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227", - RepoDigests: []string{"docker.io/centos/ruby-23-centos7@sha256:unparseable"}, - }, - Image: "centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf", - Output: false, - }, - { - // unparseable image digest is ignored - Inspected: dockertypes.ImageInspect{ - ID: "sha256:9bbdf247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227", - RepoDigests: []string{"docker.io/centos/ruby-23-centos7@sha256:unparseable"}, - }, - Image: "centos/ruby-23-centos7@sha256:unparseable", - Output: false, - }, - { - // prefix match is rejected for ID and RepoDigest - Inspected: dockertypes.ImageInspect{ - ID: "sha256:unparseable", - RepoDigests: []string{"docker.io/centos/ruby-23-centos7@sha256:unparseable"}, - }, - Image: "sha256:unparseable", - Output: false, - }, - { - // possible SHA prefix match is rejected for ID and RepoDigest because it is not in the named format - Inspected: dockertypes.ImageInspect{ - ID: "sha256:0000f247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227", - RepoDigests: []string{"docker.io/centos/ruby-23-centos7@sha256:0000f247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227"}, - }, - Image: "sha256:0000", - Output: false, - }, - } { - match := matchImageTagOrSHA(testCase.Inspected, testCase.Image) - assert.Equal(t, testCase.Output, match, testCase.Image+fmt.Sprintf(" is not a match (%d)", i)) - } -} - -func TestMatchImageIDOnly(t *testing.T) { - for i, testCase := range []struct { - Inspected dockertypes.ImageInspect - Image string - Output bool - }{ - // shouldn't match names or tagged names - { - Inspected: dockertypes.ImageInspect{RepoTags: []string{"ubuntu:latest"}}, - Image: "ubuntu", - Output: false, - }, - { - Inspected: dockertypes.ImageInspect{RepoTags: []string{"colemickens/hyperkube-amd64:217.9beff63"}}, - Image: "colemickens/hyperkube-amd64:217.9beff63", - Output: false, - }, - // should match name@digest refs if they refer to the image ID (but only the full ID) - { - Inspected: dockertypes.ImageInspect{ - ID: "sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", - }, - Image: "myimage@sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", - Output: true, - }, - { - Inspected: dockertypes.ImageInspect{ - ID: "sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", - }, - Image: "myimage@sha256:2208f7a29005", - Output: false, - }, - { - Inspected: dockertypes.ImageInspect{ - ID: "sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", - }, - Image: "myimage@sha256:2208", - Output: false, - }, - // should match when the IDs are literally the same - { - Inspected: dockertypes.ImageInspect{ - ID: "foobar", - }, - Image: "foobar", - Output: true, - }, - // shouldn't match mismatched IDs - { - Inspected: dockertypes.ImageInspect{ - ID: "sha256:2208f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", - }, - Image: "myimage@sha256:0000f7a29005d226d1ee33a63e33af1f47af6156c740d7d23c7948e8d282d53d", - Output: false, - }, - // shouldn't match invalid IDs or refs - { - Inspected: dockertypes.ImageInspect{ - ID: "sha256:unparseable", - }, - Image: "myimage@sha256:unparseable", - Output: false, - }, - // shouldn't match against repo digests - { - Inspected: dockertypes.ImageInspect{ - ID: "sha256:9bbdf247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227", - RepoDigests: []string{"centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf"}, - }, - Image: "centos/ruby-23-centos7@sha256:940584acbbfb0347272112d2eb95574625c0c60b4e2fdadb139de5859cf754bf", - Output: false, - }, - } { - match := matchImageIDOnly(testCase.Inspected, testCase.Image) - assert.Equal(t, testCase.Output, match, fmt.Sprintf("%s is not a match (%d)", testCase.Image, i)) - } - -} - +// TODO: Examine the tests and see if they can be migrated to kuberuntime. func TestPullWithNoSecrets(t *testing.T) { tests := []struct { imageName string @@ -399,7 +48,7 @@ func TestPullWithNoSecrets(t *testing.T) { } for _, test := range tests { fakeKeyring := &credentialprovider.FakeKeyring{} - fakeClient := NewFakeDockerClient() + fakeClient := libdocker.NewFakeDockerClient() dp := dockerPuller{ client: fakeClient, @@ -412,13 +61,8 @@ func TestPullWithNoSecrets(t *testing.T) { continue } - if e, a := 1, len(fakeClient.pulled); e != a { - t.Errorf("%s: expected 1 pulled image, got %d: %v", test.imageName, a, fakeClient.pulled) - continue - } - - if e, a := test.expectedImage, fakeClient.pulled[0]; e != a { - t.Errorf("%s: expected pull of %q, but got %q", test.imageName, e, a) + if err := fakeClient.AssertImagesPulled([]string{test.imageName}); err != nil { + t.Errorf("images pulled do not match the expected: %v", err) } } } @@ -442,7 +86,7 @@ func TestPullWithJSONError(t *testing.T) { } for i, test := range tests { fakeKeyring := &credentialprovider.FakeKeyring{} - fakeClient := NewFakeDockerClient() + fakeClient := libdocker.NewFakeDockerClient() fakeClient.InjectError("pull", test.err) puller := &dockerPuller{ @@ -520,7 +164,7 @@ func TestPullWithSecrets(t *testing.T) { builtInKeyRing := &credentialprovider.BasicDockerKeyring{} builtInKeyRing.Add(test.builtInDockerConfig) - fakeClient := NewFakeDockerClient() + fakeClient := libdocker.NewFakeDockerClient() dp := dockerPuller{ client: fakeClient, @@ -532,14 +176,8 @@ func TestPullWithSecrets(t *testing.T) { t.Errorf("%s: unexpected non-nil err: %s", i, err) continue } - - if e, a := 1, len(fakeClient.pulled); e != a { - t.Errorf("%s: expected 1 pulled image, got %d: %v", i, a, fakeClient.pulled) - continue - } - - if e, a := test.expectedPulls, fakeClient.pulled; !reflect.DeepEqual(e, a) { - t.Errorf("%s: expected pull of %v, but got %v", i, e, a) + if err := fakeClient.AssertImagesPulledMsgs(test.expectedPulls); err != nil { + t.Errorf("images pulled do not match the expected: %v", err) } } } diff --git a/pkg/kubelet/gpu/nvidia/nvidia_gpu_manager.go b/pkg/kubelet/gpu/nvidia/nvidia_gpu_manager.go index d83c83d386e..807cc8afd3c 100644 --- a/pkg/kubelet/gpu/nvidia/nvidia_gpu_manager.go +++ b/pkg/kubelet/gpu/nvidia/nvidia_gpu_manager.go @@ -29,7 +29,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/pkg/api/v1" - "k8s.io/kubernetes/pkg/kubelet/dockertools" + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" "k8s.io/kubernetes/pkg/kubelet/gpu" ) @@ -60,13 +60,13 @@ type nvidiaGPUManager struct { defaultDevices []string // The interface which could get GPU mapping from all the containers. // TODO: Should make this independent of Docker in the future. - dockerClient dockertools.DockerInterface + dockerClient libdocker.Interface activePodsLister activePodsLister } // NewNvidiaGPUManager returns a GPUManager that manages local Nvidia GPUs. // TODO: Migrate to use pod level cgroups and make it generic to all runtimes. -func NewNvidiaGPUManager(activePodsLister activePodsLister, dockerClient dockertools.DockerInterface) (gpu.GPUManager, error) { +func NewNvidiaGPUManager(activePodsLister activePodsLister, dockerClient libdocker.Interface) (gpu.GPUManager, error) { if dockerClient == nil { return nil, fmt.Errorf("invalid docker client specified") } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index d7a28aeacb9..87364a1d5a6 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -63,8 +63,8 @@ import ( "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockershim" + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" dockerremote "k8s.io/kubernetes/pkg/kubelet/dockershim/remote" - "k8s.io/kubernetes/pkg/kubelet/dockertools" "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/kubelet/eviction" "k8s.io/kubernetes/pkg/kubelet/gpu" @@ -219,7 +219,7 @@ type KubeletDeps struct { CAdvisorInterface cadvisor.Interface Cloud cloudprovider.Interface ContainerManager cm.ContainerManager - DockerClient dockertools.DockerInterface + DockerClient libdocker.Interface EventClient v1core.EventsGetter KubeClient clientset.Interface ExternalKubeClient clientgoclientset.Interface @@ -791,7 +791,7 @@ type Kubelet struct { hostname string nodeName types.NodeName - dockerClient dockertools.DockerInterface + dockerClient libdocker.Interface runtimeCache kubecontainer.RuntimeCache kubeClient clientset.Interface iptClient utilipt.Interface diff --git a/pkg/kubemark/hollow_kubelet.go b/pkg/kubemark/hollow_kubelet.go index 8b6eb47eeae..94edae2653a 100644 --- a/pkg/kubemark/hollow_kubelet.go +++ b/pkg/kubemark/hollow_kubelet.go @@ -30,7 +30,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/kubelet/cm" containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" - "k8s.io/kubernetes/pkg/kubelet/dockertools" + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" kubeio "k8s.io/kubernetes/pkg/util/io" "k8s.io/kubernetes/pkg/util/mount" @@ -52,7 +52,7 @@ func NewHollowKubelet( nodeName string, client *clientset.Clientset, cadvisorInterface cadvisor.Interface, - dockerClient dockertools.DockerInterface, + dockerClient libdocker.Interface, kubeletPort, kubeletReadOnlyPort int, containerManager cm.ContainerManager, maxPods int, podsPerCore int, diff --git a/test/e2e_node/garbage_collector_test.go b/test/e2e_node/garbage_collector_test.go index 80fea84de10..51db6c93ed4 100644 --- a/test/e2e_node/garbage_collector_test.go +++ b/test/e2e_node/garbage_collector_test.go @@ -23,7 +23,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/pkg/api/v1" - docker "k8s.io/kubernetes/pkg/kubelet/dockertools" + "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker" "k8s.io/kubernetes/test/e2e/framework" . "github.com/onsi/ginkgo" @@ -258,16 +258,16 @@ func containerGCTest(f *framework.Framework, test testRun) { // Runs containerGCTest using the docker runtime. func dockerContainerGCTest(f *framework.Framework, test testRun) { - var runtime docker.DockerInterface + var runtime libdocker.Interface BeforeEach(func() { - runtime = docker.ConnectToDockerOrDie(defaultDockerEndpoint, defaultRuntimeRequestTimeoutDuration, defaultImagePullProgressDeadline) + runtime = libdocker.ConnectToDockerOrDie(defaultDockerEndpoint, defaultRuntimeRequestTimeoutDuration, defaultImagePullProgressDeadline) }) for _, pod := range test.testPods { // Initialize the getContainerNames function to use the dockertools api thisPrefix := pod.containerPrefix pod.getContainerNames = func() ([]string, error) { relevantContainers := []string{} - dockerContainers, err := docker.GetKubeletDockerContainers(runtime, true) + dockerContainers, err := libdocker.GetKubeletDockerContainers(runtime, true) if err != nil { return relevantContainers, err }