From 19b12806f856c36f967bb1ab34af92618461a17d Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Wed, 1 Feb 2017 13:07:07 +0100 Subject: [PATCH 1/3] Makefile: fix make test-e2e-node example Signed-off-by: Antonio Murdaca --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a6b2fbef551..393627ee869 100644 --- a/Makefile +++ b/Makefile @@ -237,7 +237,7 @@ define TEST_E2E_NODE_HELP_INFO # Example: # make test-e2e-node FOCUS=Kubelet SKIP=container # make test-e2e-node REMOTE=true DELETE_INSTANCES=true -# make test-e2e-node TEST_ARGS="--cgroups-per-qos=true" +# make test-e2e-node TEST_ARGS='--kubelet-flags="--cgroups-per-qos=true"' # Build and run tests. endef .PHONY: test-e2e-node From caa6dd2599528515a511c9b06ed7db3dda8cbd06 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Sat, 4 Feb 2017 15:09:31 +0100 Subject: [PATCH 2/3] pkg/kubelet/remote: fix typo Signed-off-by: Antonio Murdaca --- pkg/kubelet/remote/remote_image.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/kubelet/remote/remote_image.go b/pkg/kubelet/remote/remote_image.go index fb1e3a70405..cd5c4307d64 100644 --- a/pkg/kubelet/remote/remote_image.go +++ b/pkg/kubelet/remote/remote_image.go @@ -35,16 +35,16 @@ type RemoteImageService struct { } // NewRemoteImageService creates a new internalapi.ImageManagerService. -func NewRemoteImageService(addr string, connectionTimout time.Duration) (internalapi.ImageManagerService, error) { +func NewRemoteImageService(addr string, connectionTimeout time.Duration) (internalapi.ImageManagerService, error) { glog.V(3).Infof("Connecting to image service %s", addr) - conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithTimeout(connectionTimout), grpc.WithDialer(dial)) + conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithTimeout(connectionTimeout), grpc.WithDialer(dial)) if err != nil { glog.Errorf("Connect remote image service %s failed: %v", addr, err) return nil, err } return &RemoteImageService{ - timeout: connectionTimout, + timeout: connectionTimeout, imageClient: runtimeapi.NewImageServiceClient(conn), }, nil } From 2634f57f7f48ac1ae0388e906d32855d79bd1de8 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Wed, 1 Feb 2017 14:46:47 +0100 Subject: [PATCH 3/3] test/e2e_node: prepull images with CRI Signed-off-by: Antonio Murdaca --- Makefile | 4 ++ hack/make-rules/test-e2e-node.sh | 13 ++++- test/e2e/framework/test_context.go | 4 ++ test/e2e_node/BUILD | 3 ++ test/e2e_node/image_list.go | 79 ++++++++++++++++++++++++++++-- 5 files changed, 98 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 393627ee869..89b027c76c2 100644 --- a/Makefile +++ b/Makefile @@ -233,6 +233,10 @@ define TEST_E2E_NODE_HELP_INFO # PARALLELISM: The number of gingko nodes to run. Defaults to 8. # RUNTIME: Container runtime to use (eg. docker, rkt, remote). # Defaults to "docker". +# CONTAINER_RUNTIME_ENDPOINT: remote container endpoint to connect to. +# Used when RUNTIME is set to "remote". +# IMAGE_SERVICE_ENDPOINT: remote image endpoint to connect to, to prepull images. +# Used when RUNTIME is set to "remote". # # Example: # make test-e2e-node FOCUS=Kubelet SKIP=container diff --git a/hack/make-rules/test-e2e-node.sh b/hack/make-rules/test-e2e-node.sh index 20cea09e81e..f55ddc3102d 100755 --- a/hack/make-rules/test-e2e-node.sh +++ b/hack/make-rules/test-e2e-node.sh @@ -29,6 +29,8 @@ parallelism=${PARALLELISM:-8} artifacts=${ARTIFACTS:-"/tmp/_artifacts/`date +%y%m%dT%H%M%S`"} remote=${REMOTE:-"false"} runtime=${RUNTIME:-"docker"} +container_runtime_endpoint=${CONTAINER_RUNTIME_ENDPOINT:-""} +image_service_endpoint=${IMAGE_SERVICE_ENDPOINT:-""} run_until_failure=${RUN_UNTIL_FAILURE:-"false"} test_args=${TEST_ARGS:-""} @@ -149,12 +151,21 @@ else test_args='--kubelet-flags="--container-runtime='$runtime'" '$test_args if [[ $runtime == "remote" ]] ; then test_args='--kubelet-flags="--experimental-cri=true" '$test_args + if [[ ! -z $container_runtime_endpoint ]] ; then + test_args='--kubelet-flags="--container-runtime-endpoint='$container_runtime_endpoint'" '$test_args + fi + if [[ ! -z $image_service_endpoint ]] ; then + test_args='--kubelet-flags="--image-service-endpoint='$image_service_endpoint'" '$test_args + fi fi # Test using the host the script was run on # Provided for backwards compatibility go run test/e2e_node/runner/local/run_local.go --ginkgo-flags="$ginkgoflags" \ - --test-flags="--container-runtime=${runtime} --alsologtostderr --v 4 --report-dir=${artifacts} --node-name $(hostname) \ + --test-flags="--container-runtime=${runtime} \ + --container-runtime-endpoint=${container_runtime_endpoint} \ + --image-service-endpoint=${image_service_endpoint} \ + --alsologtostderr --v 4 --report-dir=${artifacts} --node-name $(hostname) \ $test_args" --build-dependencies=true 2>&1 | tee -i "${artifacts}/build-log.txt" exit $? fi diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index acb3ff7d875..3d8f275c6c2 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -55,6 +55,8 @@ type TestContextType struct { GCEUpgradeScript string PrometheusPushGateway string ContainerRuntime string + ContainerRuntimeEndpoint string + ImageServiceEndpoint string MasterOSDistro string NodeOSDistro string VerifyServiceAccount bool @@ -165,6 +167,8 @@ func RegisterCommonFlags() { flag.StringVar(&TestContext.FeatureGates, "feature-gates", "", "A set of key=value pairs that describe feature gates for alpha/experimental features.") flag.StringVar(&TestContext.Viper, "viper-config", "e2e", "The name of the viper config i.e. 'e2e' will read values from 'e2e.json' locally. All e2e parameters are meant to be configurable by viper.") flag.StringVar(&TestContext.ContainerRuntime, "container-runtime", "docker", "The container runtime of cluster VM instances (docker/rkt/remote).") + flag.StringVar(&TestContext.ContainerRuntimeEndpoint, "container-runtime-endpoint", "", "The container runtime endpoint of cluster VM instances.") + flag.StringVar(&TestContext.ImageServiceEndpoint, "image-service-endpoint", "", "The image service endpoint of cluster VM instances.") } // Register flags specific to the cluster e2e test suite. diff --git a/test/e2e_node/BUILD b/test/e2e_node/BUILD index c12353fee1f..d97b0a8632a 100644 --- a/test/e2e_node/BUILD +++ b/test/e2e_node/BUILD @@ -29,7 +29,10 @@ go_library( "//pkg/apis/componentconfig/v1alpha1:go_default_library", "//pkg/client/clientset_generated/clientset:go_default_library", "//pkg/client/clientset_generated/clientset/typed/core/v1:go_default_library", + "//pkg/kubelet/api:go_default_library", + "//pkg/kubelet/api/v1alpha1/runtime:go_default_library", "//pkg/kubelet/api/v1alpha1/stats:go_default_library", + "//pkg/kubelet/remote:go_default_library", "//pkg/util/procfs:go_default_library", "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", diff --git a/test/e2e_node/image_list.go b/test/e2e_node/image_list.go index f4ac539147e..4497e5de6aa 100644 --- a/test/e2e_node/image_list.go +++ b/test/e2e_node/image_list.go @@ -17,6 +17,8 @@ limitations under the License. package e2e_node import ( + "errors" + "fmt" "os/exec" "os/user" "time" @@ -24,6 +26,9 @@ import ( "github.com/golang/glog" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/kubernetes/pkg/kubelet/api" + "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" + "k8s.io/kubernetes/pkg/kubelet/remote" commontest "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" ) @@ -33,6 +38,8 @@ const ( maxImagePullRetries = 5 // Sleep duration between image pull retry attempts. imagePullRetryDelay = time.Second + // connection timeout for gRPC image service connection + imageServiceConnectionTimeout = 15 * time.Minute ) // NodeImageWhiteList is a list of images used in node e2e test. These images will be prepulled @@ -54,14 +61,79 @@ func init() { framework.ImageWhiteList = NodeImageWhiteList.Union(commontest.CommonImageWhiteList) } +// puller represents a generic image puller +type puller interface { + // Pull pulls an image by name + Pull(image string) ([]byte, error) + // Name returns the name of the specific puller implementation + Name() string +} + +type dockerPuller struct { +} + +func (dp *dockerPuller) Name() string { + return "docker" +} + +func (dp *dockerPuller) Pull(image string) ([]byte, error) { + // TODO(random-liu): Use docker client to get rid of docker binary dependency. + return exec.Command("docker", "pull", image).CombinedOutput() +} + +type remotePuller struct { + imageService api.ImageManagerService +} + +func (rp *remotePuller) Name() string { + return "CRI" +} + +func (rp *remotePuller) Pull(image string) ([]byte, error) { + // TODO(runcom): should we check if the image is already pulled with ImageStatus? + _, err := rp.imageService.PullImage(&runtime.ImageSpec{Image: image}, nil) + return nil, err +} + +func getPuller() (puller, error) { + runtime := framework.TestContext.ContainerRuntime + switch runtime { + case "docker": + return &dockerPuller{}, nil + case "remote": + endpoint := framework.TestContext.ContainerRuntimeEndpoint + if framework.TestContext.ImageServiceEndpoint != "" { + //ImageServiceEndpoint is the same as ContainerRuntimeEndpoint if not + //explicitly specified + //https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/kubelet.go#L517 + endpoint = framework.TestContext.ImageServiceEndpoint + } + if endpoint == "" { + return nil, errors.New("can't prepull images, no remote endpoint provided") + } + is, err := remote.NewRemoteImageService(endpoint, imageServiceConnectionTimeout) + if err != nil { + return nil, err + } + return &remotePuller{ + imageService: is, + }, nil + } + return nil, fmt.Errorf("can't prepull images, unknown container runtime %q", runtime) +} + // Pre-fetch all images tests depend on so that we don't fail in an actual test. func PrePullAllImages() error { + puller, err := getPuller() + if err != nil { + return err + } usr, err := user.Current() if err != nil { return err } images := framework.ImageWhiteList.List() - glog.V(4).Infof("Pre-pulling images %+v", images) + glog.V(4).Infof("Pre-pulling images with %s %+v", puller.Name(), images) for _, image := range images { var ( err error @@ -71,15 +143,14 @@ func PrePullAllImages() error { if i > 0 { time.Sleep(imagePullRetryDelay) } - // TODO(random-liu): Use docker client to get rid of docker binary dependency. - if output, err = exec.Command("docker", "pull", image).CombinedOutput(); err == nil { + if output, err = puller.Pull(image); err == nil { break } glog.Warningf("Failed to pull %s as user %q, retrying in %s (%d of %d): %v", image, usr.Username, imagePullRetryDelay.String(), i+1, maxImagePullRetries, err) } if err != nil { - glog.Warningf("Could not pre-pull image %s %v output: %s", image, err, output) + glog.Warningf("Could not pre-pull image %s %v output: %s", image, err, output) return err } }