From f64c508e2e85393c5f59d26abd630e7c11791b81 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Mon, 8 Jan 2018 18:46:51 +0000 Subject: [PATCH] Add getCRIClient and set default values for CRI related flags --- hack/make-rules/test-e2e-node.sh | 20 ++++++++++---------- test/e2e/framework/test_context.go | 22 +++++++++++++--------- test/e2e_node/image_list.go | 16 +--------------- test/e2e_node/services/kubelet.go | 12 ++++++++++++ test/e2e_node/util.go | 24 ++++++++++++++++++++++++ 5 files changed, 60 insertions(+), 34 deletions(-) diff --git a/hack/make-rules/test-e2e-node.sh b/hack/make-rules/test-e2e-node.sh index 1cff4c85d8e..5733f1eb442 100755 --- a/hack/make-rules/test-e2e-node.sh +++ b/hack/make-rules/test-e2e-node.sh @@ -60,6 +60,16 @@ if [ ! -d "${artifacts}" ]; then fi echo "Test artifacts will be written to ${artifacts}" +if [[ $runtime == "remote" ]] ; then + if [[ ! -z $container_runtime_endpoint ]] ; then + test_args="--container-runtime-endpoint=${container_runtime_endpoint} $test_args" + fi + if [[ ! -z $image_service_endpoint ]] ; then + test_args="--image-service-endpoint=$image_service_endpoint $test_args" + fi +fi + + if [ $remote = true ] ; then # The following options are only valid in remote run. images=${IMAGES:-""} @@ -153,22 +163,12 @@ else # Runtime flags test_args='--kubelet-flags="--container-runtime='$runtime'" '$test_args - if [[ $runtime == "remote" ]] ; then - if [[ ! -z $container_runtime_endpoint ]] ; then - test_args='--kubelet-flags="--container-runtime-endpoint='$container_runtime_endpoint'" '$test_args - fi - if [[ ! -z $image_service_endpoint ]] ; then - test_args='--kubelet-flags="--image-service-endpoint='$image_service_endpoint'" '$test_args - fi - fi # Test using the host the script was run on # Provided for backwards compatibility go run test/e2e_node/runner/local/run_local.go \ --system-spec-name="$system_spec_name" --ginkgo-flags="$ginkgoflags" \ --test-flags="--container-runtime=${runtime} \ - --container-runtime-endpoint=${container_runtime_endpoint} \ - --image-service-endpoint=${image_service_endpoint} \ --alsologtostderr --v 4 --report-dir=${artifacts} --node-name $(hostname) \ $test_args" --build-dependencies=true 2>&1 | tee -i "${artifacts}/build-log.txt" exit $? diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 54e261abff3..31b2dde1855 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -57,14 +57,16 @@ type TestContextType struct { Prefix string MinStartupPods int // Timeout for waiting for system pods to be running - SystemPodsStartupTimeout time.Duration - UpgradeTarget string - EtcdUpgradeStorage string - EtcdUpgradeVersion string - UpgradeImage string - GCEUpgradeScript string - ContainerRuntime string - ContainerRuntimeEndpoint string + SystemPodsStartupTimeout time.Duration + UpgradeTarget string + EtcdUpgradeStorage string + EtcdUpgradeVersion string + UpgradeImage string + GCEUpgradeScript string + ContainerRuntime string + ContainerRuntimeEndpoint string + ContainerRuntimeProcessName string + ContainerRuntimePidFile string // SystemdServices are comma separated list of systemd services the test framework // will dump logs for. SystemdServices string @@ -203,7 +205,9 @@ func RegisterCommonFlags() { flag.StringVar(&TestContext.FeatureGates, "feature-gates", "", "A set of key=value pairs that describe feature gates for alpha/experimental features.") flag.StringVar(&TestContext.Viper, "viper-config", "e2e", "The name of the viper config i.e. 'e2e' will read values from 'e2e.json' locally. All e2e parameters are meant to be configurable by viper.") flag.StringVar(&TestContext.ContainerRuntime, "container-runtime", "docker", "The container runtime of cluster VM instances (docker/rkt/remote).") - flag.StringVar(&TestContext.ContainerRuntimeEndpoint, "container-runtime-endpoint", "", "The container runtime endpoint of cluster VM instances.") + flag.StringVar(&TestContext.ContainerRuntimeEndpoint, "container-runtime-endpoint", "unix:///var/run/dockershim.sock", "The container runtime endpoint of cluster VM instances.") + flag.StringVar(&TestContext.ContainerRuntimeProcessName, "container-runtime-process-name", "dockerd", "The name of the container runtime process.") + flag.StringVar(&TestContext.ContainerRuntimePidFile, "container-runtime-pid-file", "/var/run/docker.pid", "The pid file of the container runtime.") flag.StringVar(&TestContext.SystemdServices, "systemd-services", "docker", "The comma separated list of systemd services the framework will dump logs for.") flag.StringVar(&TestContext.ImageServiceEndpoint, "image-service-endpoint", "", "The image service endpoint of cluster VM instances.") flag.StringVar(&TestContext.DockershimCheckpointDir, "dockershim-checkpoint-dir", "/var/lib/dockershim/sandbox", "The directory for dockershim to store sandbox checkpoints.") diff --git a/test/e2e_node/image_list.go b/test/e2e_node/image_list.go index 01360451fcb..0c37c88752d 100644 --- a/test/e2e_node/image_list.go +++ b/test/e2e_node/image_list.go @@ -17,7 +17,6 @@ limitations under the License. package e2e_node import ( - "errors" "fmt" "os/exec" "os/user" @@ -28,7 +27,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" - "k8s.io/kubernetes/pkg/kubelet/remote" commontest "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" imageutils "k8s.io/kubernetes/test/utils/image" @@ -39,8 +37,6 @@ const ( maxImagePullRetries = 5 // Sleep duration between image pull retry attempts. imagePullRetryDelay = time.Second - // connection timeout for gRPC image service connection - imageServiceConnectionTimeout = 15 * time.Minute ) // NodeImageWhiteList is a list of images used in node e2e test. These images will be prepulled @@ -107,17 +103,7 @@ func getPuller() (puller, error) { case "docker": return &dockerPuller{}, nil case "remote": - endpoint := framework.TestContext.ContainerRuntimeEndpoint - if framework.TestContext.ImageServiceEndpoint != "" { - //ImageServiceEndpoint is the same as ContainerRuntimeEndpoint if not - //explicitly specified - //https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/kubelet.go#L517 - endpoint = framework.TestContext.ImageServiceEndpoint - } - if endpoint == "" { - return nil, errors.New("can't prepull images, no remote endpoint provided") - } - is, err := remote.NewRemoteImageService(endpoint, imageServiceConnectionTimeout) + _, is, err := getCRIClient() if err != nil { return nil, err } diff --git a/test/e2e_node/services/kubelet.go b/test/e2e_node/services/kubelet.go index aa803859a39..1fee35acfd0 100644 --- a/test/e2e_node/services/kubelet.go +++ b/test/e2e_node/services/kubelet.go @@ -303,6 +303,18 @@ func (e *E2EServices) startKubelet() (*server, error) { cmdArgs = append(cmdArgs, "--hostname-override", framework.TestContext.NodeName) } + if framework.TestContext.ContainerRuntime != "" { + cmdArgs = append(cmdArgs, "--container-runtime", framework.TestContext.ContainerRuntime) + } + + if framework.TestContext.ContainerRuntimeEndpoint != "" { + cmdArgs = append(cmdArgs, "--container-runtime-endpoint", framework.TestContext.ContainerRuntimeEndpoint) + } + + if framework.TestContext.ImageServiceEndpoint != "" { + cmdArgs = append(cmdArgs, "--image-service-endpoint", framework.TestContext.ImageServiceEndpoint) + } + // Write config file or flags, depending on whether --generate-kubelet-config-file was provided if genKubeletConfigFile { if err := writeKubeletConfigFile(kc, kubeletConfigPath); err != nil { diff --git a/test/e2e_node/util.go b/test/e2e_node/util.go index 0d00f88944e..bf1914c5e71 100644 --- a/test/e2e_node/util.go +++ b/test/e2e_node/util.go @@ -35,11 +35,13 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes/scheme" "k8s.io/kubernetes/pkg/features" + internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics" + "k8s.io/kubernetes/pkg/kubelet/remote" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/metrics" @@ -365,3 +367,25 @@ func runCommand(cmd ...string) (string, error) { } return string(output), nil } + +// getCRIClient connects CRI and returns CRI runtime service clients and image service client. +func getCRIClient() (internalapi.RuntimeService, internalapi.ImageManagerService, error) { + // connection timeout for CRI service connection + const connectionTimeout = 2 * time.Minute + runtimeEndpoint := framework.TestContext.ContainerRuntimeEndpoint + r, err := remote.NewRemoteRuntimeService(runtimeEndpoint, connectionTimeout) + if err != nil { + return nil, nil, err + } + imageManagerEndpoint := runtimeEndpoint + if framework.TestContext.ImageServiceEndpoint != "" { + //ImageServiceEndpoint is the same as ContainerRuntimeEndpoint if not + //explicitly specified + imageManagerEndpoint = framework.TestContext.ImageServiceEndpoint + } + i, err := remote.NewRemoteImageService(imageManagerEndpoint, connectionTimeout) + if err != nil { + return nil, nil, err + } + return r, i, nil +}