diff --git a/hack/ginkgo-e2e.sh b/hack/ginkgo-e2e.sh index 00e8136c62c..246b5ef945b 100755 --- a/hack/ginkgo-e2e.sh +++ b/hack/ginkgo-e2e.sh @@ -29,6 +29,7 @@ e2e_test=$(kube::util::find-binary "e2e.test") # --- Setup some env vars. GINKGO_PARALLEL=${GINKGO_PARALLEL:-n} # set to 'y' to run tests in parallel +CLOUD_CONFIG=${CLOUD_CONFIG:-""} # If 'y', Ginkgo's reporter will not print out in color when tests are run # in parallel @@ -97,6 +98,13 @@ if [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then NODE_INSTANCE_GROUP=$(kube::util::join , "${NODE_INSTANCE_GROUPS[@]}") fi +if [[ "${KUBERNETES_PROVIDER}" == "azure" ]]; then + if [[ ${CLOUD_CONFIG} == "" ]]; then + echo "Missing azure cloud config" + exit 1 + fi +fi + ginkgo_args=() if [[ -n "${CONFORMANCE_TEST_SKIP_REGEX:-}" ]]; then ginkgo_args+=("--skip=${CONFORMANCE_TEST_SKIP_REGEX}") @@ -137,6 +145,7 @@ export PATH=$(dirname "${e2e_test}"):"${PATH}" --gke-cluster="${CLUSTER_NAME:-}" \ --kube-master="${KUBE_MASTER:-}" \ --cluster-tag="${CLUSTER_ID:-}" \ + --cloud-config-file="${CLOUD_CONFIG:-}" \ --repo-root="${KUBE_ROOT}" \ --node-instance-group="${NODE_INSTANCE_GROUP:-}" \ --prefix="${KUBE_GCE_INSTANCE_PREFIX:-e2e}" \ diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 76f40b59b44..92cbfbe063a 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -97,6 +97,7 @@ clientset-name clientset-only clientset-path cloud-config +cloud-config-file cloud-provider cloud-provider-gce-lb-src-cidrs cluster-cidr diff --git a/test/e2e/BUILD b/test/e2e/BUILD index 5fa15d6ca00..d8e0a1b398f 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -130,6 +130,7 @@ go_library( "//pkg/client/clientset_generated/clientset/typed/extensions/v1beta1:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/cloudprovider:go_default_library", + "//pkg/cloudprovider/providers/azure:go_default_library", "//pkg/cloudprovider/providers/gce:go_default_library", "//pkg/controller:go_default_library", "//pkg/controller/daemon:go_default_library", diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 8e29e35de16..5df6943cea6 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -37,6 +37,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" + "k8s.io/kubernetes/pkg/cloudprovider/providers/azure" gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" "k8s.io/kubernetes/pkg/util/logs" commontest "k8s.io/kubernetes/test/e2e/common" @@ -85,6 +86,16 @@ func setupProviderConfig() error { if cloudConfig.Zone == "" { return fmt.Errorf("gce-zone must be specified for AWS") } + case "azure": + if cloudConfig.ConfigFile == "" { + return fmt.Errorf("config-file must be specified for Azure") + } + config, err := os.Open(cloudConfig.ConfigFile) + if err != nil { + framework.Logf("Couldn't open cloud provider configuration %s: %#v", + cloudConfig.ConfigFile, err) + } + cloudConfig.Provider, err = azure.NewCloud(config) } return nil diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index 0424d7f16b2..807a378c4dc 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -64,6 +64,7 @@ go_library( "//pkg/client/unversioned/remotecommand:go_default_library", "//pkg/cloudprovider:go_default_library", "//pkg/cloudprovider/providers/aws:go_default_library", + "//pkg/cloudprovider/providers/azure:go_default_library", "//pkg/cloudprovider/providers/gce:go_default_library", "//pkg/controller:go_default_library", "//pkg/controller/deployment/util:go_default_library", diff --git a/test/e2e/framework/pv_util.go b/test/e2e/framework/pv_util.go index a0cfbf5ac7b..dc6bf8693db 100644 --- a/test/e2e/framework/pv_util.go +++ b/test/e2e/framework/pv_util.go @@ -689,6 +689,19 @@ func createPD() (string, error) { volumeName := "aws://" + az + "/" + awsID return volumeName, nil + } else if TestContext.Provider == "azure" { + pdName := fmt.Sprintf("%s-%s", TestContext.Prefix, string(uuid.NewUUID())) + azureCloud, err := GetAzureCloud() + if err != nil { + return "", err + } + + _, diskUri, _, err := azureCloud.CreateVolume(pdName, "" /* account */, "" /* sku */, "" /* location */, 1 /* sizeGb */) + if err != nil { + return "", err + } + return diskUri, nil + } else { return "", fmt.Errorf("provider does not support volume creation") } @@ -728,6 +741,18 @@ func deletePD(pdName string) error { } } return nil + } else if TestContext.Provider == "azure" { + azureCloud, err := GetAzureCloud() + if err != nil { + return err + } + diskName := pdName[(strings.LastIndex(pdName, "/") + 1):] + err = azureCloud.DeleteVolume(diskName, pdName) + if err != nil { + Logf("failed to delete Azure volume %q: %v", pdName, err) + return err + } + return nil } else { return fmt.Errorf("provider does not support volume deletion") } diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 3e46c29a12f..d3931430581 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -141,6 +141,7 @@ type CloudConfig struct { NumNodes int ClusterTag string Network string + ConfigFile string // for azure and openstack Provider cloudprovider.Interface } @@ -209,6 +210,7 @@ func RegisterClusterFlags() { flag.IntVar(&cloudConfig.NumNodes, "num-nodes", -1, "Number of nodes in the cluster") flag.StringVar(&cloudConfig.ClusterTag, "cluster-tag", "", "Tag used to identify resources. Only required if provider is aws.") + flag.StringVar(&cloudConfig.ConfigFile, "cloud-config-file", "", "Cloud config file. Only required if provider is azure.") flag.IntVar(&TestContext.MinStartupPods, "minStartupPods", 0, "The number of pods which we need to see in 'Running' state with a 'Ready' condition of true, before we try running tests. This is useful in any cluster which needs some base pod-based services running before it can be used.") flag.DurationVar(&TestContext.SystemPodsStartupTimeout, "system-pods-startup-timeout", 10*time.Minute, "Timeout for waiting for all system pods to be running before starting tests.") flag.DurationVar(&TestContext.NodeSchedulableTimeout, "node-schedulable-timeout", 4*time.Hour, "Timeout for waiting for all nodes to be schedulable.") diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 20755e56b60..2b8c4a52752 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -83,6 +83,7 @@ import ( "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/client/conditions" + "k8s.io/kubernetes/pkg/cloudprovider/providers/azure" gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" "k8s.io/kubernetes/pkg/controller" deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util" @@ -5510,3 +5511,12 @@ func CreateEmptyFileOnPod(namespace string, podName string, filePath string) err _, err := RunKubectl("exec", fmt.Sprintf("--namespace=%s", namespace), podName, "--", "/bin/sh", "-c", fmt.Sprintf("touch %s", filePath)) return err } + +// GetAzureCloud returns azure cloud provider +func GetAzureCloud() (*azure.Cloud, error) { + cloud, ok := TestContext.CloudConfig.Provider.(*azure.Cloud) + if !ok { + return nil, fmt.Errorf("failed to convert CloudConfig.Provider to Azure: %#v", TestContext.CloudConfig.Provider) + } + return cloud, nil +} diff --git a/test/e2e/storage/volume_provisioning.go b/test/e2e/storage/volume_provisioning.go index ef6f1b4f2f0..950b831119a 100644 --- a/test/e2e/storage/volume_provisioning.go +++ b/test/e2e/storage/volume_provisioning.go @@ -356,6 +356,15 @@ var _ = framework.KubeDescribe("Dynamic Provisioning", func() { "1.5Gi", nil, }, + { + "Azure disk volume with empty sku and location", + []string{"azure"}, + "kubernetes.io/azure-disk", + map[string]string{}, + "1Gi", + "1Gi", + nil, + }, } var betaTest *storageClassTest @@ -463,7 +472,7 @@ var _ = framework.KubeDescribe("Dynamic Provisioning", func() { // not being deleted. // NOTE: Polls until no PVs are detected, times out at 5 minutes. - framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere") + framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure") const raceAttempts int = 100 var residualPVs []*v1.PersistentVolume @@ -561,7 +570,7 @@ var _ = framework.KubeDescribe("Dynamic Provisioning", func() { // Modifying the default storage class can be disruptive to other tests that depend on it It("should be disabled by changing the default annotation[Slow] [Serial] [Disruptive] [Volume]", func() { - framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere") + framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure") scName := getDefaultStorageClassName(c) test := storageClassTest{ name: "default", @@ -592,7 +601,7 @@ var _ = framework.KubeDescribe("Dynamic Provisioning", func() { // Modifying the default storage class can be disruptive to other tests that depend on it It("should be disabled by removing the default annotation[Slow] [Serial] [Disruptive] [Volume]", func() { - framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere") + framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure") scName := getDefaultStorageClassName(c) test := storageClassTest{ name: "default", @@ -753,6 +762,8 @@ func getDefaultPluginName() string { return "kubernetes.io/cinder" case framework.ProviderIs("vsphere"): return "kubernetes.io/vsphere-volume" + case framework.ProviderIs("azure"): + return "kubernetes.io/azure-disk" } return "" } diff --git a/test/e2e/storage/volumes.go b/test/e2e/storage/volumes.go index 7125bce1704..fed03ccaf6f 100644 --- a/test/e2e/storage/volumes.go +++ b/test/e2e/storage/volumes.go @@ -682,6 +682,56 @@ var _ = framework.KubeDescribe("Volumes [Volume]", func() { framework.InjectHtml(cs, config, tests[0].Volume, tests[0].ExpectedContent) + fsGroup := int64(1234) + framework.TestVolumeClient(cs, config, &fsGroup, tests) + }) + }) + //////////////////////////////////////////////////////////////////////// + // Azure Disk + //////////////////////////////////////////////////////////////////////// + framework.KubeDescribe("Azure Disk [Feature:Volumes]", func() { + It("should be mountable [Slow]", func() { + framework.SkipUnlessProviderIs("azure") + config := framework.VolumeTestConfig{ + Namespace: namespace.Name, + Prefix: "azure", + } + + By("creating a test azure disk volume") + volumeName, err := framework.CreatePDWithRetry() + Expect(err).NotTo(HaveOccurred()) + defer func() { + framework.DeletePDWithRetry(volumeName) + }() + + defer func() { + if clean { + framework.Logf("Running volumeTestCleanup") + framework.VolumeTestCleanup(f, config) + } + }() + fsType := "ext4" + readOnly := false + diskName := volumeName[(strings.LastIndex(volumeName, "/") + 1):] + tests := []framework.VolumeTest{ + { + Volume: v1.VolumeSource{ + AzureDisk: &v1.AzureDiskVolumeSource{ + DiskName: diskName, + DataDiskURI: volumeName, + FSType: &fsType, + ReadOnly: &readOnly, + }, + }, + File: "index.html", + // Randomize index.html to make sure we don't see the + // content from previous test runs. + ExpectedContent: "Hello from Azure from namespace " + volumeName, + }, + } + + framework.InjectHtml(cs, config, tests[0].Volume, tests[0].ExpectedContent) + fsGroup := int64(1234) framework.TestVolumeClient(cs, config, &fsGroup, tests) })