From 310168c1d2e32c4eb94a19e828c0f663c80f1166 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Fri, 27 Oct 2017 08:52:46 +0000 Subject: [PATCH 1/6] fix CreateVolume: search mode for Dedicated kind --- .../azure/azure_blobDiskController.go | 116 ++++++++---------- pkg/volume/azure_dd/azure_dd.go | 4 +- pkg/volume/azure_dd/azure_provision.go | 24 +--- test/e2e/framework/pv_util.go | 15 +-- 4 files changed, 60 insertions(+), 99 deletions(-) diff --git a/pkg/cloudprovider/providers/azure/azure_blobDiskController.go b/pkg/cloudprovider/providers/azure/azure_blobDiskController.go index 32c009483b4..3a25452da7c 100644 --- a/pkg/cloudprovider/providers/azure/azure_blobDiskController.go +++ b/pkg/cloudprovider/providers/azure/azure_blobDiskController.go @@ -80,52 +80,55 @@ func newBlobDiskController(common *controllerCommon) (*BlobDiskController, error return &c, nil } -// CreateVolume creates a VHD blob in a given storage account, will create the given storage account if it does not exist in current resource group +// CreateVolume creates a VHD blob in a storage account that has storageType and location using the given storage account. +// If no storage account is given, search all the storage accounts associated with the resource group and pick one that +// fits storage type and location. func (c *BlobDiskController) CreateVolume(name, storageAccount string, storageAccountType storage.SkuName, location string, requestGB int) (string, string, int, error) { - key, err := c.common.cloud.getStorageAccesskey(storageAccount) - if err != nil { - glog.V(2).Infof("azureDisk - no key found for storage account %s in resource group %s, begin to create a new storage account", storageAccount, c.common.resourceGroup) - - cp := storage.AccountCreateParameters{ - Sku: &storage.Sku{Name: storageAccountType}, - Tags: &map[string]*string{"created-by": to.StringPtr("azure-dd")}, - Location: &location} - cancel := make(chan struct{}) - - _, errchan := c.common.cloud.StorageAccountClient.Create(c.common.resourceGroup, storageAccount, cp, cancel) - err = <-errchan + var err error + accounts := []accountWithLocation{} + if len(storageAccount) > 0 { + accounts = append(accounts, accountWithLocation{Name: storageAccount}) + } else { + // find a storage account + accounts, err = c.common.cloud.getStorageAccounts() if err != nil { - return "", "", 0, fmt.Errorf(fmt.Sprintf("Create Storage Account %s, error: %s", storageAccount, err)) + // TODO: create a storage account and container + return "", "", 0, err } + } + for _, account := range accounts { + glog.V(4).Infof("account %s type %s location %s", account.Name, account.StorageType, account.Location) + if (account.StorageType == string(storageAccountType)) && (location == "" || account.Location == location) { + // find the access key with this account + key, err := c.common.cloud.getStorageAccesskey(account.Name) + if err != nil { + glog.V(2).Infof("no key found for storage account %s", account.Name) + continue + } - key, err = c.common.cloud.getStorageAccesskey(storageAccount) - if err != nil { - return "", "", 0, fmt.Errorf("no key found for storage account %s even after creating a new storage account", storageAccount) + client, err := azstorage.NewBasicClientOnSovereignCloud(account.Name, key, c.common.cloud.Environment) + if err != nil { + return "", "", 0, err + } + blobClient := client.GetBlobService() + + container := blobClient.GetContainerReference(vhdContainerName) + _, err = container.CreateIfNotExists(&azstorage.CreateContainerOptions{Access: azstorage.ContainerAccessTypePrivate}) + if err != nil { + return "", "", 0, err + } + + // create a page blob in this account's vhd container + diskName, diskURI, err := c.createVHDBlobDisk(blobClient, account.Name, name, vhdContainerName, int64(requestGB)) + if err != nil { + return "", "", 0, err + } + + glog.V(4).Infof("azureDisk - created vhd blob uri: %s", diskURI) + return diskName, diskURI, requestGB, err } - - glog.Errorf("no key found for storage account %s in resource group %s", storageAccount, c.common.resourceGroup) - return "", "", 0, err } - - client, err := azstorage.NewBasicClientOnSovereignCloud(storageAccount, key, c.common.cloud.Environment) - if err != nil { - return "", "", 0, err - } - blobClient := client.GetBlobService() - - container := blobClient.GetContainerReference(vhdContainerName) - _, err = container.CreateIfNotExists(&azstorage.CreateContainerOptions{Access: azstorage.ContainerAccessTypePrivate}) - if err != nil { - return "", "", 0, err - } - - diskName, diskURI, err := c.createVHDBlobDisk(blobClient, storageAccount, name, vhdContainerName, int64(requestGB)) - if err != nil { - return "", "", 0, err - } - - glog.V(4).Infof("azureDisk - created vhd blob uri: %s", diskURI) - return diskName, diskURI, requestGB, err + return "", "", 0, fmt.Errorf("failed to find a matching storage account") } // DeleteVolume deletes a VHD blob @@ -236,24 +239,12 @@ func (c *BlobDiskController) deleteVhdBlob(accountName, accountKey, blobName str } //CreateBlobDisk : create a blob disk in a node -func (c *BlobDiskController) CreateBlobDisk(dataDiskName string, storageAccountType storage.SkuName, sizeGB int, forceStandAlone bool) (string, error) { - glog.V(4).Infof("azureDisk - creating blob data disk named:%s on StorageAccountType:%s StandAlone:%v", dataDiskName, storageAccountType, forceStandAlone) +func (c *BlobDiskController) CreateBlobDisk(dataDiskName string, storageAccountType storage.SkuName, sizeGB int) (string, error) { + glog.V(4).Infof("azureDisk - creating blob data disk named:%s on StorageAccountType:%s", dataDiskName, storageAccountType) - var storageAccountName = "" - var err error - - if forceStandAlone { - // we have to wait until the storage account is is created - storageAccountName = "p" + MakeCRC32(c.common.subscriptionID+c.common.resourceGroup+dataDiskName) - err = c.createStorageAccount(storageAccountName, storageAccountType, c.common.location, false) - if err != nil { - return "", err - } - } else { - storageAccountName, err = c.findSANameForDisk(storageAccountType) - if err != nil { - return "", err - } + storageAccountName, err := c.findSANameForDisk(storageAccountType) + if err != nil { + return "", err } blobClient, err := c.getBlobSvcClient(storageAccountName) @@ -266,15 +257,13 @@ func (c *BlobDiskController) CreateBlobDisk(dataDiskName string, storageAccountT return "", err } - if !forceStandAlone { - atomic.AddInt32(&c.accounts[storageAccountName].diskCount, 1) - } + atomic.AddInt32(&c.accounts[storageAccountName].diskCount, 1) return diskURI, nil } //DeleteBlobDisk : delete a blob disk from a node -func (c *BlobDiskController) DeleteBlobDisk(diskURI string, wasForced bool) error { +func (c *BlobDiskController) DeleteBlobDisk(diskURI string) error { storageAccountName, vhdName, err := diskNameandSANameFromURI(diskURI) if err != nil { return err @@ -286,11 +275,6 @@ func (c *BlobDiskController) DeleteBlobDisk(diskURI string, wasForced bool) erro glog.V(4).Infof("azureDisk - deleting volume %s", diskURI) return c.DeleteVolume(diskURI) } - // if forced (as in one disk = one storage account) - // delete the account completely - if wasForced { - return c.deleteStorageAccount(storageAccountName) - } blobSvc, err := c.getBlobSvcClient(storageAccountName) if err != nil { diff --git a/pkg/volume/azure_dd/azure_dd.go b/pkg/volume/azure_dd/azure_dd.go index bb45bf3b4d9..5ebbc151e39 100644 --- a/pkg/volume/azure_dd/azure_dd.go +++ b/pkg/volume/azure_dd/azure_dd.go @@ -28,8 +28,8 @@ import ( // interface exposed by the cloud provider implementing Disk functionlity type DiskController interface { - CreateBlobDisk(dataDiskName string, storageAccountType storage.SkuName, sizeGB int, forceStandAlone bool) (string, error) - DeleteBlobDisk(diskUri string, wasForced bool) error + CreateBlobDisk(dataDiskName string, storageAccountType storage.SkuName, sizeGB int) (string, error) + DeleteBlobDisk(diskUri string) error CreateManagedDisk(diskName string, storageAccountType storage.SkuName, sizeGB int, tags map[string]string) (string, error) DeleteManagedDisk(diskURI string) error diff --git a/pkg/volume/azure_dd/azure_provision.go b/pkg/volume/azure_dd/azure_provision.go index d037f636d80..408f871dabb 100644 --- a/pkg/volume/azure_dd/azure_provision.go +++ b/pkg/volume/azure_dd/azure_provision.go @@ -55,14 +55,13 @@ func (d *azureDiskDeleter) Delete() error { return err } - wasStandAlone := (*volumeSource.Kind != v1.AzureSharedBlobDisk) managed := (*volumeSource.Kind == v1.AzureManagedDisk) if managed { return diskController.DeleteManagedDisk(volumeSource.DataDiskURI) } - return diskController.DeleteBlobDisk(volumeSource.DataDiskURI, wasStandAlone) + return diskController.DeleteBlobDisk(volumeSource.DataDiskURI) } func (p *azureDiskProvisioner) Provision() (*v1.PersistentVolume, error) { @@ -149,26 +148,13 @@ func (p *azureDiskProvisioner) Provision() (*v1.PersistentVolume, error) { return nil, err } } else { - forceStandAlone := (kind == v1.AzureDedicatedBlobDisk) if kind == v1.AzureDedicatedBlobDisk { - if location != "" && account != "" { - // use dedicated kind (by default) for compatibility - _, diskURI, _, err = diskController.CreateVolume(name, account, skuName, location, requestGB) - if err != nil { - return nil, err - } - } else { - if location != "" || account != "" { - return nil, fmt.Errorf("AzureDisk - location(%s) and account(%s) must be both empty or specified for dedicated kind, only one value specified is not allowed", - location, account) - } - diskURI, err = diskController.CreateBlobDisk(name, skuName, requestGB, forceStandAlone) - if err != nil { - return nil, err - } + _, diskURI, _, err = diskController.CreateVolume(name, account, skuName, location, requestGB) + if err != nil { + return nil, err } } else { - diskURI, err = diskController.CreateBlobDisk(name, skuName, requestGB, forceStandAlone) + diskURI, err = diskController.CreateBlobDisk(name, skuName, requestGB) if err != nil { return nil, err } diff --git a/test/e2e/framework/pv_util.go b/test/e2e/framework/pv_util.go index 92c91a70cb9..f39b1e6d5e4 100644 --- a/test/e2e/framework/pv_util.go +++ b/test/e2e/framework/pv_util.go @@ -718,16 +718,11 @@ func createPD(zone string) (string, error) { return "", err } - if azureCloud.BlobDiskController == nil { - return "", fmt.Errorf("BlobDiskController is nil, it's not expected.") - } - - diskUri, err := azureCloud.BlobDiskController.CreateBlobDisk(pdName, "standard_lrs", 1, false) + _, diskURI, _, err := azureCloud.CreateVolume(pdName, "" /* account */, "" /* sku */, "" /* location */, 1 /* sizeGb */) if err != nil { return "", err } - - return diskUri, nil + return diskURI, nil } else { return "", fmt.Errorf("provider does not support volume creation") } @@ -772,11 +767,7 @@ func deletePD(pdName string) error { if err != nil { return err } - if azureCloud.BlobDiskController == nil { - return fmt.Errorf("BlobDiskController is nil, it's not expected.") - } - diskName := pdName[(strings.LastIndex(pdName, "/") + 1):] - err = azureCloud.BlobDiskController.DeleteBlobDisk(diskName, false) + err = azureCloud.DeleteVolume(pdName) if err != nil { Logf("failed to delete Azure volume %q: %v", pdName, err) return err From 760cc6f78bb9c6b09726f93b8bc7063a20c9d799 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Fri, 27 Oct 2017 09:12:52 +0000 Subject: [PATCH 2/6] search by accounttype in CreateVolume func fix review comments --- .../azure/azure_blobDiskController.go | 38 +++++++++---------- pkg/volume/azure_dd/azure_dd.go | 2 +- pkg/volume/azure_dd/azure_provision.go | 2 +- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/pkg/cloudprovider/providers/azure/azure_blobDiskController.go b/pkg/cloudprovider/providers/azure/azure_blobDiskController.go index 3a25452da7c..d502808b46c 100644 --- a/pkg/cloudprovider/providers/azure/azure_blobDiskController.go +++ b/pkg/cloudprovider/providers/azure/azure_blobDiskController.go @@ -59,11 +59,12 @@ type BlobDiskController struct { accounts map[string]*storageAccountState } -var defaultContainerName = "" -var storageAccountNamePrefix = "" -var storageAccountNameMatch = "" - -var accountsLock = &sync.Mutex{} +var ( + defaultContainerName = "" + storageAccountNamePrefix = "" + storageAccountNameMatch = "" + accountsLock = &sync.Mutex{} +) func newBlobDiskController(common *controllerCommon) (*BlobDiskController, error) { c := BlobDiskController{common: common} @@ -83,7 +84,7 @@ func newBlobDiskController(common *controllerCommon) (*BlobDiskController, error // CreateVolume creates a VHD blob in a storage account that has storageType and location using the given storage account. // If no storage account is given, search all the storage accounts associated with the resource group and pick one that // fits storage type and location. -func (c *BlobDiskController) CreateVolume(name, storageAccount string, storageAccountType storage.SkuName, location string, requestGB int) (string, string, int, error) { +func (c *BlobDiskController) CreateVolume(name, storageAccount, storageAccountType, location string, requestGB int) (string, string, int, error) { var err error accounts := []accountWithLocation{} if len(storageAccount) > 0 { @@ -98,7 +99,7 @@ func (c *BlobDiskController) CreateVolume(name, storageAccount string, storageAc } for _, account := range accounts { glog.V(4).Infof("account %s type %s location %s", account.Name, account.StorageType, account.Location) - if (account.StorageType == string(storageAccountType)) && (location == "" || account.Location == location) { + if (storageAccountType == "" || account.StorageType == storageAccountType) && (location == "" || account.Location == location) || len(storageAccount) > 0 { // find the access key with this account key, err := c.common.cloud.getStorageAccesskey(account.Name) if err != nil { @@ -112,12 +113,6 @@ func (c *BlobDiskController) CreateVolume(name, storageAccount string, storageAc } blobClient := client.GetBlobService() - container := blobClient.GetContainerReference(vhdContainerName) - _, err = container.CreateIfNotExists(&azstorage.CreateContainerOptions{Access: azstorage.ContainerAccessTypePrivate}) - if err != nil { - return "", "", 0, err - } - // create a page blob in this account's vhd container diskName, diskURI, err := c.createVHDBlobDisk(blobClient, account.Name, name, vhdContainerName, int64(requestGB)) if err != nil { @@ -176,11 +171,6 @@ func (c *BlobDiskController) getBlobNameAndAccountFromURI(diskURI string) (strin func (c *BlobDiskController) createVHDBlobDisk(blobClient azstorage.BlobStorageClient, accountName, vhdName, containerName string, sizeGB int64) (string, string, error) { container := blobClient.GetContainerReference(containerName) - _, err := container.CreateIfNotExists(&azstorage.CreateContainerOptions{Access: azstorage.ContainerAccessTypePrivate}) - if err != nil { - return "", "", err - } - size := 1024 * 1024 * 1024 * sizeGB vhdSize := size + vhd.VHD_HEADER_SIZE /* header size */ // Blob name in URL must end with '.vhd' extension. @@ -193,7 +183,17 @@ func (c *BlobDiskController) createVHDBlobDisk(blobClient azstorage.BlobStorageC blob := container.GetBlobReference(vhdName) blob.Properties.ContentLength = vhdSize blob.Metadata = tags - err = blob.PutPageBlob(nil) + err := blob.PutPageBlob(nil) + if err != nil { + // if container doesn't exist, create one and retry PutPageBlob + detail := err.Error() + if strings.Contains(detail, errContainerNotFound) { + err = container.Create(&azstorage.CreateContainerOptions{Access: azstorage.ContainerAccessTypePrivate}) + if err == nil { + err = blob.PutPageBlob(nil) + } + } + } if err != nil { return "", "", fmt.Errorf("failed to put page blob %s in container %s: %v", vhdName, containerName, err) } diff --git a/pkg/volume/azure_dd/azure_dd.go b/pkg/volume/azure_dd/azure_dd.go index 5ebbc151e39..09e4cdb6353 100644 --- a/pkg/volume/azure_dd/azure_dd.go +++ b/pkg/volume/azure_dd/azure_dd.go @@ -48,7 +48,7 @@ type DiskController interface { GetNextDiskLun(nodeName types.NodeName) (int32, error) // Create a VHD blob - CreateVolume(name, storageAccount string, storageAccountType storage.SkuName, location string, requestGB int) (string, string, int, error) + CreateVolume(name, storageAccount, storageAccountType, location string, requestGB int) (string, string, int, error) // Delete a VHD blob DeleteVolume(diskURI string) error } diff --git a/pkg/volume/azure_dd/azure_provision.go b/pkg/volume/azure_dd/azure_provision.go index 408f871dabb..5f11743d52b 100644 --- a/pkg/volume/azure_dd/azure_provision.go +++ b/pkg/volume/azure_dd/azure_provision.go @@ -149,7 +149,7 @@ func (p *azureDiskProvisioner) Provision() (*v1.PersistentVolume, error) { } } else { if kind == v1.AzureDedicatedBlobDisk { - _, diskURI, _, err = diskController.CreateVolume(name, account, skuName, location, requestGB) + _, diskURI, _, err = diskController.CreateVolume(name, account, storageAccountType, location, requestGB) if err != nil { return nil, err } From 445393fdcefa6d0354b7ce32a2304a7765fbd305 Mon Sep 17 00:00:00 2001 From: Slava Semushin Date: Tue, 28 Nov 2017 17:28:58 +0100 Subject: [PATCH 3/6] kubelet: MustRunAsNonRoot should reject a pod if it has non-numeric USER. --- pkg/kubelet/kuberuntime/kuberuntime_container.go | 11 ++++------- .../kuberuntime/kuberuntime_container_test.go | 14 +++++++++++++- pkg/kubelet/kuberuntime/security_context.go | 11 +++++++---- pkg/kubelet/kuberuntime/security_context_test.go | 3 ++- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container.go b/pkg/kubelet/kuberuntime/kuberuntime_container.go index 1b622b2ca66..de0b6fdd7f3 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container.go @@ -183,13 +183,10 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Contai if err != nil { return nil, err } - if uid != nil { - // Verify RunAsNonRoot. Non-root verification only supports numeric user. - if err := verifyRunAsNonRoot(pod, container, *uid); err != nil { - return nil, err - } - } else if username != "" { - glog.Warningf("Non-root verification doesn't support non-numeric user (%s)", username) + + // Verify RunAsNonRoot. Non-root verification only supports numeric user. + if err := verifyRunAsNonRoot(pod, container, uid, username); err != nil { + return nil, err } command, args := kubecontainer.ExpandContainerCommandAndArgs(container, opts.Envs) diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_test.go b/pkg/kubelet/kuberuntime/kuberuntime_container_test.go index 86f153ca40f..c91a3a82e3f 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_test.go @@ -236,7 +236,7 @@ func makeExpectedConfig(m *kubeGenericRuntimeManager, pod *v1.Pod, containerInde } func TestGenerateContainerConfig(t *testing.T) { - _, _, m, err := createTestRuntimeManager() + _, imageService, m, err := createTestRuntimeManager() assert.NoError(t, err) pod := &v1.Pod{ @@ -290,6 +290,18 @@ func TestGenerateContainerConfig(t *testing.T) { _, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image) assert.Error(t, err) + + imageId, _ := imageService.PullImage(&runtimeapi.ImageSpec{Image: "busybox"}, nil) + image, _ := imageService.ImageStatus(&runtimeapi.ImageSpec{Image: imageId}) + + image.Uid = nil + image.Username = "test" + + podWithContainerSecurityContext.Spec.Containers[0].SecurityContext.RunAsUser = nil + podWithContainerSecurityContext.Spec.Containers[0].SecurityContext.RunAsNonRoot = &runAsNonRootTrue + + _, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image) + assert.Error(t, err, "RunAsNonRoot should fail for non-numeric username") } func TestLifeCycleHook(t *testing.T) { diff --git a/pkg/kubelet/kuberuntime/security_context.go b/pkg/kubelet/kuberuntime/security_context.go index c0d4ce7347a..3d48d372ac4 100644 --- a/pkg/kubelet/kuberuntime/security_context.go +++ b/pkg/kubelet/kuberuntime/security_context.go @@ -75,7 +75,7 @@ func (m *kubeGenericRuntimeManager) determineEffectiveSecurityContext(pod *v1.Po } // verifyRunAsNonRoot verifies RunAsNonRoot. -func verifyRunAsNonRoot(pod *v1.Pod, container *v1.Container, uid int64) error { +func verifyRunAsNonRoot(pod *v1.Pod, container *v1.Container, uid *int64, username string) error { effectiveSc := securitycontext.DetermineEffectiveSecurityContext(pod, container) // If the option is not set, or if running as root is allowed, return nil. if effectiveSc == nil || effectiveSc.RunAsNonRoot == nil || !*effectiveSc.RunAsNonRoot { @@ -89,11 +89,14 @@ func verifyRunAsNonRoot(pod *v1.Pod, container *v1.Container, uid int64) error { return nil } - if uid == 0 { + switch { + case uid != nil && *uid == 0: return fmt.Errorf("container has runAsNonRoot and image will run as root") + case uid == nil && len(username) > 0: + return fmt.Errorf("container has runAsNonRoot and image has non-numeric user (%s), cannot verify user is non-root", username) + default: + return nil } - - return nil } // convertToRuntimeSecurityContext converts v1.SecurityContext to runtimeapi.SecurityContext. diff --git a/pkg/kubelet/kuberuntime/security_context_test.go b/pkg/kubelet/kuberuntime/security_context_test.go index ae724ab314a..d3261b51f5b 100644 --- a/pkg/kubelet/kuberuntime/security_context_test.go +++ b/pkg/kubelet/kuberuntime/security_context_test.go @@ -105,7 +105,8 @@ func TestVerifyRunAsNonRoot(t *testing.T) { }, } { pod.Spec.Containers[0].SecurityContext = test.sc - err := verifyRunAsNonRoot(pod, &pod.Spec.Containers[0], int64(0)) + uid := int64(0) + err := verifyRunAsNonRoot(pod, &pod.Spec.Containers[0], &uid, "") if test.fail { assert.Error(t, err, test.desc) } else { From d7341749ff26a071b0971dba3f779277b93d76df Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Wed, 29 Nov 2017 11:31:39 -0800 Subject: [PATCH 4/6] nvidia-gpu-device-plugin daemonset should tolerate nvidia.com/gpu taint. It is expected that nodes with extended resources attached will be tainted with the resouce name, so that we can create dedicated nodes. If ExtendedResourceToleration admission controller is enabled, pods requesting such resources will automatically tolerate such taints. nvidia-gpu-device-plugin daemonset doesn't request such resources but still needs to run on such nodes, so it needs this toleration. --- cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml b/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml index 6b5edbf733d..de66faecb30 100644 --- a/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml +++ b/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml @@ -22,6 +22,10 @@ spec: - matchExpressions: - key: cloud.google.com/gke-accelerator operator: Exists + tolerations: + - key: "nvidia.com/gpu" + effect: "NoSchedule" + operator: "Exists" hostNetwork: true hostPID: true volumes: From ad05928c6e5b2bab5d7222da45f0ab0d54f13416 Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Wed, 29 Nov 2017 12:36:55 -0800 Subject: [PATCH 5/6] Add wildcard tolerations to kube-proxy. fluend-gcp already has these tolerations. kube-proxy when it runs as a static pod gets wildcard `NoExecute` toleration (all static pods get that). So, added the same toleration to kube-proxy when it runs as a daemonset. Also added wildcard `NoSchedule` toleration to kube-proxy. --- cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml | 1 - cluster/addons/kube-proxy/kube-proxy-ds.yaml | 5 +++++ cluster/saltbase/salt/kube-proxy/kube-proxy.manifest | 5 +++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml b/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml index 69021f4263c..7f6a47deb4b 100644 --- a/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml +++ b/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml @@ -107,7 +107,6 @@ spec: effect: "NoSchedule" - operator: "Exists" effect: "NoExecute" - #TODO: remove this toleration once #44445 is properly fixed. - operator: "Exists" effect: "NoSchedule" terminationGracePeriodSeconds: 30 diff --git a/cluster/addons/kube-proxy/kube-proxy-ds.yaml b/cluster/addons/kube-proxy/kube-proxy-ds.yaml index 479c6eeb023..2134e875fba 100644 --- a/cluster/addons/kube-proxy/kube-proxy-ds.yaml +++ b/cluster/addons/kube-proxy/kube-proxy-ds.yaml @@ -28,6 +28,11 @@ spec: hostNetwork: true nodeSelector: beta.kubernetes.io/kube-proxy-ds-ready: "true" + tolerations: + - operator: "Exists" + effect: "NoExecute" + - operator: "Exists" + effect: "NoSchedule" containers: - name: kube-proxy image: {{pillar['kube_docker_registry']}}/kube-proxy:{{pillar['kube-proxy_docker_tag']}} diff --git a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest index 4c9882a6ffa..69075cb9d04 100644 --- a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest +++ b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest @@ -65,6 +65,11 @@ metadata: spec: {{pod_priority}} hostNetwork: true + tolerations: + - operator: "Exists" + effect: "NoExecute" + - operator: "Exists" + effect: "NoSchedule" containers: - name: kube-proxy image: {{pillar['kube_docker_registry']}}/kube-proxy:{{pillar['kube-proxy_docker_tag']}} From 38a1ba5ca463435b29db2f3a06eb774973a3ae3f Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Thu, 30 Nov 2017 12:20:22 -0800 Subject: [PATCH 6/6] Update CHANGELOG-1.9.md for v1.9.0-beta.1. --- CHANGELOG-1.9.md | 191 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 182 insertions(+), 9 deletions(-) diff --git a/CHANGELOG-1.9.md b/CHANGELOG-1.9.md index c03c7673cbc..e6b1479ec92 100644 --- a/CHANGELOG-1.9.md +++ b/CHANGELOG-1.9.md @@ -1,33 +1,206 @@ -- [v1.9.0-alpha.3](#v190-alpha3) - - [Downloads for v1.9.0-alpha.3](#downloads-for-v190-alpha3) +- [v1.9.0-beta.1](#v190-beta1) + - [Downloads for v1.9.0-beta.1](#downloads-for-v190-beta1) - [Client Binaries](#client-binaries) - [Server Binaries](#server-binaries) - [Node Binaries](#node-binaries) - - [Changelog since v1.9.0-alpha.2](#changelog-since-v190-alpha2) + - [Changelog since v1.9.0-alpha.3](#changelog-since-v190-alpha3) - [Action Required](#action-required) - [Other notable changes](#other-notable-changes) -- [v1.9.0-alpha.2](#v190-alpha2) - - [Downloads for v1.9.0-alpha.2](#downloads-for-v190-alpha2) +- [v1.9.0-alpha.3](#v190-alpha3) + - [Downloads for v1.9.0-alpha.3](#downloads-for-v190-alpha3) - [Client Binaries](#client-binaries-1) - [Server Binaries](#server-binaries-1) - [Node Binaries](#node-binaries-1) - - [Changelog since v1.8.0](#changelog-since-v180) + - [Changelog since v1.9.0-alpha.2](#changelog-since-v190-alpha2) - [Action Required](#action-required-1) - [Other notable changes](#other-notable-changes-1) -- [v1.9.0-alpha.1](#v190-alpha1) - - [Downloads for v1.9.0-alpha.1](#downloads-for-v190-alpha1) +- [v1.9.0-alpha.2](#v190-alpha2) + - [Downloads for v1.9.0-alpha.2](#downloads-for-v190-alpha2) - [Client Binaries](#client-binaries-2) - [Server Binaries](#server-binaries-2) - [Node Binaries](#node-binaries-2) - - [Changelog since v1.8.0-alpha.3](#changelog-since-v180-alpha3) + - [Changelog since v1.8.0](#changelog-since-v180) - [Action Required](#action-required-2) - [Other notable changes](#other-notable-changes-2) +- [v1.9.0-alpha.1](#v190-alpha1) + - [Downloads for v1.9.0-alpha.1](#downloads-for-v190-alpha1) + - [Client Binaries](#client-binaries-3) + - [Server Binaries](#server-binaries-3) + - [Node Binaries](#node-binaries-3) + - [Changelog since v1.8.0-alpha.3](#changelog-since-v180-alpha3) + - [Action Required](#action-required-3) + - [Other notable changes](#other-notable-changes-3) +# v1.9.0-beta.1 + +[Documentation](https://docs.k8s.io) & [Examples](https://releases.k8s.io/release-1.9/examples) + +## Downloads for v1.9.0-beta.1 + + +filename | sha256 hash +-------- | ----------- +[kubernetes.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes.tar.gz) | `ffdcf0f7cd972340bc666395d759fc18573a32775d38ed3f4fd99d4369e856e4` +[kubernetes-src.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-src.tar.gz) | `09bee9a955987d53c7a65d2f1a3129854ca3a34f9fb38218f0c58f5bd603494a` + +### Client Binaries + +filename | sha256 hash +-------- | ----------- +[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-client-darwin-386.tar.gz) | `9d54db976ca7a12e9208e5595b552b094e0cc532b49ba6e919d776e52e56f4a8` +[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-client-darwin-amd64.tar.gz) | `0a22af2c6c84ff8b3022c0ecebf4ba3021048fceddf7375c87c13a83488ffe2c` +[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-client-linux-386.tar.gz) | `84bb638c8e61d7a7b415d49d76d166f3924052338c454d1ae57ae36eb37445c6` +[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-client-linux-amd64.tar.gz) | `08b56240288d17f147485e79c5f6594391c5b46e26450d64e7510f65db1f9a79` +[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-client-linux-arm64.tar.gz) | `7206573b131a8915d3bc14aa660fb44890ed79fdbd498bc8f9951c221aa12ea5` +[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-client-linux-arm.tar.gz) | `7ad21796b0e0a9d247beb41d6b3a3d0aaa822b85adae4c90533ba0ef94c05b2e` +[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-client-linux-ppc64le.tar.gz) | `2076328ca0958a96c8f551b91a393aa2d6fc24bef92991a1a4d9fc8df52519a7` +[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-client-linux-s390x.tar.gz) | `17ac0aba9a4e2003cb3d06bd631032b760d1a2d521c60a25dc26687aadb5ba14` +[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-client-windows-386.tar.gz) | `3a2bebd4adb6e1bf2b30a8cedb7ec212fc43c4b02e26a0a60c3429e478a86073` +[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-client-windows-amd64.tar.gz) | `fcc852e97f0e64d1025344aefd042ceff05227bfded80142bfa99927de1a5f0e` + +### Server Binaries + +filename | sha256 hash +-------- | ----------- +[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-server-linux-amd64.tar.gz) | `7ed2a789b86f258f1739cb165276150512a171a715da9372aeff000e946548fd` +[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-server-linux-arm64.tar.gz) | `e4e04a33698ac665a3e61fd8d60d4010fec6b0e3b0627dee9a965c2c2a510e3a` +[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-server-linux-arm.tar.gz) | `befce41457fc15c8fadf37ee5bf80b83405279c60665cfb9ecfc9f61fcd549c7` +[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-server-linux-ppc64le.tar.gz) | `e59e4fb84d6b890e9c6cb216ebb20546212e6c14feb077d9d0761c88e2685f4c` +[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-server-linux-s390x.tar.gz) | `0aa47d01907ea78b9a1a8001536d5091fca93409b81bac6eb3e90a4dff6c3faa` + +### Node Binaries + +filename | sha256 hash +-------- | ----------- +[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-node-linux-amd64.tar.gz) | `107bfaf72b8b6d3b5c163e61ed169c89288958750636c16bc3d781cf94bf5f4c` +[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-node-linux-arm64.tar.gz) | `6bc58e913a2467548664ece743617a1e595f6223100a1bad27e9a90bdf2e2927` +[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-node-linux-arm.tar.gz) | `d4ff8f37d7c95f7ca3aca30fa3c191f2cc5e48f0159ac6a5395ec09092574baa` +[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-node-linux-ppc64le.tar.gz) | `a88d65343ccb515c4eaab11352e69afee4a19c7fa345b08aaffa854b225cf305` +[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-node-linux-s390x.tar.gz) | `16d6a67d18273460cab4c293a5b130d4827f41ee4bf5b79b07c60ef517f580cd` +[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.9.0-beta.1/kubernetes-node-windows-amd64.tar.gz) | `f086659462b6dcdd78abdf13bed339dd67c1111931bae962044aa4ae2396921d` + +## Changelog since v1.9.0-alpha.3 + +### Action Required + +* Adds alpha support for volume scheduling, which allows the scheduler to make PersistentVolume binding decisions while respecting the Pod's scheduling requirements. Dynamic provisioning is not supported with this feature yet. ([#55039](https://github.com/kubernetes/kubernetes/pull/55039), [@msau42](https://github.com/msau42)) + * Action required for existing users of the LocalPersistentVolumes alpha feature: + * The VolumeScheduling feature gate also has to be enabled on kube-scheduler and kube-controller-manager. + * The NoVolumeNodeConflict predicate has been removed. For non-default schedulers, update your scheduler policy. + * The CheckVolumeBinding predicate has to be enabled in non-default schedulers. +* Action required: ([#56004](https://github.com/kubernetes/kubernetes/pull/56004), [@caesarxuchao](https://github.com/caesarxuchao)) + * The `admission/v1alpha1` API has graduated to `v1beta1`. Please delete your existing webhooks before upgrading the cluster, and update your admission webhooks to use the latest API, because the API has backwards incompatible changes. + * The webhook registration related part of the `admissionregistration` API has graduated to `v1beta1`. Please delete your existing configurations before upgrading the cluster, and update your configuration file to use the latest API. +* [action required] kubeadm join: Error out if CA pinning isn't used or opted out of ([#55468](https://github.com/kubernetes/kubernetes/pull/55468), [@yuexiao-wang](https://github.com/yuexiao-wang)) + * kubeadm now requires the user to specify either the `--discovery-token-ca-cert-hash` flag or the `--discovery-token-unsafe-skip-ca-verification` flag. + +### Other notable changes + +* A new priority function `ResourceLimitsPriorityMap` (disabled by default and behind alpha feature gate and not part of the scheduler's default priority functions list) that assigns a lowest possible score of 1 to a node that satisfies one or both of input pod's cpu and memory limits, mainly to break ties between nodes with same scores. ([#55906](https://github.com/kubernetes/kubernetes/pull/55906), [@aveshagarwal](https://github.com/aveshagarwal)) +* AWS: Fix detaching volume from stopped nodes. ([#55893](https://github.com/kubernetes/kubernetes/pull/55893), [@gnufied](https://github.com/gnufied)) +* Fix stats summary network value when multiple network interfaces are available. ([#52144](https://github.com/kubernetes/kubernetes/pull/52144), [@andyxning](https://github.com/andyxning)) +* Fix a typo in prometheus-to-sd configuration, that drops some stackdriver metrics. ([#56473](https://github.com/kubernetes/kubernetes/pull/56473), [@loburm](https://github.com/loburm)) +* Fixes server name verification of aggregated API servers and webhook admission endpoints ([#56415](https://github.com/kubernetes/kubernetes/pull/56415), [@liggitt](https://github.com/liggitt)) +* OpenStack cloud provider supports Cinder v3 API. ([#52910](https://github.com/kubernetes/kubernetes/pull/52910), [@FengyunPan](https://github.com/FengyunPan)) +* kube-up: Add optional addon CoreDNS. ([#55728](https://github.com/kubernetes/kubernetes/pull/55728), [@rajansandeep](https://github.com/rajansandeep)) + * Install CoreDNS instead of kube-dns by setting CLUSTER_DNS_CORE_DNS value to 'true'. +* kubeadm health checks can also be skipped with `--ignore-checks-errors` ([#56130](https://github.com/kubernetes/kubernetes/pull/56130), [@anguslees](https://github.com/anguslees)) +* Adds kubeadm support for using ComponentConfig for the kube-proxy ([#55972](https://github.com/kubernetes/kubernetes/pull/55972), [@rpothier](https://github.com/rpothier)) +* Pod Security Policy can now manage access to specific FlexVolume drivers ([#53179](https://github.com/kubernetes/kubernetes/pull/53179), [@wanghaoran1988](https://github.com/wanghaoran1988)) +* PVC Finalizing Controller is introduced in order to prevent deletion of a PVC that is being used by a pod. ([#55824](https://github.com/kubernetes/kubernetes/pull/55824), [@pospispa](https://github.com/pospispa)) +* Kubelet can provide full summary api support except container log stats for CRI container runtime now. ([#55810](https://github.com/kubernetes/kubernetes/pull/55810), [@abhi](https://github.com/abhi)) +* Add support for resizing EBS disks ([#56118](https://github.com/kubernetes/kubernetes/pull/56118), [@gnufied](https://github.com/gnufied)) +* Add PodDisruptionBudget support during pod preemption ([#56178](https://github.com/kubernetes/kubernetes/pull/56178), [@bsalamat](https://github.com/bsalamat)) +* Fix CRI localhost seccomp path in format localhost//profileRoot/profileName. ([#55450](https://github.com/kubernetes/kubernetes/pull/55450), [@feiskyer](https://github.com/feiskyer)) +* kubeadm: Add CoreDNS support for kubeadm "upgrade" and "alpha phases addons". ([#55952](https://github.com/kubernetes/kubernetes/pull/55952), [@rajansandeep](https://github.com/rajansandeep)) +* The default garbage collection policy for Deployment, DaemonSet, StatefulSet, and ReplicaSet has changed from OrphanDependents to DeleteDependents when the deletion is requested through an `apps/v1` endpoint. Clients using older endpoints will be unaffected. This change is only at the REST API level and is independent of the default behavior of particular clients (e.g. this does not affect the default for the kubectl `--cascade` flag). ([#55148](https://github.com/kubernetes/kubernetes/pull/55148), [@dixudx](https://github.com/dixudx)) + * If you upgrade your client-go libs and use the `AppsV1()` interface, please note that the default garbage collection behavior is changed. +* Add resize support for ceph RBD ([#52767](https://github.com/kubernetes/kubernetes/pull/52767), [@NickrenREN](https://github.com/NickrenREN)) +* Expose single annotation/label via downward API ([#55902](https://github.com/kubernetes/kubernetes/pull/55902), [@yguo0905](https://github.com/yguo0905)) +* kubeadm: added `--print-join-command` flag for `kubeadm token create`. ([#56185](https://github.com/kubernetes/kubernetes/pull/56185), [@mattmoyer](https://github.com/mattmoyer)) +* Implement kubelet side file system resizing. Also implement GCE PD resizing ([#55815](https://github.com/kubernetes/kubernetes/pull/55815), [@gnufied](https://github.com/gnufied)) +* Improved PodSecurityPolicy admission latency, but validation errors are no longer limited to only errors from authorized policies. ([#55643](https://github.com/kubernetes/kubernetes/pull/55643), [@tallclair](https://github.com/tallclair)) +* Add containerd monitoring support ([#56109](https://github.com/kubernetes/kubernetes/pull/56109), [@dashpole](https://github.com/dashpole)) +* Add pod-level CPU and memory stats from pod cgroup information ([#55969](https://github.com/kubernetes/kubernetes/pull/55969), [@jingxu97](https://github.com/jingxu97)) +* kubectl apply use openapi to calculate diff be default. It will fall back to use baked-in types when openapi is not available. ([#51321](https://github.com/kubernetes/kubernetes/pull/51321), [@mengqiy](https://github.com/mengqiy)) +* It is now possible to override the healthcheck parameters for AWS ELBs via annotations on the corresponding service. The new annotations are `healthy-threshold`, `unhealthy-threshold`, `timeout`, `interval` (all prefixed with `service.beta.kubernetes.io/aws-load-balancer-healthcheck-`) ([#56024](https://github.com/kubernetes/kubernetes/pull/56024), [@dimpavloff](https://github.com/dimpavloff)) +* Adding etcd version display to kubeadm upgrade plan subcommand ([#56156](https://github.com/kubernetes/kubernetes/pull/56156), [@sbezverk](https://github.com/sbezverk)) +* [fluentd-gcp addon] Fixes fluentd deployment on GCP when custom resources are set. ([#55950](https://github.com/kubernetes/kubernetes/pull/55950), [@crassirostris](https://github.com/crassirostris)) +* [fluentd-elasticsearch addon] Elasticsearch and Kibana are updated to version 5.6.4 ([#55400](https://github.com/kubernetes/kubernetes/pull/55400), [@mrahbar](https://github.com/mrahbar)) +* install ipset in debian-iptables docker image ([#56115](https://github.com/kubernetes/kubernetes/pull/56115), [@m1093782566](https://github.com/m1093782566)) +* Add cleanup-ipvs flag for kube-proxy ([#56036](https://github.com/kubernetes/kubernetes/pull/56036), [@m1093782566](https://github.com/m1093782566)) +* Remove opaque integer resources (OIR) support (deprecated in v1.8.) ([#55103](https://github.com/kubernetes/kubernetes/pull/55103), [@ConnorDoyle](https://github.com/ConnorDoyle)) +* Implement volume resize for cinder ([#51498](https://github.com/kubernetes/kubernetes/pull/51498), [@NickrenREN](https://github.com/NickrenREN)) +* Block volumes Support: FC plugin update ([#51493](https://github.com/kubernetes/kubernetes/pull/51493), [@mtanino](https://github.com/mtanino)) +* kube-apiserver: fixed --oidc-username-prefix and --oidc-group-prefix flags which previously weren't correctly enabled ([#56175](https://github.com/kubernetes/kubernetes/pull/56175), [@ericchiang](https://github.com/ericchiang)) +* New kubeadm flag `--ignore-preflight-errors` that enables to decrease severity of each individual error to warning. ([#56072](https://github.com/kubernetes/kubernetes/pull/56072), [@kad](https://github.com/kad)) + * Old flag `--skip-preflight-checks` is marked as deprecated and acts as `--ignore-preflight-errors=all` +* Block volumes Support: CRI, volumemanager and operationexecutor changes ([#51494](https://github.com/kubernetes/kubernetes/pull/51494), [@mtanino](https://github.com/mtanino)) +* StatefulSet controller will create a label for each Pod in a StatefulSet. The label is named statefulset.kubernetes.io/pod-name and it is equal to the name of the Pod. This allows users to create a Service per Pod to expose a connection to individual Pods. ([#55329](https://github.com/kubernetes/kubernetes/pull/55329), [@kow3ns](https://github.com/kow3ns)) +* Initial basic bootstrap-checkpoint support ([#50984](https://github.com/kubernetes/kubernetes/pull/50984), [@timothysc](https://github.com/timothysc)) +* Add DNSConfig field to PodSpec and support "None" mode for DNSPolicy (Alpha). ([#55848](https://github.com/kubernetes/kubernetes/pull/55848), [@MrHohn](https://github.com/MrHohn)) +* Add pod-level local ephemeral storage metric in Summary API. Pod-level ephemeral storage reports the total filesystem usage for the containers and emptyDir volumes in the measured Pod. ([#55447](https://github.com/kubernetes/kubernetes/pull/55447), [@jingxu97](https://github.com/jingxu97)) +* Kubernetes update Azure nsg rules based on not just difference in Name, but also in Protocol, SourcePortRange, DestinationPortRange, SourceAddressPrefix, DestinationAddressPrefix, Access, and Direction. ([#55752](https://github.com/kubernetes/kubernetes/pull/55752), [@kevinkim9264](https://github.com/kevinkim9264)) +* Add support to take nominated pods into account during scheduling to avoid starvation of higher priority pods. ([#55933](https://github.com/kubernetes/kubernetes/pull/55933), [@bsalamat](https://github.com/bsalamat)) +* Add Amazon NLB support - Fixes [#52173](https://github.com/kubernetes/kubernetes/pull/52173) ([#53400](https://github.com/kubernetes/kubernetes/pull/53400), [@micahhausler](https://github.com/micahhausler)) +* Extends deviceplugin to gracefully handle full device plugin lifecycle. ([#55088](https://github.com/kubernetes/kubernetes/pull/55088), [@jiayingz](https://github.com/jiayingz)) +* A new field is added to CRI container log format to support splitting a long log line into multiple lines. ([#55922](https://github.com/kubernetes/kubernetes/pull/55922), [@Random-Liu](https://github.com/Random-Liu)) +* [advanced audit]add a policy wide omitStage ([#54634](https://github.com/kubernetes/kubernetes/pull/54634), [@CaoShuFeng](https://github.com/CaoShuFeng)) +* Fix a bug in GCE multizonal clusters where PersistentVolumes were sometimes created in zones without nodes. ([#52322](https://github.com/kubernetes/kubernetes/pull/52322), [@davidz627](https://github.com/davidz627)) +* With this change ([#55845](https://github.com/kubernetes/kubernetes/pull/55845), [@rohitjogvmw](https://github.com/rohitjogvmw)) + * - User should be able to create k8s cluster which spans across multiple ESXi clusters, datacenters or even vCenters. + * - vSphere cloud provider (VCP) uses OS hostname and not vSphere Inventory VM Name. + * That means, now VCP can handle cases where user changes VM inventory name. + * - VCP can handle cases where VM migrates to other ESXi cluster or datacenter or vCenter. + * The only requirement is the shared storage. VCP needs shared storage on all Node VMs. +* The RBAC bootstrapping policy now allows authenticated users to create selfsubjectrulesreviews. ([#56095](https://github.com/kubernetes/kubernetes/pull/56095), [@ericchiang](https://github.com/ericchiang)) +* Defaulting of controller-manager options for --cluster-signing-cert-file and --cluster-signing-key-file is deprecated and will be removed in a later release. ([#54495](https://github.com/kubernetes/kubernetes/pull/54495), [@mikedanese](https://github.com/mikedanese)) +* Add ExtendedResourceToleration admission controller. This facilitates creation of dedicated nodes with extended resources. If operators want to create dedicated nodes with extended resources (like GPUs, FPGAs etc.), they are expected to taint the node with extended resource name as the key. This admission controller, if enabled, automatically adds tolerations for such taints to pods requesting extended resources, so users don't have to manually add these tolerations. ([#55839](https://github.com/kubernetes/kubernetes/pull/55839), [@mindprince](https://github.com/mindprince)) +* Move unreachable taint key out of alpha. ([#54208](https://github.com/kubernetes/kubernetes/pull/54208), [@resouer](https://github.com/resouer)) + * Please note the existing pods with the alpha toleration should be updated by user himself to tolerate the GA taint. +* add GRS, RAGRS storage account type support for azure disk ([#55931](https://github.com/kubernetes/kubernetes/pull/55931), [@andyzhangx](https://github.com/andyzhangx)) +* Upgrading the kubernetes-master units now results in staged upgrades just like the kubernetes-worker nodes. Use the upgrade action in order to continue the upgrade process on each unit such as `juju run-action kubernetes-master/0 upgrade` ([#55990](https://github.com/kubernetes/kubernetes/pull/55990), [@hyperbolic2346](https://github.com/hyperbolic2346)) +* Using ipset doing SNAT and packet filtering in IPVS kube-proxy ([#54219](https://github.com/kubernetes/kubernetes/pull/54219), [@m1093782566](https://github.com/m1093782566)) +* Add a new scheduling queue that helps schedule the highest priority pending pod first. ([#55109](https://github.com/kubernetes/kubernetes/pull/55109), [@bsalamat](https://github.com/bsalamat)) +* Adds to **kubeadm upgrade apply**, a new **--etcd-upgrade** keyword. When this keyword is specified, etcd's static pod gets upgraded to the etcd version officially recommended for a target kubernetes release. ([#55010](https://github.com/kubernetes/kubernetes/pull/55010), [@sbezverk](https://github.com/sbezverk)) +* Adding vishh as an reviewer/approver for hack directory ([#54007](https://github.com/kubernetes/kubernetes/pull/54007), [@vishh](https://github.com/vishh)) +* The `GenericAdmissionWebhook` is renamed as `ValidatingAdmissionWebhook`. Please update you apiserver configuration file to use the new name to pass to the apiserver's `--admission-control` flag. ([#55988](https://github.com/kubernetes/kubernetes/pull/55988), [@caesarxuchao](https://github.com/caesarxuchao)) +* iSCSI Persistent Volume Sources can now reference CHAP Secrets in namespaces other than the namespace of the bound Persistent Volume Claim ([#51530](https://github.com/kubernetes/kubernetes/pull/51530), [@rootfs](https://github.com/rootfs)) +* Bugfix: master startup script on GCP no longer fails randomly due to concurrent iptables invocations. ([#55945](https://github.com/kubernetes/kubernetes/pull/55945), [@x13n](https://github.com/x13n)) +* fix azure disk storage account init issue ([#55927](https://github.com/kubernetes/kubernetes/pull/55927), [@andyzhangx](https://github.com/andyzhangx)) +* Allow code-generator tags in the 2nd closest comment block and directly above a statement. ([#55233](https://github.com/kubernetes/kubernetes/pull/55233), [@sttts](https://github.com/sttts)) +* Ensure additional resource tags are set/updated AWS load balancers ([#55731](https://github.com/kubernetes/kubernetes/pull/55731), [@georgebuckerfield](https://github.com/georgebuckerfield)) +* `kubectl get` will now use OpenAPI schema extensions by default to select columns for custom types. ([#53483](https://github.com/kubernetes/kubernetes/pull/53483), [@apelisse](https://github.com/apelisse)) +* AWS: Apply taint to a node if volumes being attached to it are stuck in attaching state ([#55558](https://github.com/kubernetes/kubernetes/pull/55558), [@gnufied](https://github.com/gnufied)) +* Kubeadm now supports for Kubelet Dynamic Configuration. ([#55803](https://github.com/kubernetes/kubernetes/pull/55803), [@xiangpengzhao](https://github.com/xiangpengzhao)) +* Added mutation supports to admission webhooks. ([#54892](https://github.com/kubernetes/kubernetes/pull/54892), [@caesarxuchao](https://github.com/caesarxuchao)) +* Upgrade to go1.9.2 ([#55420](https://github.com/kubernetes/kubernetes/pull/55420), [@cblecker](https://github.com/cblecker)) +* If a non-absolute mountPath is passed to the kubelet, prefix it with the appropriate root path. ([#55665](https://github.com/kubernetes/kubernetes/pull/55665), [@brendandburns](https://github.com/brendandburns)) +* action-required: please update your admission webhook to use the latest [Admission API](https://github.com/kubernetes/api/tree/master/admission). ([#55829](https://github.com/kubernetes/kubernetes/pull/55829), [@cheftako](https://github.com/cheftako)) + * `admission/v1alpha1#AdmissionReview` now contains `AdmissionRequest` and `AdmissionResponse`. `AdmissionResponse` includes a `Patch` field to allow mutating webhooks to send json patch to the apiserver. +* support mount options in azure file ([#54674](https://github.com/kubernetes/kubernetes/pull/54674), [@andyzhangx](https://github.com/andyzhangx)) +* Support AWS ECR credentials in China ([#50108](https://github.com/kubernetes/kubernetes/pull/50108), [@zzq889](https://github.com/zzq889)) +* The EvictionHard, EvictionSoft, EvictionSoftGracePeriod, EvictionMinimumReclaim, SystemReserved, and KubeReserved fields in the KubeletConfiguration object (kubeletconfig/v1alpha1) are now of type map[string]string, which facilitates writing JSON and YAML files. ([#54823](https://github.com/kubernetes/kubernetes/pull/54823), [@mtaufen](https://github.com/mtaufen)) +* Added service annotation for AWS ELB SSL policy ([#54507](https://github.com/kubernetes/kubernetes/pull/54507), [@micahhausler](https://github.com/micahhausler)) +* Implement correction mechanism for dangling volumes attached for deleted pods ([#55491](https://github.com/kubernetes/kubernetes/pull/55491), [@gnufied](https://github.com/gnufied)) +* Promote validation for custom resources defined through CRD to beta ([#54647](https://github.com/kubernetes/kubernetes/pull/54647), [@colemickens](https://github.com/colemickens)) +* Octavia v2 now supported as a LB provider ([#55393](https://github.com/kubernetes/kubernetes/pull/55393), [@jamiehannaford](https://github.com/jamiehannaford)) +* Kubelet now exposes metrics for NVIDIA GPUs attached to the containers. ([#55188](https://github.com/kubernetes/kubernetes/pull/55188), [@mindprince](https://github.com/mindprince)) +* Addon manager supports HA masters. ([#55782](https://github.com/kubernetes/kubernetes/pull/55782), [@x13n](https://github.com/x13n)) +* Fix kubeadm reset crictl command ([#55717](https://github.com/kubernetes/kubernetes/pull/55717), [@runcom](https://github.com/runcom)) +* Fix code-generators to produce correct code when GroupName, PackageName and/or GoName differ. ([#55614](https://github.com/kubernetes/kubernetes/pull/55614), [@sttts](https://github.com/sttts)) +* Fixes bad conversion in host port chain name generating func which leads to some unreachable host ports. ([#55153](https://github.com/kubernetes/kubernetes/pull/55153), [@chenchun](https://github.com/chenchun)) +* Relative paths in the Kubelet's local config files (--init-config-dir) will be resolved relative to the location of the containing files. ([#55648](https://github.com/kubernetes/kubernetes/pull/55648), [@mtaufen](https://github.com/mtaufen)) +* kubeadm: Fix a bug on some OSes where the kubelet tried to mount a volume path that is non-existent and on a read-only filesystem ([#55320](https://github.com/kubernetes/kubernetes/pull/55320), [@andrewrynhard](https://github.com/andrewrynhard)) +* add hostIP and protocol to the original hostport predicates procedure in scheduler. ([#52421](https://github.com/kubernetes/kubernetes/pull/52421), [@WIZARD-CXY](https://github.com/WIZARD-CXY)) + + + # v1.9.0-alpha.3 [Documentation](https://docs.k8s.io) & [Examples](https://releases.k8s.io/master/examples)