github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-27 13:37:30 +00:00
Code Issues Projects Releases Wiki Activity

Wait for APIServer 'ok' forever during CSINode initialization during Kubelet init

Browse Source
This commit is contained in:
David Zhu 2020-03-27 18:27:57 +01:00 committed by Jan Safranek
parent 3842a92f5f
commit 8205f815fb
1 changed files with 37 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
pkg/volume/csi/csi_plugin.go
View File

@ -227,10 +227,10 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error {
if utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) && if utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) &&
utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) { utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) {
// This function prevents Kubelet from posting Ready status until CSINodeInfo // This function prevents Kubelet from posting Ready status until CSINode
// is both installed and initialized // is both installed and initialized
if err := initializeCSINode(host); err != nil { if err := initializeCSINode(host); err != nil {
return errors.New(log("failed to initialize CSINodeInfo: %v", err)) return errors.New(log("failed to initialize CSINode: %v", err))
} }
} }
@ -240,21 +240,27 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error {
func initializeCSINode(host volume.VolumeHost) error { func initializeCSINode(host volume.VolumeHost) error {
kvh, ok := host.(volume.KubeletVolumeHost) kvh, ok := host.(volume.KubeletVolumeHost)
if !ok { if !ok {
klog.V(4).Info("Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINodeInfo initialization, not running on kubelet") klog.V(4).Info("Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet")
return nil return nil
} }
kubeClient := host.GetKubeClient() kubeClient := host.GetKubeClient()
if kubeClient == nil { if kubeClient == nil {
// Kubelet running in standalone mode. Skip CSINodeInfo initialization // Kubelet running in standalone mode. Skip CSINode initialization
klog.Warning("Skipping CSINodeInfo initialization, kubelet running in standalone mode") klog.Warning("Skipping CSINode initialization, kubelet running in standalone mode")
return nil return nil
} }
kvh.SetKubeletError(errors.New("CSINodeInfo is not yet initialized")) kvh.SetKubeletError(errors.New("CSINode is not yet initialized"))
go func() { go func() {
defer utilruntime.HandleCrash() defer utilruntime.HandleCrash()
// First wait indefinitely to talk to Kube APIServer
err := waitForAPIServerForever(kubeClient)
if err != nil {
klog.Fatalf("Failed to initialize CSINode while waiting for API server to report ok: %v", err)
}
// Backoff parameters tuned to retry over 140 seconds. Will fail and restart the Kubelet // Backoff parameters tuned to retry over 140 seconds. Will fail and restart the Kubelet
// after max retry steps. // after max retry steps.
initBackoff := wait.Backoff{ initBackoff := wait.Backoff{
@ -263,12 +269,12 @@ func initializeCSINode(host volume.VolumeHost) error {
Factor: 6.0, Factor: 6.0,
Jitter: 0.1, Jitter: 0.1,
} }
err := wait.ExponentialBackoff(initBackoff, func() (bool, error) { err = wait.ExponentialBackoff(initBackoff, func() (bool, error) {
klog.V(4).Infof("Initializing migrated drivers on CSINodeInfo") klog.V(4).Infof("Initializing migrated drivers on CSINode")
err := nim.InitializeCSINodeWithAnnotation() err := nim.InitializeCSINodeWithAnnotation()
if err != nil { if err != nil {
kvh.SetKubeletError(fmt.Errorf("Failed to initialize CSINodeInfo: %v", err)) kvh.SetKubeletError(fmt.Errorf("Failed to initialize CSINode: %v", err))
klog.Errorf("Failed to initialize CSINodeInfo: %v", err) klog.Errorf("Failed to initialize CSINode: %v", err)
return false, nil return false, nil
} }
@ -282,7 +288,7 @@ func initializeCSINode(host volume.VolumeHost) error {
// using CSI for all Migrated volume plugins. Then all the CSINode initialization // using CSI for all Migrated volume plugins. Then all the CSINode initialization
// code can be dropped from Kubelet. // code can be dropped from Kubelet.
// Kill the Kubelet process and allow it to restart to retry initialization // Kill the Kubelet process and allow it to restart to retry initialization
klog.Fatalf("Failed to initialize CSINodeInfo after retrying") klog.Fatalf("Failed to initialize CSINode after retrying: %v", err)
} }
}() }()
return nil return nil
@ -914,3 +920,23 @@ func highestSupportedVersion(versions []string) (*utilversion.Version, error) {
} }
return highestSupportedVersion, nil return highestSupportedVersion, nil
} }
// waitForAPIServerForever waits forever to get the APIServer Version as a proxy
// for a healthy APIServer.
func waitForAPIServerForever(client clientset.Interface) error {
var lastErr error
err := wait.PollInfinite(time.Second, func() (bool, error) {
_, lastErr = client.Discovery().ServerVersion()
if lastErr != nil {
lastErr = fmt.Errorf("failed to get apiserver version: %v", lastErr)
return false, nil
}
return true, nil
})
if err != nil {
return fmt.Errorf("%v: %v", err, lastErr)
}
return nil
}