github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 19:56:01 +00:00
Code Issues Projects Releases Wiki Activity

Add warnings about cache invalidation.

Browse Source 
Part of https://github.com/kubernetes/kubernetes/pull/63040 is the
assumption that scheduler cache updates must happen before equivalence
cache updates for any given informer event.

The reason for this is that the equivalence cache implementation checks
the main cache for staleness while holding the equiv. cache write lock.

case 1: If an informer invalidates an equiv. cache entry before the
staleness check, then we know that the main cache update completed.

case 2: If an informer blocks trying to grab the equiv. cache lock, then
invalidation will occur right after the potentially stale update is
written.

This patch adds a note to places where we invalidate the equivalence
cache so that hopefully nobody violates this invariant.
This commit is contained in:
Jonathan Basseri 2018-05-15 16:01:12 -07:00
parent f4e909d6c2
commit 18a8184dce
2 changed files with 18 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
pkg/scheduler/factory/factory.go
View File

@ -634,6 +634,9 @@ func (c *configFactory) updatePodInCache(oldObj, newObj interface{}) {
return
}
// NOTE: Because the scheduler uses snapshots of schedulerCache and the live
// version of equivalencePodCache, updates must be written to schedulerCache
// before invalidating equivalencePodCache.
if err := c.schedulerCache.UpdatePod(oldPod, newPod); err != nil {
glog.Errorf("scheduler cache UpdatePod failed: %v", err)
}
@ -720,6 +723,9 @@ func (c *configFactory) deletePodFromCache(obj interface{}) {
glog.Errorf("cannot convert to *v1.Pod: %v", t)
return
}
// NOTE: Because the scheduler uses snapshots of schedulerCache and the live
// version of equivalencePodCache, updates must be written to schedulerCache
// before invalidating equivalencePodCache.
if err := c.schedulerCache.RemovePod(pod); err != nil {
glog.Errorf("scheduler cache RemovePod failed: %v", err)
}
@ -776,6 +782,9 @@ func (c *configFactory) updateNodeInCache(oldObj, newObj interface{}) {
return
}
// NOTE: Because the scheduler uses snapshots of schedulerCache and the live
// version of equivalencePodCache, updates must be written to schedulerCache
// before invalidating equivalencePodCache.
if err := c.schedulerCache.UpdateNode(oldNode, newNode); err != nil {
glog.Errorf("scheduler cache UpdateNode failed: %v", err)
}
@ -869,6 +878,9 @@ func (c *configFactory) deleteNodeFromCache(obj interface{}) {
glog.Errorf("cannot convert to *v1.Node: %v", t)
return
}
// NOTE: Because the scheduler uses snapshots of schedulerCache and the live
// version of equivalencePodCache, updates must be written to schedulerCache
// before invalidating equivalencePodCache.
if err := c.schedulerCache.RemoveNode(node); err != nil {
glog.Errorf("scheduler cache RemoveNode failed: %v", err)
}
@ -1297,6 +1309,9 @@ func (c *configFactory) MakeDefaultErrorFunc(backoff *util.PodBackoff, podQueue
_, err := c.client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
if err != nil && errors.IsNotFound(err) {
node := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: nodeName}}
// NOTE: Because the scheduler uses snapshots of schedulerCache and the live
// version of equivalencePodCache, updates must be written to schedulerCache
// before invalidating equivalencePodCache.
c.schedulerCache.RemoveNode(&node)
// invalidate cached predicate for the node
if c.enableEquivalenceClassCache {

3
pkg/scheduler/scheduler.go
View File

@ -373,6 +373,9 @@ func (sched *Scheduler) assume(assumed *v1.Pod, host string) error {
// If the binding fails, scheduler will release resources allocated to assumed pod
// immediately.
assumed.Spec.NodeName = host
// NOTE: Because the scheduler uses snapshots of SchedulerCache and the live
// version of Ecache, updates must be written to SchedulerCache before
// invalidating Ecache.
if err := sched.config.SchedulerCache.AssumePod(assumed); err != nil {
glog.Errorf("scheduler cache AssumePod failed: %v", err)