From ffcf653e0666366e6241c99d9418e830840afa0f Mon Sep 17 00:00:00 2001 From: Tom Wieczorek Date: Mon, 13 Mar 2023 08:37:13 +0100 Subject: [PATCH] Properly align synctrack.SingleFileTracker struct count is used with atomic operations so it must be 64-bit aligned, otherwise atomic operations will panic. Having it at the top of the struct will guarantee that, even on 32-bit arches. This fixes panics like that one observed in kube-apiserver: E0310 13:48:47.476124 676 runtime.go:77] Observed a panic: unaligned 64-bit atomic operation goroutine 141 [running]: k8s.io/apimachinery/pkg/util/runtime.logPanic({0x2482378, 0x2db2ff8}) vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:75 +0x94 k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0x0}) vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:49 +0x78 panic({0x2482378, 0x2db2ff8}) /usr/local/go/src/runtime/panic.go:884 +0x218 runtime/internal/atomic.panicUnaligned() /usr/local/go/src/runtime/internal/atomic/unaligned.go:8 +0x24 runtime/internal/atomic.Load64(0x685f794) /usr/local/go/src/runtime/internal/atomic/atomic_arm.s:280 +0x14 k8s.io/client-go/tools/cache/synctrack.(*SingleFileTracker).HasSynced(0x685f790) vendor/k8s.io/client-go/tools/cache/synctrack/synctrack.go:115 +0x3c k8s.io/client-go/tools/cache.(*processorListener).HasSynced(0x6013e60) vendor/k8s.io/client-go/tools/cache/shared_informer.go:907 +0x20 k8s.io/client-go/tools/cache.WaitForCacheSync.func1() vendor/k8s.io/client-go/tools/cache/shared_informer.go:332 +0x50 k8s.io/apimachinery/pkg/util/wait.ConditionFunc.WithContext.func1({0x2dcf274, 0x607c600}) vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:222 +0x1c k8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x2dcf274, 0x607c600}, 0x6382050) vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:262 +0x64 k8s.io/apimachinery/pkg/util/wait.waitForWithContext({0x2dcf274, 0x607c600}, 0x64a6060, 0x6382050) vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:649 +0x11c k8s.io/apimachinery/pkg/util/wait.poll({0x2dcf274, 0x607c600}, 0x1, 0x64a6060, 0x6382050) vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:600 +0xc4 k8s.io/apimachinery/pkg/util/wait.PollImmediateUntilWithContext({0x2dcf274, 0x607c600}, 0x5f5e100, 0x6382050) vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:551 +0x60 k8s.io/apimachinery/pkg/util/wait.PollImmediateUntil(0x5f5e100, 0x6298020, 0x607c600) vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:542 +0x48 k8s.io/client-go/tools/cache.WaitForCacheSync(0x607c600, {0x6298000, 0x3, 0x3}) vendor/k8s.io/client-go/tools/cache/shared_informer.go:329 +0x80 k8s.io/client-go/tools/cache.WaitForNamedCacheSync({0x283c5e1, 0xf}, 0x607c600, {0x6298000, 0x3, 0x3}) vendor/k8s.io/client-go/tools/cache/shared_informer.go:316 +0xe8 created by k8s.io/kubernetes/plugin/pkg/auth/authorizer/node.AddGraphEventHandlers plugin/pkg/auth/authorizer/node/graph_populator.go:65 +0x5b0 panic: unaligned 64-bit atomic operation [recovered] panic: unaligned 64-bit atomic operation goroutine 141 [running]: k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0x0}) vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:56 +0xf4 panic({0x2482378, 0x2db2ff8}) /usr/local/go/src/runtime/panic.go:884 +0x218 runtime/internal/atomic.panicUnaligned() /usr/local/go/src/runtime/internal/atomic/unaligned.go:8 +0x24 runtime/internal/atomic.Load64(0x685f794) /usr/local/go/src/runtime/internal/atomic/atomic_arm.s:280 +0x14 k8s.io/client-go/tools/cache/synctrack.(*SingleFileTracker).HasSynced(0x685f790) vendor/k8s.io/client-go/tools/cache/synctrack/synctrack.go:115 +0x3c k8s.io/client-go/tools/cache.(*processorListener).HasSynced(0x6013e60) vendor/k8s.io/client-go/tools/cache/shared_informer.go:907 +0x20 k8s.io/client-go/tools/cache.WaitForCacheSync.func1() vendor/k8s.io/client-go/tools/cache/shared_informer.go:332 +0x50 k8s.io/apimachinery/pkg/util/wait.ConditionFunc.WithContext.func1({0x2dcf274, 0x607c600}) vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:222 +0x1c k8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x2dcf274, 0x607c600}, 0x6382050) vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:262 +0x64 k8s.io/apimachinery/pkg/util/wait.waitForWithContext({0x2dcf274, 0x607c600}, 0x64a6060, 0x6382050) vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:649 +0x11c k8s.io/apimachinery/pkg/util/wait.poll({0x2dcf274, 0x607c600}, 0x1, 0x64a6060, 0x6382050) vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:600 +0xc4 k8s.io/apimachinery/pkg/util/wait.PollImmediateUntilWithContext({0x2dcf274, 0x607c600}, 0x5f5e100, 0x6382050) vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:551 +0x60 k8s.io/apimachinery/pkg/util/wait.PollImmediateUntil(0x5f5e100, 0x6298020, 0x607c600) vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:542 +0x48 k8s.io/client-go/tools/cache.WaitForCacheSync(0x607c600, {0x6298000, 0x3, 0x3}) vendor/k8s.io/client-go/tools/cache/shared_informer.go:329 +0x80 k8s.io/client-go/tools/cache.WaitForNamedCacheSync({0x283c5e1, 0xf}, 0x607c600, {0x6298000, 0x3, 0x3}) vendor/k8s.io/client-go/tools/cache/shared_informer.go:316 +0xe8 created by k8s.io/kubernetes/plugin/pkg/auth/authorizer/node.AddGraphEventHandlers plugin/pkg/auth/authorizer/node/graph_populator.go:65 +0x5b0 --- .../k8s.io/client-go/tools/cache/synctrack/synctrack.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/staging/src/k8s.io/client-go/tools/cache/synctrack/synctrack.go b/staging/src/k8s.io/client-go/tools/cache/synctrack/synctrack.go index c488b497ff1..3fa2beb6b71 100644 --- a/staging/src/k8s.io/client-go/tools/cache/synctrack/synctrack.go +++ b/staging/src/k8s.io/client-go/tools/cache/synctrack/synctrack.go @@ -78,9 +78,13 @@ func (t *AsyncTracker[T]) HasSynced() bool { // SingleFileTracker helps propagate HasSynced when events are processed in // order (i.e. via a queue). type SingleFileTracker struct { - UpstreamHasSynced func() bool - + // Important: count is used with atomic operations so it must be 64-bit + // aligned, otherwise atomic operations will panic. Having it at the top of + // the struct will guarantee that, even on 32-bit arches. + // See https://pkg.go.dev/sync/atomic#pkg-note-BUG for more information. count int64 + + UpstreamHasSynced func() bool } // Start should be called prior to processing each key which is part of the