kubelet: add userns manager

it is used to allocate and keep track of the unique users ranges
assigned to each pod that runs in a user namespace.

Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com>
Signed-off-by: Rodrigo Campos <rodrigoca@microsoft.com>
Co-authored-by: Rodrigo Campos <rodrigoca@microsoft.com>
This commit is contained in:
Giuseppe Scrivano 2022-07-07 11:21:33 +02:00 committed by Rodrigo Campos
parent cf8164bccf
commit 63462285d5
4 changed files with 760 additions and 0 deletions

View File

@ -837,6 +837,10 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
StateDirectory: rootDirectory,
})
klet.shutdownManager = shutdownManager
klet.usernsManager, err = MakeUserNsManager(klet)
if err != nil {
return nil, err
}
klet.admitHandlers.AddPodAdmitHandler(shutdownAdmitHandler)
// Finally, put the most recent version of the config on the Kubelet, so
@ -1175,6 +1179,9 @@ type Kubelet struct {
// Handles node shutdown events for the Node.
shutdownManager nodeshutdown.Manager
// Manage user namespaces
usernsManager *usernsManager
}
// ListPodStats is delegated to StatsProvider, which implements stats.Provider interface
@ -1889,6 +1896,8 @@ func (kl *Kubelet) syncTerminatedPod(ctx context.Context, pod *v1.Pod, podStatus
klog.V(4).InfoS("Pod termination removed cgroups", "pod", klog.KObj(pod), "podUID", pod.UID)
}
kl.usernsManager.Release(pod.UID)
// mark the final pod status
kl.statusManager.TerminatePod(pod)
klog.V(4).InfoS("Pod is terminated and will need no more status updates", "pod", klog.KObj(pod), "podUID", pod.UID)

View File

@ -1154,6 +1154,12 @@ func (kl *Kubelet) HandlePodCleanups() error {
return err
}
// Remove orphaned pod user namespace allocations (if any).
klog.V(3).InfoS("Clean up orphaned pod user namespace allocations")
if err = kl.usernsManager.CleanupOrphanedPodUsernsAllocations(allPods, runningRuntimePods); err != nil {
klog.ErrorS(err, "Failed cleaning up orphaned pod user namespaces allocations")
}
// Remove orphaned volumes from pods that are known not to have any
// containers. Note that we pass all pods (including terminated pods) to
// the function, so that we don't remove volumes associated with terminated

View File

@ -0,0 +1,530 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"encoding/json"
"fmt"
"math"
"os"
"path/filepath"
"sync"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
utilfeature "k8s.io/apiserver/pkg/util/feature"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/features"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
utilstore "k8s.io/kubernetes/pkg/kubelet/util/store"
utilfs "k8s.io/kubernetes/pkg/util/filesystem"
)
// bitsDataElement is the number of bits in a bitArray.data element.
const bitsDataElement = 32
type bitArray struct {
data []uint32
firstIndex int
}
func makeBitArray(size uint32) *bitArray {
m := bitArray{
data: make([]uint32, (size+bitsDataElement-1)/bitsDataElement),
firstIndex: 0,
}
return &m
}
func (b *bitArray) set(index uint32) {
b.data[index/bitsDataElement] |= (uint32(1) << (index % bitsDataElement))
}
func (b *bitArray) isSet(index uint32) bool {
return (b.data[index/bitsDataElement]>>(index%bitsDataElement))&0x1 == 1
}
func (b *bitArray) findAvailable() (uint32, bool) {
for i := b.firstIndex; i < len(b.data); i++ {
// Check if all bits are used (all 1s).
if b.data[i] == math.MaxUint32 {
continue
}
for j := uint32(0); j < bitsDataElement; j++ {
if (b.data[i]>>j)&0x1 == 0 {
v := uint32(i)*bitsDataElement + j
b.set(v)
// Update firstIndex to the current
// data element since there are no other
// unset bits before the current index.
b.firstIndex = int(i)
return v, true
}
}
}
return 0, false
}
func (b *bitArray) clear(index uint32) {
i := index / bitsDataElement
// update firstIndex if the index found is less than
// the current one.
if i < uint32(b.firstIndex) {
b.firstIndex = int(i)
}
// clear the bit by ANDing the data element with the
// complement of the bitmask to be cleared.
b.data[i] &= ^(1 << (index % bitsDataElement))
}
// length for the user namespace to create (65536).
const userNsLength = (1 << 16)
// Limit the total number of pods using userns in this node to this value.
// This is an alpha limitation that will probably be lifted later.
const maxPods = 1024
// Create a new map when we removed enough pods to avoid memory leaks
// since Go maps never free memory.
const mapReInitializeThreshold = 1000
type userNsPodsManager interface {
getPodDir(podUID types.UID) string
listPodsFromDisk() ([]types.UID, error)
}
type usernsManager struct {
used *bitArray
usedBy map[types.UID]uint32 // Map pod.UID to range used
removed int
numAllocated int
kl userNsPodsManager
// This protects all members except for kl.anager
lock sync.Mutex
}
// UserNamespace holds the configuration for the user namespace.
type userNamespace struct {
// UIDs mappings for the user namespace.
UIDMappings []idMapping `json:"uidMappings"`
// GIDs mappings for the user namespace.
GIDMappings []idMapping `json:"gidMappings"`
}
// Pod user namespace mapping
type idMapping struct {
// Required.
HostId uint32 `json:"hostId"`
// Required.
ContainerId uint32 `json:"containerId"`
// Required.
Length uint32 `json:"length"`
}
// mappingsFile is the file where the user namespace mappings are persisted.
const mappingsFile = "userns"
// writeMappingsToFile writes the specified user namespace configuration to the pod
// directory.
func (m *usernsManager) writeMappingsToFile(pod types.UID, userNs userNamespace) error {
dir := m.kl.getPodDir(pod)
data, err := json.Marshal(userNs)
if err != nil {
return err
}
fstore, err := utilstore.NewFileStore(dir, &utilfs.DefaultFs{})
if err != nil {
return err
}
if err := fstore.Write(mappingsFile, data); err != nil {
return err
}
// We need to fsync the parent dir so the file is guaranteed to be there.
// fstore guarantees an atomic write, we need durability too.
parentDir, err := os.Open(dir)
if err != nil {
return err
}
if err = parentDir.Sync(); err != nil {
// Ignore return here, there is already an error reported.
parentDir.Close()
return err
}
return parentDir.Close()
}
// readMappingsFromFile reads the user namespace configuration from the pod directory.
func (m *usernsManager) readMappingsFromFile(pod types.UID) ([]byte, error) {
dir := m.kl.getPodDir(pod)
fstore, err := utilstore.NewFileStore(dir, &utilfs.DefaultFs{})
if err != nil {
return nil, err
}
return fstore.Read(mappingsFile)
}
func MakeUserNsManager(kl userNsPodsManager) (*usernsManager, error) {
m := usernsManager{
// Create a bitArray for all the UID space (2^32).
// As a by product of that, no index param to bitArray can be out of bounds (index is uint32).
used: makeBitArray((math.MaxUint32 + 1) / userNsLength),
usedBy: make(map[types.UID]uint32),
kl: kl,
}
// First block is reserved for the host.
m.used.set(0)
// Second block will be used for phase II. Don't assign that range for now.
m.used.set(1)
// do not bother reading the list of pods if user namespaces are not enabled.
if !utilfeature.DefaultFeatureGate.Enabled(features.UserNamespacesStatelessPodsSupport) {
return &m, nil
}
found, err := kl.listPodsFromDisk()
if err != nil {
if os.IsNotExist(err) {
return &m, nil
}
return nil, fmt.Errorf("user namespace manager can't read pods from disk: %w", err)
}
for _, podUID := range found {
klog.V(5).InfoS("reading pod from disk for user namespace", "podUID", podUID)
if err := m.recordPodMappings(podUID); err != nil {
return nil, err
}
}
return &m, nil
}
// recordPodMappings registers the range used for the user namespace if the
// usernsConfFile exists in the pod directory.
func (m *usernsManager) recordPodMappings(pod types.UID) error {
content, err := m.readMappingsFromFile(pod)
if err != nil && err != utilstore.ErrKeyNotFound {
return err
}
// If no content, it means the pod doesn't have userns. Nothing else to do
if len(content) == 0 {
return nil
}
_, err = m.parseUserNsFileAndRecord(pod, content)
return err
}
// isSet checks if the specified index is already set.
func (m *usernsManager) isSet(v uint32) bool {
index := v / userNsLength
return m.used.isSet(index)
}
// allocateOne finds a free user namespace and allocate it to the specified pod.
// The first return value is the first ID in the user namespace, the second returns
// the length for the user namespace range.
func (m *usernsManager) allocateOne(pod types.UID) (firstID uint32, length uint32, err error) {
if m.numAllocated >= maxPods {
return 0, 0, fmt.Errorf("limit on count of pods with user namespaces exceeded (limit is %v, current pods with userns: %v)", maxPods, m.numAllocated)
}
m.numAllocated++
defer func() {
if err != nil {
m.numAllocated--
}
}()
firstZero, found := m.used.findAvailable()
if !found {
return 0, 0, fmt.Errorf("could not find an empty slot to allocate a user namespace")
}
klog.V(5).InfoS("new pod user namespace allocation", "podUID", pod)
firstID = firstZero * userNsLength
m.usedBy[pod] = firstID
return firstID, userNsLength, nil
}
// record stores the user namespace [from; from+length] to the specified pod.
func (m *usernsManager) record(pod types.UID, from, length uint32) (err error) {
if length != userNsLength {
return fmt.Errorf("wrong user namespace length %v", length)
}
if from%userNsLength != 0 {
return fmt.Errorf("wrong user namespace offset specified %v", from)
}
prevFrom, found := m.usedBy[pod]
if found && prevFrom != from {
return fmt.Errorf("different user namespace range already used by pod %q", pod)
}
index := from / userNsLength
// if the pod wasn't found then verify the range is free.
if !found && m.used.isSet(index) {
return fmt.Errorf("range picked for pod %q already taken", pod)
}
// The pod is already registered, nothing to do.
if found && prevFrom == from {
return nil
}
if m.numAllocated >= maxPods {
return fmt.Errorf("limit on count of pods with user namespaces exceeded (limit is %v, current pods with userns: %v)", maxPods, m.numAllocated)
}
m.numAllocated++
defer func() {
if err != nil {
m.numAllocated--
}
}()
klog.V(5).InfoS("new pod user namespace allocation", "podUID", pod)
// "from" is a ID (UID/GID), set the corresponding userns of size
// userNsLength in the bit-array.
m.used.set(index)
m.usedBy[pod] = from
return nil
}
// Release releases the user namespace allocated to the specified pod.
func (m *usernsManager) Release(podUID types.UID) {
if !utilfeature.DefaultFeatureGate.Enabled(features.UserNamespacesStatelessPodsSupport) {
return
}
m.lock.Lock()
defer m.lock.Unlock()
m.releaseWithLock(podUID)
}
func (m *usernsManager) releaseWithLock(pod types.UID) {
v, ok := m.usedBy[pod]
if !ok {
klog.V(5).InfoS("pod user namespace allocation not present", "podUID", pod)
return
}
delete(m.usedBy, pod)
klog.V(5).InfoS("releasing pod user namespace allocation", "podUID", pod)
m.numAllocated--
m.removed++
_ = os.Remove(filepath.Join(m.kl.getPodDir(pod), mappingsFile))
if m.removed%mapReInitializeThreshold == 0 {
n := make(map[types.UID]uint32)
for k, v := range m.usedBy {
n[k] = v
}
m.usedBy = n
m.removed = 0
}
m.used.clear(v / userNsLength)
}
func (m *usernsManager) parseUserNsFileAndRecord(pod types.UID, content []byte) (userNs userNamespace, err error) {
if err = json.Unmarshal([]byte(content), &userNs); err != nil {
err = fmt.Errorf("can't parse file: %w", err)
return
}
if len(userNs.UIDMappings) != 1 {
err = fmt.Errorf("invalid user namespace configuration: no more than one mapping allowed.")
return
}
if len(userNs.UIDMappings) != len(userNs.GIDMappings) {
err = fmt.Errorf("invalid user namespace configuration: GID and UID mappings should be identical.")
return
}
if userNs.UIDMappings[0] != userNs.GIDMappings[0] {
err = fmt.Errorf("invalid user namespace configuration: GID and UID mapping should be identical")
return
}
// We don't produce configs without root mapped and some runtimes assume it is mapped.
// Validate the file has something we produced and can digest.
if userNs.UIDMappings[0].ContainerId != 0 {
err = fmt.Errorf("invalid user namespace configuration: UID 0 must be mapped")
return
}
if userNs.GIDMappings[0].ContainerId != 0 {
err = fmt.Errorf("invalid user namespace configuration: GID 0 must be mapped")
return
}
hostId := userNs.UIDMappings[0].HostId
length := userNs.UIDMappings[0].Length
err = m.record(pod, hostId, length)
return
}
func (m *usernsManager) createUserNs(pod *v1.Pod) (userNs userNamespace, err error) {
firstID, length, err := m.allocateOne(pod.UID)
if err != nil {
return
}
defer func() {
if err != nil {
m.releaseWithLock(pod.UID)
}
}()
userNs = userNamespace{
UIDMappings: []idMapping{
{
ContainerId: 0,
HostId: firstID,
Length: length,
},
},
GIDMappings: []idMapping{
{
ContainerId: 0,
HostId: firstID,
Length: length,
},
},
}
return userNs, m.writeMappingsToFile(pod.UID, userNs)
}
// GetOrCreateUserNamespaceMappings returns the configuration for the sandbox user namespace
func (m *usernsManager) GetOrCreateUserNamespaceMappings(pod *v1.Pod) (*runtimeapi.UserNamespace, error) {
if !utilfeature.DefaultFeatureGate.Enabled(features.UserNamespacesStatelessPodsSupport) {
return nil, nil
}
m.lock.Lock()
defer m.lock.Unlock()
if pod.Spec.HostUsers == nil || *pod.Spec.HostUsers == true {
return &runtimeapi.UserNamespace{
Mode: runtimeapi.NamespaceMode_NODE,
}, nil
}
content, err := m.readMappingsFromFile(pod.UID)
if err != nil && err != utilstore.ErrKeyNotFound {
return nil, err
}
var userNs userNamespace
if string(content) != "" {
userNs, err = m.parseUserNsFileAndRecord(pod.UID, content)
if err != nil {
return nil, err
}
} else {
userNs, err = m.createUserNs(pod)
if err != nil {
return nil, err
}
}
var uids []*runtimeapi.IDMapping
var gids []*runtimeapi.IDMapping
for _, u := range userNs.UIDMappings {
uids = append(uids, &runtimeapi.IDMapping{
HostId: u.HostId,
ContainerId: u.ContainerId,
Length: u.Length,
})
}
for _, g := range userNs.GIDMappings {
gids = append(gids, &runtimeapi.IDMapping{
HostId: g.HostId,
ContainerId: g.ContainerId,
Length: g.Length,
})
}
return &runtimeapi.UserNamespace{
Mode: runtimeapi.NamespaceMode_POD,
Uids: uids,
Gids: gids,
}, nil
}
// CleanupOrphanedPodUsernsAllocations reconciliates the state of user namespace
// allocations with the pods actually running. It frees any user namespace
// allocation for orphaned pods.
func (m *usernsManager) CleanupOrphanedPodUsernsAllocations(pods []*v1.Pod, runningPods []*kubecontainer.Pod) error {
if !utilfeature.DefaultFeatureGate.Enabled(features.UserNamespacesStatelessPodsSupport) {
return nil
}
m.lock.Lock()
defer m.lock.Unlock()
allPods := sets.NewString()
for _, pod := range pods {
allPods.Insert(string(pod.UID))
}
for _, pod := range runningPods {
allPods.Insert(string(pod.ID))
}
allFound := sets.NewString()
found, err := m.kl.listPodsFromDisk()
if err != nil {
return err
}
for _, podUID := range found {
allFound.Insert(string(podUID))
}
// Lets remove all the pods "found" that are not known.
for _, podUID := range found {
if allPods.Has(string(podUID)) {
continue
}
klog.V(5).InfoS("Clean up orphaned pod user namespace possible allocation", "podUID", podUID)
m.releaseWithLock(podUID)
}
// Lets remove any existing allocation for a pod that is not "found".
for podUID := range m.usedBy {
if allFound.Has(string(podUID)) {
continue
}
klog.V(5).InfoS("Clean up orphaned pod user namespace possible allocation", "podUID", podUID)
m.releaseWithLock(podUID)
}
return nil
}

View File

@ -0,0 +1,215 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/types"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
pkgfeatures "k8s.io/kubernetes/pkg/features"
)
type testUserNsPodsManager struct {
}
func (m *testUserNsPodsManager) getPodDir(podUID types.UID) string {
return "/tmp/non-existant-dir.This-is-not-used-in-tests"
}
func (m *testUserNsPodsManager) listPodsFromDisk() ([]types.UID, error) {
return nil, nil
}
func TestUserNsManagerAllocate(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesStatelessPodsSupport, true)()
testUserNsPodsManager := &testUserNsPodsManager{}
m, err := MakeUserNsManager(testUserNsPodsManager)
require.NoError(t, err)
assert.Equal(t, true, m.isSet(0), "m.isSet(0) should be true")
assert.Equal(t, true, m.isSet(1), "m.isSet(1) should be true")
allocated, length, err := m.allocateOne("one")
assert.NoError(t, err)
assert.Equal(t, userNsLength, int(length), "m.isSet(%d).length=%v", allocated, length)
assert.Equal(t, true, m.isSet(allocated), "m.isSet(%d)", allocated)
assert.Equal(t, userNsLength*2, int(allocated))
allocated2, length2, err := m.allocateOne("two")
assert.NoError(t, err)
assert.NotEqual(t, allocated, allocated2, "allocated != allocated2")
assert.Equal(t, length, length2, "length == length2")
assert.Equal(t, uint32(userNsLength*3), allocated2)
// verify that re-adding the same pod with the same settings won't fail
err = m.record("two", allocated2, length2)
assert.NoError(t, err)
// but it fails if anyting is different
err = m.record("two", allocated2+1, length2)
assert.Error(t, err)
m.Release("one")
m.Release("two")
assert.Equal(t, false, m.isSet(allocated), "m.isSet(%d)", allocated)
assert.Equal(t, false, m.isSet(allocated2), "m.nsSet(%d)", allocated2)
var allocs []uint32
for i := 0; i < 1000; i++ {
allocated, length, err = m.allocateOne(types.UID(fmt.Sprintf("%d", i)))
assert.Equal(t, userNsLength, int(length), "length is not the expected. iter: %v", i)
assert.Equal(t, userNsLength*(i+2), int(allocated), "firstID is not the expected. iter: %v", i)
assert.NoError(t, err)
allocs = append(allocs, allocated)
}
for i, v := range allocs {
assert.Equal(t, true, m.isSet(v), "m.isSet(%d) should be true", v)
m.Release(types.UID(fmt.Sprintf("%d", i)))
assert.Equal(t, false, m.isSet(v), "m.isSet(%d) should be false", v)
err = m.record(types.UID(fmt.Sprintf("%d", i)), v, userNsLength)
assert.NoError(t, err)
m.Release(types.UID(fmt.Sprintf("%d", i)))
assert.Equal(t, false, m.isSet(v), "m.isSet(%d) should be false", v)
}
}
func TestUserNsManagerParseUserNsFile(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesStatelessPodsSupport, true)()
cases := []struct {
name string
file string
success bool
}{
{
name: "basic",
file: `{
"uidMappings":[ { "hostId":131072, "containerId":0, "length":65536 } ],
"gidMappings":[ { "hostId":131072, "containerId":0, "length":65536 } ]
}`,
success: true,
},
{
name: "invalid length",
file: `{
"uidMappings":[ { "hostId":131072, "containerId":0, "length":0 } ],
"gidMappings":[ { "hostId":131072, "containerId":0, "length":0 } ]
}`,
success: false,
},
{
name: "wrong offset",
file: `{
"uidMappings":[ {"hostId":131072, "containerId":0, "length":65536 } ],
"gidMappings":[ {"hostId":1, "containerId":0, "length":65536 } ]
}`,
success: false,
},
{
name: "two GID mappings",
file: `{
"uidMappings":[ { "hostId":131072, "containerId":0, "length":userNsLength } ],
"gidMappings":[ { "hostId":131072, "containerId":0, "length":userNsLength }, { "hostId":196608, "containerId":0, "length":65536 } ]
}`,
success: false,
},
{
name: "two UID mappings",
file: `{
"uidMappings":[ { "hostId":131072, "containerId":0, "length":65536 }, { "hostId":196608, "containerId":0, "length":65536 } ],
"gidMappings":[ { "hostId":131072, "containerId":0, "length":65536 } ]
}`,
success: false,
},
{
name: "no root UID",
file: `{
"uidMappings":[ { "hostId":131072, "containerId":1, "length":65536 } ],
"gidMappings":[ { "hostId":131072, "containerId":0, "length":65536 } ]
}`,
success: false,
},
{
name: "no root GID",
file: `{
"uidMappings":[ { "hostId":131072, "containerId":0, "length":65536 } ],
"gidMappings":[ { "hostId":131072, "containerId":1, "length":65536 } ]
}`,
success: false,
},
}
testUserNsPodsManager := &testUserNsPodsManager{}
m, err := MakeUserNsManager(testUserNsPodsManager)
assert.NoError(t, err)
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
// We don't validate the result. It was parsed with the json parser, we trust that.
_, err = m.parseUserNsFileAndRecord(types.UID(tc.name), []byte(tc.file))
if (tc.success && err == nil) || (!tc.success && err != nil) {
return
}
t.Errorf("expected success: %v but got error: %v", tc.success, err)
})
}
}
func BenchmarkBitmaskFindAndSetFirstZero(t *testing.B) {
b := makeBitArray(userNsLength)
for i := 0; i < userNsLength; i++ {
_, found := b.findAvailable()
assert.True(t, found)
}
}
func BenchmarkBitmaskSetAndClear(t *testing.B) {
b := makeBitArray(userNsLength)
for i := uint32(0); i < userNsLength; i++ {
b.set(i)
b.clear(i)
}
}
func BenchmarkBitmaskFindAndSetFirstZeroAndClear(t *testing.B) {
b := makeBitArray(userNsLength)
for i := 0; i < userNsLength; i++ {
ret, found := b.findAvailable()
assert.True(t, found)
b.clear(ret)
}
}
func BenchmarkBitmaskFindAndSetFirstZeroAndClear0Every2(t *testing.B) {
// it is an interesting edge case as it forces a full scan
// on each second allocation.
b := makeBitArray(userNsLength)
for i := 0; i < userNsLength; i++ {
_, found := b.findAvailable()
assert.True(t, found)
if i%2 == 0 {
b.clear(0)
}
}
}