mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-22 03:11:40 +00:00
kubelet: add userns manager
it is used to allocate and keep track of the unique users ranges assigned to each pod that runs in a user namespace. Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com> Signed-off-by: Rodrigo Campos <rodrigoca@microsoft.com> Co-authored-by: Rodrigo Campos <rodrigoca@microsoft.com>
This commit is contained in:
parent
cf8164bccf
commit
63462285d5
@ -837,6 +837,10 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
|
||||
StateDirectory: rootDirectory,
|
||||
})
|
||||
klet.shutdownManager = shutdownManager
|
||||
klet.usernsManager, err = MakeUserNsManager(klet)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
klet.admitHandlers.AddPodAdmitHandler(shutdownAdmitHandler)
|
||||
|
||||
// Finally, put the most recent version of the config on the Kubelet, so
|
||||
@ -1175,6 +1179,9 @@ type Kubelet struct {
|
||||
|
||||
// Handles node shutdown events for the Node.
|
||||
shutdownManager nodeshutdown.Manager
|
||||
|
||||
// Manage user namespaces
|
||||
usernsManager *usernsManager
|
||||
}
|
||||
|
||||
// ListPodStats is delegated to StatsProvider, which implements stats.Provider interface
|
||||
@ -1889,6 +1896,8 @@ func (kl *Kubelet) syncTerminatedPod(ctx context.Context, pod *v1.Pod, podStatus
|
||||
klog.V(4).InfoS("Pod termination removed cgroups", "pod", klog.KObj(pod), "podUID", pod.UID)
|
||||
}
|
||||
|
||||
kl.usernsManager.Release(pod.UID)
|
||||
|
||||
// mark the final pod status
|
||||
kl.statusManager.TerminatePod(pod)
|
||||
klog.V(4).InfoS("Pod is terminated and will need no more status updates", "pod", klog.KObj(pod), "podUID", pod.UID)
|
||||
|
@ -1154,6 +1154,12 @@ func (kl *Kubelet) HandlePodCleanups() error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove orphaned pod user namespace allocations (if any).
|
||||
klog.V(3).InfoS("Clean up orphaned pod user namespace allocations")
|
||||
if err = kl.usernsManager.CleanupOrphanedPodUsernsAllocations(allPods, runningRuntimePods); err != nil {
|
||||
klog.ErrorS(err, "Failed cleaning up orphaned pod user namespaces allocations")
|
||||
}
|
||||
|
||||
// Remove orphaned volumes from pods that are known not to have any
|
||||
// containers. Note that we pass all pods (including terminated pods) to
|
||||
// the function, so that we don't remove volumes associated with terminated
|
||||
|
530
pkg/kubelet/userns_manager.go
Normal file
530
pkg/kubelet/userns_manager.go
Normal file
@ -0,0 +1,530 @@
|
||||
/*
|
||||
Copyright 2022 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package kubelet
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
|
||||
"k8s.io/klog/v2"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
||||
utilstore "k8s.io/kubernetes/pkg/kubelet/util/store"
|
||||
utilfs "k8s.io/kubernetes/pkg/util/filesystem"
|
||||
)
|
||||
|
||||
// bitsDataElement is the number of bits in a bitArray.data element.
|
||||
const bitsDataElement = 32
|
||||
|
||||
type bitArray struct {
|
||||
data []uint32
|
||||
firstIndex int
|
||||
}
|
||||
|
||||
func makeBitArray(size uint32) *bitArray {
|
||||
m := bitArray{
|
||||
data: make([]uint32, (size+bitsDataElement-1)/bitsDataElement),
|
||||
firstIndex: 0,
|
||||
}
|
||||
return &m
|
||||
}
|
||||
|
||||
func (b *bitArray) set(index uint32) {
|
||||
b.data[index/bitsDataElement] |= (uint32(1) << (index % bitsDataElement))
|
||||
}
|
||||
|
||||
func (b *bitArray) isSet(index uint32) bool {
|
||||
return (b.data[index/bitsDataElement]>>(index%bitsDataElement))&0x1 == 1
|
||||
}
|
||||
|
||||
func (b *bitArray) findAvailable() (uint32, bool) {
|
||||
for i := b.firstIndex; i < len(b.data); i++ {
|
||||
// Check if all bits are used (all 1s).
|
||||
if b.data[i] == math.MaxUint32 {
|
||||
continue
|
||||
}
|
||||
for j := uint32(0); j < bitsDataElement; j++ {
|
||||
if (b.data[i]>>j)&0x1 == 0 {
|
||||
v := uint32(i)*bitsDataElement + j
|
||||
b.set(v)
|
||||
// Update firstIndex to the current
|
||||
// data element since there are no other
|
||||
// unset bits before the current index.
|
||||
b.firstIndex = int(i)
|
||||
return v, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func (b *bitArray) clear(index uint32) {
|
||||
i := index / bitsDataElement
|
||||
// update firstIndex if the index found is less than
|
||||
// the current one.
|
||||
if i < uint32(b.firstIndex) {
|
||||
b.firstIndex = int(i)
|
||||
}
|
||||
// clear the bit by ANDing the data element with the
|
||||
// complement of the bitmask to be cleared.
|
||||
b.data[i] &= ^(1 << (index % bitsDataElement))
|
||||
}
|
||||
|
||||
// length for the user namespace to create (65536).
|
||||
const userNsLength = (1 << 16)
|
||||
|
||||
// Limit the total number of pods using userns in this node to this value.
|
||||
// This is an alpha limitation that will probably be lifted later.
|
||||
const maxPods = 1024
|
||||
|
||||
// Create a new map when we removed enough pods to avoid memory leaks
|
||||
// since Go maps never free memory.
|
||||
const mapReInitializeThreshold = 1000
|
||||
|
||||
type userNsPodsManager interface {
|
||||
getPodDir(podUID types.UID) string
|
||||
listPodsFromDisk() ([]types.UID, error)
|
||||
}
|
||||
|
||||
type usernsManager struct {
|
||||
used *bitArray
|
||||
usedBy map[types.UID]uint32 // Map pod.UID to range used
|
||||
removed int
|
||||
numAllocated int
|
||||
kl userNsPodsManager
|
||||
// This protects all members except for kl.anager
|
||||
lock sync.Mutex
|
||||
}
|
||||
|
||||
// UserNamespace holds the configuration for the user namespace.
|
||||
type userNamespace struct {
|
||||
// UIDs mappings for the user namespace.
|
||||
UIDMappings []idMapping `json:"uidMappings"`
|
||||
// GIDs mappings for the user namespace.
|
||||
GIDMappings []idMapping `json:"gidMappings"`
|
||||
}
|
||||
|
||||
// Pod user namespace mapping
|
||||
type idMapping struct {
|
||||
// Required.
|
||||
HostId uint32 `json:"hostId"`
|
||||
// Required.
|
||||
ContainerId uint32 `json:"containerId"`
|
||||
// Required.
|
||||
Length uint32 `json:"length"`
|
||||
}
|
||||
|
||||
// mappingsFile is the file where the user namespace mappings are persisted.
|
||||
const mappingsFile = "userns"
|
||||
|
||||
// writeMappingsToFile writes the specified user namespace configuration to the pod
|
||||
// directory.
|
||||
func (m *usernsManager) writeMappingsToFile(pod types.UID, userNs userNamespace) error {
|
||||
dir := m.kl.getPodDir(pod)
|
||||
|
||||
data, err := json.Marshal(userNs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fstore, err := utilstore.NewFileStore(dir, &utilfs.DefaultFs{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := fstore.Write(mappingsFile, data); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// We need to fsync the parent dir so the file is guaranteed to be there.
|
||||
// fstore guarantees an atomic write, we need durability too.
|
||||
parentDir, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = parentDir.Sync(); err != nil {
|
||||
// Ignore return here, there is already an error reported.
|
||||
parentDir.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
return parentDir.Close()
|
||||
}
|
||||
|
||||
// readMappingsFromFile reads the user namespace configuration from the pod directory.
|
||||
func (m *usernsManager) readMappingsFromFile(pod types.UID) ([]byte, error) {
|
||||
dir := m.kl.getPodDir(pod)
|
||||
fstore, err := utilstore.NewFileStore(dir, &utilfs.DefaultFs{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fstore.Read(mappingsFile)
|
||||
}
|
||||
|
||||
func MakeUserNsManager(kl userNsPodsManager) (*usernsManager, error) {
|
||||
m := usernsManager{
|
||||
// Create a bitArray for all the UID space (2^32).
|
||||
// As a by product of that, no index param to bitArray can be out of bounds (index is uint32).
|
||||
used: makeBitArray((math.MaxUint32 + 1) / userNsLength),
|
||||
usedBy: make(map[types.UID]uint32),
|
||||
kl: kl,
|
||||
}
|
||||
// First block is reserved for the host.
|
||||
m.used.set(0)
|
||||
|
||||
// Second block will be used for phase II. Don't assign that range for now.
|
||||
m.used.set(1)
|
||||
|
||||
// do not bother reading the list of pods if user namespaces are not enabled.
|
||||
if !utilfeature.DefaultFeatureGate.Enabled(features.UserNamespacesStatelessPodsSupport) {
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
found, err := kl.listPodsFromDisk()
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return &m, nil
|
||||
}
|
||||
return nil, fmt.Errorf("user namespace manager can't read pods from disk: %w", err)
|
||||
|
||||
}
|
||||
for _, podUID := range found {
|
||||
klog.V(5).InfoS("reading pod from disk for user namespace", "podUID", podUID)
|
||||
if err := m.recordPodMappings(podUID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
// recordPodMappings registers the range used for the user namespace if the
|
||||
// usernsConfFile exists in the pod directory.
|
||||
func (m *usernsManager) recordPodMappings(pod types.UID) error {
|
||||
content, err := m.readMappingsFromFile(pod)
|
||||
if err != nil && err != utilstore.ErrKeyNotFound {
|
||||
return err
|
||||
}
|
||||
|
||||
// If no content, it means the pod doesn't have userns. Nothing else to do
|
||||
if len(content) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err = m.parseUserNsFileAndRecord(pod, content)
|
||||
return err
|
||||
}
|
||||
|
||||
// isSet checks if the specified index is already set.
|
||||
func (m *usernsManager) isSet(v uint32) bool {
|
||||
index := v / userNsLength
|
||||
return m.used.isSet(index)
|
||||
}
|
||||
|
||||
// allocateOne finds a free user namespace and allocate it to the specified pod.
|
||||
// The first return value is the first ID in the user namespace, the second returns
|
||||
// the length for the user namespace range.
|
||||
func (m *usernsManager) allocateOne(pod types.UID) (firstID uint32, length uint32, err error) {
|
||||
if m.numAllocated >= maxPods {
|
||||
return 0, 0, fmt.Errorf("limit on count of pods with user namespaces exceeded (limit is %v, current pods with userns: %v)", maxPods, m.numAllocated)
|
||||
}
|
||||
m.numAllocated++
|
||||
defer func() {
|
||||
if err != nil {
|
||||
m.numAllocated--
|
||||
}
|
||||
}()
|
||||
|
||||
firstZero, found := m.used.findAvailable()
|
||||
if !found {
|
||||
return 0, 0, fmt.Errorf("could not find an empty slot to allocate a user namespace")
|
||||
}
|
||||
|
||||
klog.V(5).InfoS("new pod user namespace allocation", "podUID", pod)
|
||||
|
||||
firstID = firstZero * userNsLength
|
||||
m.usedBy[pod] = firstID
|
||||
return firstID, userNsLength, nil
|
||||
}
|
||||
|
||||
// record stores the user namespace [from; from+length] to the specified pod.
|
||||
func (m *usernsManager) record(pod types.UID, from, length uint32) (err error) {
|
||||
if length != userNsLength {
|
||||
return fmt.Errorf("wrong user namespace length %v", length)
|
||||
}
|
||||
if from%userNsLength != 0 {
|
||||
return fmt.Errorf("wrong user namespace offset specified %v", from)
|
||||
}
|
||||
prevFrom, found := m.usedBy[pod]
|
||||
if found && prevFrom != from {
|
||||
return fmt.Errorf("different user namespace range already used by pod %q", pod)
|
||||
}
|
||||
index := from / userNsLength
|
||||
// if the pod wasn't found then verify the range is free.
|
||||
if !found && m.used.isSet(index) {
|
||||
return fmt.Errorf("range picked for pod %q already taken", pod)
|
||||
}
|
||||
// The pod is already registered, nothing to do.
|
||||
if found && prevFrom == from {
|
||||
return nil
|
||||
}
|
||||
if m.numAllocated >= maxPods {
|
||||
return fmt.Errorf("limit on count of pods with user namespaces exceeded (limit is %v, current pods with userns: %v)", maxPods, m.numAllocated)
|
||||
}
|
||||
m.numAllocated++
|
||||
defer func() {
|
||||
if err != nil {
|
||||
m.numAllocated--
|
||||
}
|
||||
}()
|
||||
|
||||
klog.V(5).InfoS("new pod user namespace allocation", "podUID", pod)
|
||||
|
||||
// "from" is a ID (UID/GID), set the corresponding userns of size
|
||||
// userNsLength in the bit-array.
|
||||
m.used.set(index)
|
||||
m.usedBy[pod] = from
|
||||
return nil
|
||||
}
|
||||
|
||||
// Release releases the user namespace allocated to the specified pod.
|
||||
func (m *usernsManager) Release(podUID types.UID) {
|
||||
if !utilfeature.DefaultFeatureGate.Enabled(features.UserNamespacesStatelessPodsSupport) {
|
||||
return
|
||||
}
|
||||
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
|
||||
m.releaseWithLock(podUID)
|
||||
}
|
||||
|
||||
func (m *usernsManager) releaseWithLock(pod types.UID) {
|
||||
v, ok := m.usedBy[pod]
|
||||
if !ok {
|
||||
klog.V(5).InfoS("pod user namespace allocation not present", "podUID", pod)
|
||||
return
|
||||
}
|
||||
delete(m.usedBy, pod)
|
||||
|
||||
klog.V(5).InfoS("releasing pod user namespace allocation", "podUID", pod)
|
||||
m.numAllocated--
|
||||
m.removed++
|
||||
|
||||
_ = os.Remove(filepath.Join(m.kl.getPodDir(pod), mappingsFile))
|
||||
|
||||
if m.removed%mapReInitializeThreshold == 0 {
|
||||
n := make(map[types.UID]uint32)
|
||||
for k, v := range m.usedBy {
|
||||
n[k] = v
|
||||
}
|
||||
m.usedBy = n
|
||||
m.removed = 0
|
||||
}
|
||||
m.used.clear(v / userNsLength)
|
||||
}
|
||||
|
||||
func (m *usernsManager) parseUserNsFileAndRecord(pod types.UID, content []byte) (userNs userNamespace, err error) {
|
||||
if err = json.Unmarshal([]byte(content), &userNs); err != nil {
|
||||
err = fmt.Errorf("can't parse file: %w", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(userNs.UIDMappings) != 1 {
|
||||
err = fmt.Errorf("invalid user namespace configuration: no more than one mapping allowed.")
|
||||
return
|
||||
}
|
||||
|
||||
if len(userNs.UIDMappings) != len(userNs.GIDMappings) {
|
||||
err = fmt.Errorf("invalid user namespace configuration: GID and UID mappings should be identical.")
|
||||
return
|
||||
}
|
||||
|
||||
if userNs.UIDMappings[0] != userNs.GIDMappings[0] {
|
||||
err = fmt.Errorf("invalid user namespace configuration: GID and UID mapping should be identical")
|
||||
return
|
||||
}
|
||||
|
||||
// We don't produce configs without root mapped and some runtimes assume it is mapped.
|
||||
// Validate the file has something we produced and can digest.
|
||||
if userNs.UIDMappings[0].ContainerId != 0 {
|
||||
err = fmt.Errorf("invalid user namespace configuration: UID 0 must be mapped")
|
||||
return
|
||||
}
|
||||
|
||||
if userNs.GIDMappings[0].ContainerId != 0 {
|
||||
err = fmt.Errorf("invalid user namespace configuration: GID 0 must be mapped")
|
||||
return
|
||||
}
|
||||
|
||||
hostId := userNs.UIDMappings[0].HostId
|
||||
length := userNs.UIDMappings[0].Length
|
||||
|
||||
err = m.record(pod, hostId, length)
|
||||
return
|
||||
}
|
||||
|
||||
func (m *usernsManager) createUserNs(pod *v1.Pod) (userNs userNamespace, err error) {
|
||||
firstID, length, err := m.allocateOne(pod.UID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
m.releaseWithLock(pod.UID)
|
||||
}
|
||||
}()
|
||||
|
||||
userNs = userNamespace{
|
||||
UIDMappings: []idMapping{
|
||||
{
|
||||
ContainerId: 0,
|
||||
HostId: firstID,
|
||||
Length: length,
|
||||
},
|
||||
},
|
||||
GIDMappings: []idMapping{
|
||||
{
|
||||
ContainerId: 0,
|
||||
HostId: firstID,
|
||||
Length: length,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return userNs, m.writeMappingsToFile(pod.UID, userNs)
|
||||
}
|
||||
|
||||
// GetOrCreateUserNamespaceMappings returns the configuration for the sandbox user namespace
|
||||
func (m *usernsManager) GetOrCreateUserNamespaceMappings(pod *v1.Pod) (*runtimeapi.UserNamespace, error) {
|
||||
if !utilfeature.DefaultFeatureGate.Enabled(features.UserNamespacesStatelessPodsSupport) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
|
||||
if pod.Spec.HostUsers == nil || *pod.Spec.HostUsers == true {
|
||||
return &runtimeapi.UserNamespace{
|
||||
Mode: runtimeapi.NamespaceMode_NODE,
|
||||
}, nil
|
||||
}
|
||||
|
||||
content, err := m.readMappingsFromFile(pod.UID)
|
||||
if err != nil && err != utilstore.ErrKeyNotFound {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var userNs userNamespace
|
||||
if string(content) != "" {
|
||||
userNs, err = m.parseUserNsFileAndRecord(pod.UID, content)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
userNs, err = m.createUserNs(pod)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
var uids []*runtimeapi.IDMapping
|
||||
var gids []*runtimeapi.IDMapping
|
||||
|
||||
for _, u := range userNs.UIDMappings {
|
||||
uids = append(uids, &runtimeapi.IDMapping{
|
||||
HostId: u.HostId,
|
||||
ContainerId: u.ContainerId,
|
||||
Length: u.Length,
|
||||
})
|
||||
}
|
||||
for _, g := range userNs.GIDMappings {
|
||||
gids = append(gids, &runtimeapi.IDMapping{
|
||||
HostId: g.HostId,
|
||||
ContainerId: g.ContainerId,
|
||||
Length: g.Length,
|
||||
})
|
||||
}
|
||||
|
||||
return &runtimeapi.UserNamespace{
|
||||
Mode: runtimeapi.NamespaceMode_POD,
|
||||
Uids: uids,
|
||||
Gids: gids,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CleanupOrphanedPodUsernsAllocations reconciliates the state of user namespace
|
||||
// allocations with the pods actually running. It frees any user namespace
|
||||
// allocation for orphaned pods.
|
||||
func (m *usernsManager) CleanupOrphanedPodUsernsAllocations(pods []*v1.Pod, runningPods []*kubecontainer.Pod) error {
|
||||
if !utilfeature.DefaultFeatureGate.Enabled(features.UserNamespacesStatelessPodsSupport) {
|
||||
return nil
|
||||
}
|
||||
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
|
||||
allPods := sets.NewString()
|
||||
for _, pod := range pods {
|
||||
allPods.Insert(string(pod.UID))
|
||||
}
|
||||
for _, pod := range runningPods {
|
||||
allPods.Insert(string(pod.ID))
|
||||
}
|
||||
|
||||
allFound := sets.NewString()
|
||||
found, err := m.kl.listPodsFromDisk()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, podUID := range found {
|
||||
allFound.Insert(string(podUID))
|
||||
}
|
||||
|
||||
// Lets remove all the pods "found" that are not known.
|
||||
for _, podUID := range found {
|
||||
if allPods.Has(string(podUID)) {
|
||||
continue
|
||||
}
|
||||
|
||||
klog.V(5).InfoS("Clean up orphaned pod user namespace possible allocation", "podUID", podUID)
|
||||
m.releaseWithLock(podUID)
|
||||
}
|
||||
|
||||
// Lets remove any existing allocation for a pod that is not "found".
|
||||
for podUID := range m.usedBy {
|
||||
if allFound.Has(string(podUID)) {
|
||||
continue
|
||||
}
|
||||
|
||||
klog.V(5).InfoS("Clean up orphaned pod user namespace possible allocation", "podUID", podUID)
|
||||
m.releaseWithLock(podUID)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
215
pkg/kubelet/userns_manager_test.go
Normal file
215
pkg/kubelet/userns_manager_test.go
Normal file
@ -0,0 +1,215 @@
|
||||
/*
|
||||
Copyright 2022 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package kubelet
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||
pkgfeatures "k8s.io/kubernetes/pkg/features"
|
||||
)
|
||||
|
||||
type testUserNsPodsManager struct {
|
||||
}
|
||||
|
||||
func (m *testUserNsPodsManager) getPodDir(podUID types.UID) string {
|
||||
return "/tmp/non-existant-dir.This-is-not-used-in-tests"
|
||||
}
|
||||
|
||||
func (m *testUserNsPodsManager) listPodsFromDisk() ([]types.UID, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestUserNsManagerAllocate(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesStatelessPodsSupport, true)()
|
||||
|
||||
testUserNsPodsManager := &testUserNsPodsManager{}
|
||||
m, err := MakeUserNsManager(testUserNsPodsManager)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, true, m.isSet(0), "m.isSet(0) should be true")
|
||||
assert.Equal(t, true, m.isSet(1), "m.isSet(1) should be true")
|
||||
|
||||
allocated, length, err := m.allocateOne("one")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, userNsLength, int(length), "m.isSet(%d).length=%v", allocated, length)
|
||||
assert.Equal(t, true, m.isSet(allocated), "m.isSet(%d)", allocated)
|
||||
assert.Equal(t, userNsLength*2, int(allocated))
|
||||
|
||||
allocated2, length2, err := m.allocateOne("two")
|
||||
assert.NoError(t, err)
|
||||
assert.NotEqual(t, allocated, allocated2, "allocated != allocated2")
|
||||
assert.Equal(t, length, length2, "length == length2")
|
||||
assert.Equal(t, uint32(userNsLength*3), allocated2)
|
||||
|
||||
// verify that re-adding the same pod with the same settings won't fail
|
||||
err = m.record("two", allocated2, length2)
|
||||
assert.NoError(t, err)
|
||||
// but it fails if anyting is different
|
||||
err = m.record("two", allocated2+1, length2)
|
||||
assert.Error(t, err)
|
||||
|
||||
m.Release("one")
|
||||
m.Release("two")
|
||||
assert.Equal(t, false, m.isSet(allocated), "m.isSet(%d)", allocated)
|
||||
assert.Equal(t, false, m.isSet(allocated2), "m.nsSet(%d)", allocated2)
|
||||
|
||||
var allocs []uint32
|
||||
for i := 0; i < 1000; i++ {
|
||||
allocated, length, err = m.allocateOne(types.UID(fmt.Sprintf("%d", i)))
|
||||
assert.Equal(t, userNsLength, int(length), "length is not the expected. iter: %v", i)
|
||||
assert.Equal(t, userNsLength*(i+2), int(allocated), "firstID is not the expected. iter: %v", i)
|
||||
assert.NoError(t, err)
|
||||
allocs = append(allocs, allocated)
|
||||
}
|
||||
for i, v := range allocs {
|
||||
assert.Equal(t, true, m.isSet(v), "m.isSet(%d) should be true", v)
|
||||
m.Release(types.UID(fmt.Sprintf("%d", i)))
|
||||
assert.Equal(t, false, m.isSet(v), "m.isSet(%d) should be false", v)
|
||||
|
||||
err = m.record(types.UID(fmt.Sprintf("%d", i)), v, userNsLength)
|
||||
assert.NoError(t, err)
|
||||
m.Release(types.UID(fmt.Sprintf("%d", i)))
|
||||
assert.Equal(t, false, m.isSet(v), "m.isSet(%d) should be false", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserNsManagerParseUserNsFile(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesStatelessPodsSupport, true)()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
file string
|
||||
success bool
|
||||
}{
|
||||
{
|
||||
name: "basic",
|
||||
file: `{
|
||||
"uidMappings":[ { "hostId":131072, "containerId":0, "length":65536 } ],
|
||||
"gidMappings":[ { "hostId":131072, "containerId":0, "length":65536 } ]
|
||||
}`,
|
||||
success: true,
|
||||
},
|
||||
{
|
||||
name: "invalid length",
|
||||
file: `{
|
||||
"uidMappings":[ { "hostId":131072, "containerId":0, "length":0 } ],
|
||||
"gidMappings":[ { "hostId":131072, "containerId":0, "length":0 } ]
|
||||
}`,
|
||||
success: false,
|
||||
},
|
||||
{
|
||||
name: "wrong offset",
|
||||
file: `{
|
||||
"uidMappings":[ {"hostId":131072, "containerId":0, "length":65536 } ],
|
||||
"gidMappings":[ {"hostId":1, "containerId":0, "length":65536 } ]
|
||||
}`,
|
||||
success: false,
|
||||
},
|
||||
{
|
||||
name: "two GID mappings",
|
||||
file: `{
|
||||
"uidMappings":[ { "hostId":131072, "containerId":0, "length":userNsLength } ],
|
||||
"gidMappings":[ { "hostId":131072, "containerId":0, "length":userNsLength }, { "hostId":196608, "containerId":0, "length":65536 } ]
|
||||
}`,
|
||||
success: false,
|
||||
},
|
||||
{
|
||||
name: "two UID mappings",
|
||||
file: `{
|
||||
"uidMappings":[ { "hostId":131072, "containerId":0, "length":65536 }, { "hostId":196608, "containerId":0, "length":65536 } ],
|
||||
"gidMappings":[ { "hostId":131072, "containerId":0, "length":65536 } ]
|
||||
}`,
|
||||
success: false,
|
||||
},
|
||||
{
|
||||
name: "no root UID",
|
||||
file: `{
|
||||
"uidMappings":[ { "hostId":131072, "containerId":1, "length":65536 } ],
|
||||
"gidMappings":[ { "hostId":131072, "containerId":0, "length":65536 } ]
|
||||
}`,
|
||||
success: false,
|
||||
},
|
||||
{
|
||||
name: "no root GID",
|
||||
file: `{
|
||||
"uidMappings":[ { "hostId":131072, "containerId":0, "length":65536 } ],
|
||||
"gidMappings":[ { "hostId":131072, "containerId":1, "length":65536 } ]
|
||||
}`,
|
||||
success: false,
|
||||
},
|
||||
}
|
||||
|
||||
testUserNsPodsManager := &testUserNsPodsManager{}
|
||||
m, err := MakeUserNsManager(testUserNsPodsManager)
|
||||
assert.NoError(t, err)
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// We don't validate the result. It was parsed with the json parser, we trust that.
|
||||
_, err = m.parseUserNsFileAndRecord(types.UID(tc.name), []byte(tc.file))
|
||||
if (tc.success && err == nil) || (!tc.success && err != nil) {
|
||||
return
|
||||
}
|
||||
|
||||
t.Errorf("expected success: %v but got error: %v", tc.success, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkBitmaskFindAndSetFirstZero(t *testing.B) {
|
||||
b := makeBitArray(userNsLength)
|
||||
for i := 0; i < userNsLength; i++ {
|
||||
_, found := b.findAvailable()
|
||||
assert.True(t, found)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkBitmaskSetAndClear(t *testing.B) {
|
||||
b := makeBitArray(userNsLength)
|
||||
for i := uint32(0); i < userNsLength; i++ {
|
||||
b.set(i)
|
||||
b.clear(i)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkBitmaskFindAndSetFirstZeroAndClear(t *testing.B) {
|
||||
b := makeBitArray(userNsLength)
|
||||
for i := 0; i < userNsLength; i++ {
|
||||
ret, found := b.findAvailable()
|
||||
assert.True(t, found)
|
||||
b.clear(ret)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkBitmaskFindAndSetFirstZeroAndClear0Every2(t *testing.B) {
|
||||
// it is an interesting edge case as it forces a full scan
|
||||
// on each second allocation.
|
||||
b := makeBitArray(userNsLength)
|
||||
for i := 0; i < userNsLength; i++ {
|
||||
_, found := b.findAvailable()
|
||||
assert.True(t, found)
|
||||
if i%2 == 0 {
|
||||
b.clear(0)
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user