mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-09 20:17:41 +00:00
Merge pull request #123593 from giuseppe/userns-use-kubelet-user-mappings
KEP-127: kubelet: honor kubelet user mappings
This commit is contained in:
commit
89cbd94e68
@ -823,6 +823,7 @@ const (
|
|||||||
// owner: @rata, @giuseppe
|
// owner: @rata, @giuseppe
|
||||||
// kep: https://kep.k8s.io/127
|
// kep: https://kep.k8s.io/127
|
||||||
// alpha: v1.25
|
// alpha: v1.25
|
||||||
|
// beta: v1.30
|
||||||
//
|
//
|
||||||
// Enables user namespace support for stateless pods.
|
// Enables user namespace support for stateless pods.
|
||||||
UserNamespacesSupport featuregate.Feature = "UserNamespacesSupport"
|
UserNamespacesSupport featuregate.Feature = "UserNamespacesSupport"
|
||||||
@ -1154,7 +1155,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
|
|||||||
|
|
||||||
VolumeCapacityPriority: {Default: false, PreRelease: featuregate.Alpha},
|
VolumeCapacityPriority: {Default: false, PreRelease: featuregate.Alpha},
|
||||||
|
|
||||||
UserNamespacesSupport: {Default: false, PreRelease: featuregate.Alpha},
|
UserNamespacesSupport: {Default: false, PreRelease: featuregate.Beta},
|
||||||
|
|
||||||
WinDSR: {Default: false, PreRelease: featuregate.Alpha},
|
WinDSR: {Default: false, PreRelease: featuregate.Alpha},
|
||||||
|
|
||||||
|
@ -123,6 +123,15 @@ func (kl *Kubelet) HandlerSupportsUserNamespaces(rtHandler string) (bool, error)
|
|||||||
return h.SupportsUserNamespaces, nil
|
return h.SupportsUserNamespaces, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetKubeletMappings gets the additional IDs allocated for the Kubelet.
|
||||||
|
func (kl *Kubelet) GetKubeletMappings() (uint32, uint32, error) {
|
||||||
|
return kl.getKubeletMappings()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (kl *Kubelet) GetMaxPods() int {
|
||||||
|
return kl.maxPods
|
||||||
|
}
|
||||||
|
|
||||||
// getPodDir returns the full path to the per-pod directory for the pod with
|
// getPodDir returns the full path to the per-pod directory for the pod with
|
||||||
// the given UID.
|
// the given UID.
|
||||||
func (kl *Kubelet) getPodDir(podUID types.UID) string {
|
func (kl *Kubelet) getPodDir(podUID types.UID) string {
|
||||||
|
@ -19,14 +19,18 @@ package kubelet
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
goerrors "errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"os/user"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
"runtime"
|
||||||
"sort"
|
"sort"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/google/go-cmp/cmp"
|
"github.com/google/go-cmp/cmp"
|
||||||
@ -76,8 +80,90 @@ const (
|
|||||||
const (
|
const (
|
||||||
PodInitializing = "PodInitializing"
|
PodInitializing = "PodInitializing"
|
||||||
ContainerCreating = "ContainerCreating"
|
ContainerCreating = "ContainerCreating"
|
||||||
|
|
||||||
|
kubeletUser = "kubelet"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// parseGetSubIdsOutput parses the output from the `getsubids` tool, which is used to query subordinate user or group ID ranges for
|
||||||
|
// a given user or group. getsubids produces a line for each mapping configured.
|
||||||
|
// Here we expect that there is a single mapping, and the same values are used for the subordinate user and group ID ranges.
|
||||||
|
// The output is something like:
|
||||||
|
// $ getsubids kubelet
|
||||||
|
// 0: kubelet 65536 2147483648
|
||||||
|
// $ getsubids -g kubelet
|
||||||
|
// 0: kubelet 65536 2147483648
|
||||||
|
func parseGetSubIdsOutput(input string) (uint32, uint32, error) {
|
||||||
|
lines := strings.Split(strings.Trim(input, "\n"), "\n")
|
||||||
|
if len(lines) != 1 {
|
||||||
|
return 0, 0, fmt.Errorf("error parsing line %q: it must contain only one line", input)
|
||||||
|
}
|
||||||
|
|
||||||
|
parts := strings.Fields(lines[0])
|
||||||
|
if len(parts) != 4 {
|
||||||
|
return 0, 0, fmt.Errorf("invalid line %q", input)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parsing the numbers
|
||||||
|
num1, err := strconv.ParseUint(parts[2], 10, 32)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("error parsing line %q: %w", input, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
num2, err := strconv.ParseUint(parts[3], 10, 32)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("error parsing line %q: %w", input, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return uint32(num1), uint32(num2), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// getKubeletMappings returns the range of IDs that can be used to configure user namespaces.
|
||||||
|
// If subordinate user or group ID ranges are specified for the kubelet user and the getsubids tool
|
||||||
|
// is installed, then the single mapping specified both for user and group IDs will be used.
|
||||||
|
// If the tool is not installed, or there are no IDs configured, the default mapping is returned.
|
||||||
|
// The default mapping includes the entire IDs range except IDs below 65536.
|
||||||
|
func (kl *Kubelet) getKubeletMappings() (uint32, uint32, error) {
|
||||||
|
// default mappings to return if there is no specific configuration
|
||||||
|
const defaultFirstID = 1 << 16
|
||||||
|
const defaultLen = 1<<32 - defaultFirstID
|
||||||
|
|
||||||
|
if !utilfeature.DefaultFeatureGate.Enabled(features.UserNamespacesSupport) {
|
||||||
|
return defaultFirstID, defaultLen, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := user.Lookup(kubeletUser)
|
||||||
|
if err != nil {
|
||||||
|
var unknownUserErr user.UnknownUserError
|
||||||
|
if goerrors.As(err, &unknownUserErr) {
|
||||||
|
// if the user is not found, we assume that the user is not configured
|
||||||
|
return defaultFirstID, defaultLen, nil
|
||||||
|
}
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
execName := "getsubids"
|
||||||
|
cmd, err := exec.LookPath(execName)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
klog.V(2).InfoS("Could not find executable, default mappings will be used for the user namespaces", "executable", execName, "err", err)
|
||||||
|
return defaultFirstID, defaultLen, nil
|
||||||
|
}
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
outUids, err := exec.Command(cmd, kubeletUser).Output()
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("error retrieving additional ids for user %q", kubeletUser)
|
||||||
|
}
|
||||||
|
outGids, err := exec.Command(cmd, "-g", kubeletUser).Output()
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("error retrieving additional gids for user %q", kubeletUser)
|
||||||
|
}
|
||||||
|
if string(outUids) != string(outGids) {
|
||||||
|
return 0, 0, fmt.Errorf("mismatched subuids and subgids for user %q", kubeletUser)
|
||||||
|
}
|
||||||
|
return parseGetSubIdsOutput(string(outUids))
|
||||||
|
}
|
||||||
|
|
||||||
// Get a list of pods that have data directories.
|
// Get a list of pods that have data directories.
|
||||||
func (kl *Kubelet) listPodsFromDisk() ([]types.UID, error) {
|
func (kl *Kubelet) listPodsFromDisk() ([]types.UID, error) {
|
||||||
podInfos, err := os.ReadDir(kl.getPodsDir())
|
podInfos, err := os.ReadDir(kl.getPodsDir())
|
||||||
|
@ -6013,3 +6013,77 @@ func TestGetNonExistentImagePullSecret(t *testing.T) {
|
|||||||
event := <-fakeRecorder.Events
|
event := <-fakeRecorder.Events
|
||||||
assert.Equal(t, event, expectedEvent)
|
assert.Equal(t, event, expectedEvent)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseGetSubIdsOutput(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
input string
|
||||||
|
wantFirstID uint32
|
||||||
|
wantRangeLen uint32
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "valid",
|
||||||
|
input: "0: kubelet 65536 2147483648",
|
||||||
|
wantFirstID: 65536,
|
||||||
|
wantRangeLen: 2147483648,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "multiple lines",
|
||||||
|
input: "0: kubelet 1 2\n1: kubelet 3 4\n",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "wrong format",
|
||||||
|
input: "0: kubelet 65536",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non numeric 1",
|
||||||
|
input: "0: kubelet Foo 65536",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non numeric 2",
|
||||||
|
input: "0: kubelet 0 Bar",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "overflow 1",
|
||||||
|
input: "0: kubelet 4294967296 2147483648",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "overflow 2",
|
||||||
|
input: "0: kubelet 65536 4294967296",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "negative value 1",
|
||||||
|
input: "0: kubelet -1 2147483648",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "negative value 2",
|
||||||
|
input: "0: kubelet 65536 -1",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
gotFirstID, gotRangeLen, err := parseGetSubIdsOutput(tc.input)
|
||||||
|
if tc.wantErr {
|
||||||
|
if err == nil {
|
||||||
|
t.Errorf("%s: expected error, got nil", tc.name)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("%s: unexpected error: %v", tc.name, err)
|
||||||
|
}
|
||||||
|
if gotFirstID != tc.wantFirstID || gotRangeLen != tc.wantRangeLen {
|
||||||
|
t.Errorf("%s: got (%d, %d), want (%d, %d)", tc.name, gotFirstID, gotRangeLen, tc.wantFirstID, tc.wantRangeLen)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@ -19,7 +19,6 @@ package userns
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sync"
|
"sync"
|
||||||
@ -40,10 +39,6 @@ import (
|
|||||||
// length for the user namespace to create (65536).
|
// length for the user namespace to create (65536).
|
||||||
const userNsLength = (1 << 16)
|
const userNsLength = (1 << 16)
|
||||||
|
|
||||||
// Limit the total number of pods using userns in this node to this value.
|
|
||||||
// This is an alpha limitation that will probably be lifted later.
|
|
||||||
const maxPods = 1024
|
|
||||||
|
|
||||||
// Create a new map when we removed enough pods to avoid memory leaks
|
// Create a new map when we removed enough pods to avoid memory leaks
|
||||||
// since Go maps never free memory.
|
// since Go maps never free memory.
|
||||||
const mapReInitializeThreshold = 1000
|
const mapReInitializeThreshold = 1000
|
||||||
@ -52,14 +47,19 @@ type userNsPodsManager interface {
|
|||||||
HandlerSupportsUserNamespaces(runtimeHandler string) (bool, error)
|
HandlerSupportsUserNamespaces(runtimeHandler string) (bool, error)
|
||||||
GetPodDir(podUID types.UID) string
|
GetPodDir(podUID types.UID) string
|
||||||
ListPodsFromDisk() ([]types.UID, error)
|
ListPodsFromDisk() ([]types.UID, error)
|
||||||
|
GetKubeletMappings() (uint32, uint32, error)
|
||||||
|
GetMaxPods() int
|
||||||
}
|
}
|
||||||
|
|
||||||
type UsernsManager struct {
|
type UsernsManager struct {
|
||||||
used *allocator.AllocationBitmap
|
used *allocator.AllocationBitmap
|
||||||
usedBy map[types.UID]uint32 // Map pod.UID to range used
|
usedBy map[types.UID]uint32 // Map pod.UID to range used
|
||||||
removed int
|
removed int
|
||||||
numAllocated int
|
|
||||||
kl userNsPodsManager
|
off int
|
||||||
|
len int
|
||||||
|
|
||||||
|
kl userNsPodsManager
|
||||||
// This protects all members except for kl.anager
|
// This protects all members except for kl.anager
|
||||||
lock sync.Mutex
|
lock sync.Mutex
|
||||||
}
|
}
|
||||||
@ -130,16 +130,33 @@ func (m *UsernsManager) readMappingsFromFile(pod types.UID) ([]byte, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func MakeUserNsManager(kl userNsPodsManager) (*UsernsManager, error) {
|
func MakeUserNsManager(kl userNsPodsManager) (*UsernsManager, error) {
|
||||||
|
kubeletMappingID, kubeletMappingLen, err := kl.GetKubeletMappings()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if kubeletMappingID%userNsLength != 0 {
|
||||||
|
return nil, fmt.Errorf("kubelet user assigned ID %v is not a multiple of %v", kubeletMappingID, userNsLength)
|
||||||
|
}
|
||||||
|
if kubeletMappingID < userNsLength {
|
||||||
|
// We don't allow to map 0, as security is circumvented.
|
||||||
|
return nil, fmt.Errorf("kubelet user assigned ID %v must be greater or equal to %v", kubeletMappingID, userNsLength)
|
||||||
|
}
|
||||||
|
if kubeletMappingLen%userNsLength != 0 {
|
||||||
|
return nil, fmt.Errorf("kubelet user assigned IDs length %v is not a multiple of %v", kubeletMappingLen, userNsLength)
|
||||||
|
}
|
||||||
|
if kubeletMappingLen/userNsLength < uint32(kl.GetMaxPods()) {
|
||||||
|
return nil, fmt.Errorf("kubelet user assigned IDs are not enough to support %v pods", kl.GetMaxPods())
|
||||||
|
}
|
||||||
|
off := int(kubeletMappingID / userNsLength)
|
||||||
|
len := int(kubeletMappingLen / userNsLength)
|
||||||
|
|
||||||
m := UsernsManager{
|
m := UsernsManager{
|
||||||
// Create a bitArray for all the UID space (2^32).
|
used: allocator.NewAllocationMap(len, "user namespaces"),
|
||||||
// As a by product of that, no index param to bitArray can be out of bounds (index is uint32).
|
|
||||||
used: allocator.NewAllocationMap((math.MaxUint32+1)/userNsLength, "user namespaces"),
|
|
||||||
usedBy: make(map[types.UID]uint32),
|
usedBy: make(map[types.UID]uint32),
|
||||||
kl: kl,
|
kl: kl,
|
||||||
}
|
off: off,
|
||||||
// First block is reserved for the host.
|
len: len,
|
||||||
if _, err := m.used.Allocate(0); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// do not bother reading the list of pods if user namespaces are not enabled.
|
// do not bother reading the list of pods if user namespaces are not enabled.
|
||||||
@ -184,7 +201,10 @@ func (m *UsernsManager) recordPodMappings(pod types.UID) error {
|
|||||||
|
|
||||||
// isSet checks if the specified index is already set.
|
// isSet checks if the specified index is already set.
|
||||||
func (m *UsernsManager) isSet(v uint32) bool {
|
func (m *UsernsManager) isSet(v uint32) bool {
|
||||||
index := int(v / userNsLength)
|
index := int(v/userNsLength) - m.off
|
||||||
|
if index < 0 || index >= m.len {
|
||||||
|
return true
|
||||||
|
}
|
||||||
return m.used.Has(index)
|
return m.used.Has(index)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -192,16 +212,6 @@ func (m *UsernsManager) isSet(v uint32) bool {
|
|||||||
// The first return value is the first ID in the user namespace, the second returns
|
// The first return value is the first ID in the user namespace, the second returns
|
||||||
// the length for the user namespace range.
|
// the length for the user namespace range.
|
||||||
func (m *UsernsManager) allocateOne(pod types.UID) (firstID uint32, length uint32, err error) {
|
func (m *UsernsManager) allocateOne(pod types.UID) (firstID uint32, length uint32, err error) {
|
||||||
if m.numAllocated >= maxPods {
|
|
||||||
return 0, 0, fmt.Errorf("limit on count of pods with user namespaces exceeded (limit is %v, current pods with userns: %v)", maxPods, m.numAllocated)
|
|
||||||
}
|
|
||||||
m.numAllocated++
|
|
||||||
defer func() {
|
|
||||||
if err != nil {
|
|
||||||
m.numAllocated--
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
firstZero, found, err := m.used.AllocateNext()
|
firstZero, found, err := m.used.AllocateNext()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, err
|
return 0, 0, err
|
||||||
@ -212,7 +222,7 @@ func (m *UsernsManager) allocateOne(pod types.UID) (firstID uint32, length uint3
|
|||||||
|
|
||||||
klog.V(5).InfoS("new pod user namespace allocation", "podUID", pod)
|
klog.V(5).InfoS("new pod user namespace allocation", "podUID", pod)
|
||||||
|
|
||||||
firstID = uint32(firstZero * userNsLength)
|
firstID = uint32((firstZero + m.off) * userNsLength)
|
||||||
m.usedBy[pod] = firstID
|
m.usedBy[pod] = firstID
|
||||||
return firstID, userNsLength, nil
|
return firstID, userNsLength, nil
|
||||||
}
|
}
|
||||||
@ -229,7 +239,10 @@ func (m *UsernsManager) record(pod types.UID, from, length uint32) (err error) {
|
|||||||
if found && prevFrom != from {
|
if found && prevFrom != from {
|
||||||
return fmt.Errorf("different user namespace range already used by pod %q", pod)
|
return fmt.Errorf("different user namespace range already used by pod %q", pod)
|
||||||
}
|
}
|
||||||
index := int(from / userNsLength)
|
index := int(from/userNsLength) - m.off
|
||||||
|
if index < 0 || index >= m.len {
|
||||||
|
return fmt.Errorf("id %v is out of range", from)
|
||||||
|
}
|
||||||
// if the pod wasn't found then verify the range is free.
|
// if the pod wasn't found then verify the range is free.
|
||||||
if !found && m.used.Has(index) {
|
if !found && m.used.Has(index) {
|
||||||
return fmt.Errorf("range picked for pod %q already taken", pod)
|
return fmt.Errorf("range picked for pod %q already taken", pod)
|
||||||
@ -238,15 +251,6 @@ func (m *UsernsManager) record(pod types.UID, from, length uint32) (err error) {
|
|||||||
if found && prevFrom == from {
|
if found && prevFrom == from {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if m.numAllocated >= maxPods {
|
|
||||||
return fmt.Errorf("limit on count of pods with user namespaces exceeded (limit is %v, current pods with userns: %v)", maxPods, m.numAllocated)
|
|
||||||
}
|
|
||||||
m.numAllocated++
|
|
||||||
defer func() {
|
|
||||||
if err != nil {
|
|
||||||
m.numAllocated--
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
klog.V(5).InfoS("new pod user namespace allocation", "podUID", pod)
|
klog.V(5).InfoS("new pod user namespace allocation", "podUID", pod)
|
||||||
|
|
||||||
@ -291,7 +295,6 @@ func (m *UsernsManager) releaseWithLock(pod types.UID) {
|
|||||||
delete(m.usedBy, pod)
|
delete(m.usedBy, pod)
|
||||||
|
|
||||||
klog.V(5).InfoS("releasing pod user namespace allocation", "podUID", pod)
|
klog.V(5).InfoS("releasing pod user namespace allocation", "podUID", pod)
|
||||||
m.numAllocated--
|
|
||||||
m.removed++
|
m.removed++
|
||||||
|
|
||||||
_ = os.Remove(filepath.Join(m.kl.GetPodDir(pod), mappingsFile))
|
_ = os.Remove(filepath.Join(m.kl.GetPodDir(pod), mappingsFile))
|
||||||
@ -304,7 +307,7 @@ func (m *UsernsManager) releaseWithLock(pod types.UID) {
|
|||||||
m.usedBy = n
|
m.usedBy = n
|
||||||
m.removed = 0
|
m.removed = 0
|
||||||
}
|
}
|
||||||
m.used.Release(int(v / userNsLength))
|
_ = m.used.Release(int(v/userNsLength) - m.off)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *UsernsManager) parseUserNsFileAndRecord(pod types.UID, content []byte) (userNs userNamespace, err error) {
|
func (m *UsernsManager) parseUserNsFileAndRecord(pod types.UID, content []byte) (userNs userNamespace, err error) {
|
||||||
|
137
pkg/kubelet/userns/userns_manager_switch_test.go
Normal file
137
pkg/kubelet/userns/userns_manager_switch_test.go
Normal file
@ -0,0 +1,137 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2024 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package userns
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
v1 "k8s.io/api/core/v1"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/types"
|
||||||
|
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||||
|
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||||
|
pkgfeatures "k8s.io/kubernetes/pkg/features"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMakeUserNsManagerSwitch(t *testing.T) {
|
||||||
|
// Create the manager with the feature gate enabled, to record some pods on disk.
|
||||||
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)()
|
||||||
|
|
||||||
|
pods := []types.UID{"pod-1", "pod-2"}
|
||||||
|
|
||||||
|
testUserNsPodsManager := &testUserNsPodsManager{
|
||||||
|
podDir: t.TempDir(),
|
||||||
|
// List the same pods we will record, so the second time we create the userns
|
||||||
|
// manager, it will find these pods on disk with userns data.
|
||||||
|
podList: pods,
|
||||||
|
}
|
||||||
|
m, err := MakeUserNsManager(testUserNsPodsManager)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Record the pods on disk.
|
||||||
|
for _, podUID := range pods {
|
||||||
|
pod := v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: podUID}}
|
||||||
|
_, err := m.GetOrCreateUserNamespaceMappings(&pod, "")
|
||||||
|
require.NoError(t, err, "failed to record userns range for pod %v", podUID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test re-init works when the feature gate is disabled and there were some
|
||||||
|
// pods written on disk.
|
||||||
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, false)()
|
||||||
|
m2, err := MakeUserNsManager(testUserNsPodsManager)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// The feature gate is off, no pods should be allocated.
|
||||||
|
for _, pod := range pods {
|
||||||
|
ok := m2.podAllocated(pod)
|
||||||
|
assert.False(t, ok, "pod %q should not be allocated", pod)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetOrCreateUserNamespaceMappingsSwitch(t *testing.T) {
|
||||||
|
// Enable the feature gate to create some pods on disk.
|
||||||
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)()
|
||||||
|
|
||||||
|
pods := []types.UID{"pod-1", "pod-2"}
|
||||||
|
|
||||||
|
testUserNsPodsManager := &testUserNsPodsManager{
|
||||||
|
podDir: t.TempDir(),
|
||||||
|
// List the same pods we will record, so the second time we create the userns
|
||||||
|
// manager, it will find these pods on disk with userns data.
|
||||||
|
podList: pods,
|
||||||
|
}
|
||||||
|
m, err := MakeUserNsManager(testUserNsPodsManager)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Record the pods on disk.
|
||||||
|
for _, podUID := range pods {
|
||||||
|
pod := v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: podUID}}
|
||||||
|
_, err := m.GetOrCreateUserNamespaceMappings(&pod, "")
|
||||||
|
require.NoError(t, err, "failed to record userns range for pod %v", podUID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test no-op when the feature gate is disabled and there were some
|
||||||
|
// pods registered on disk.
|
||||||
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, false)()
|
||||||
|
// Create a new manager with the feature gate off and verify the userns range is nil.
|
||||||
|
m2, err := MakeUserNsManager(testUserNsPodsManager)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
for _, podUID := range pods {
|
||||||
|
pod := v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: podUID}}
|
||||||
|
userns, err := m2.GetOrCreateUserNamespaceMappings(&pod, "")
|
||||||
|
|
||||||
|
assert.NoError(t, err, "failed to record userns range for pod %v", podUID)
|
||||||
|
assert.Nil(t, userns, "userns range should be nil for pod %v", podUID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCleanupOrphanedPodUsernsAllocationsSwitch(t *testing.T) {
|
||||||
|
// Enable the feature gate to create some pods on disk.
|
||||||
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)()
|
||||||
|
|
||||||
|
listPods := []types.UID{"pod-1", "pod-2"}
|
||||||
|
pods := []types.UID{"pod-3", "pod-4"}
|
||||||
|
testUserNsPodsManager := &testUserNsPodsManager{
|
||||||
|
podDir: t.TempDir(),
|
||||||
|
podList: listPods,
|
||||||
|
}
|
||||||
|
|
||||||
|
m, err := MakeUserNsManager(testUserNsPodsManager)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Record the pods on disk.
|
||||||
|
for _, podUID := range pods {
|
||||||
|
pod := v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: podUID}}
|
||||||
|
_, err := m.GetOrCreateUserNamespaceMappings(&pod, "")
|
||||||
|
require.NoError(t, err, "failed to record userns range for pod %v", podUID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test cleanup works when the feature gate is disabled and there were some
|
||||||
|
// pods registered.
|
||||||
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, false)()
|
||||||
|
err = m.CleanupOrphanedPodUsernsAllocations(nil, nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// The feature gate is off, no pods should be allocated.
|
||||||
|
for _, pod := range append(listPods, pods...) {
|
||||||
|
ok := m.podAllocated(pod)
|
||||||
|
assert.False(t, ok, "pod %q should not be allocated", pod)
|
||||||
|
}
|
||||||
|
}
|
@ -34,10 +34,21 @@ import (
|
|||||||
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// skip the first block
|
||||||
|
minimumMappingUID = userNsLength
|
||||||
|
// allocate enough space for 2000 user namespaces
|
||||||
|
mappingLen = userNsLength * 2000
|
||||||
|
testMaxPods = 110
|
||||||
|
)
|
||||||
|
|
||||||
type testUserNsPodsManager struct {
|
type testUserNsPodsManager struct {
|
||||||
podDir string
|
podDir string
|
||||||
podList []types.UID
|
podList []types.UID
|
||||||
userns bool
|
userns bool
|
||||||
|
maxPods int
|
||||||
|
mappingFirstID uint32
|
||||||
|
mappingLen uint32
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *testUserNsPodsManager) GetPodDir(podUID types.UID) string {
|
func (m *testUserNsPodsManager) GetPodDir(podUID types.UID) string {
|
||||||
@ -61,6 +72,21 @@ func (m *testUserNsPodsManager) HandlerSupportsUserNamespaces(runtimeHandler str
|
|||||||
return m.userns, nil
|
return m.userns, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *testUserNsPodsManager) GetKubeletMappings() (uint32, uint32, error) {
|
||||||
|
if m.mappingFirstID != 0 {
|
||||||
|
return m.mappingFirstID, m.mappingLen, nil
|
||||||
|
}
|
||||||
|
return minimumMappingUID, mappingLen, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *testUserNsPodsManager) GetMaxPods() int {
|
||||||
|
if m.maxPods != 0 {
|
||||||
|
return m.maxPods
|
||||||
|
}
|
||||||
|
|
||||||
|
return testMaxPods
|
||||||
|
}
|
||||||
|
|
||||||
func TestUserNsManagerAllocate(t *testing.T) {
|
func TestUserNsManagerAllocate(t *testing.T) {
|
||||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)()
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)()
|
||||||
|
|
||||||
@ -68,8 +94,6 @@ func TestUserNsManagerAllocate(t *testing.T) {
|
|||||||
m, err := MakeUserNsManager(testUserNsPodsManager)
|
m, err := MakeUserNsManager(testUserNsPodsManager)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
assert.Equal(t, true, m.isSet(0*65536), "m.isSet(0) should be true")
|
|
||||||
|
|
||||||
allocated, length, err := m.allocateOne("one")
|
allocated, length, err := m.allocateOne("one")
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.Equal(t, userNsLength, int(length), "m.isSet(%d).length=%v", allocated, length)
|
assert.Equal(t, userNsLength, int(length), "m.isSet(%d).length=%v", allocated, length)
|
||||||
@ -97,6 +121,9 @@ func TestUserNsManagerAllocate(t *testing.T) {
|
|||||||
allocated, length, err = m.allocateOne(types.UID(fmt.Sprintf("%d", i)))
|
allocated, length, err = m.allocateOne(types.UID(fmt.Sprintf("%d", i)))
|
||||||
assert.Equal(t, userNsLength, int(length), "length is not the expected. iter: %v", i)
|
assert.Equal(t, userNsLength, int(length), "length is not the expected. iter: %v", i)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
assert.True(t, allocated >= minimumMappingUID)
|
||||||
|
// The last ID of the userns range (allocated+userNsLength) should be within bounds.
|
||||||
|
assert.True(t, allocated <= minimumMappingUID+mappingLen-userNsLength)
|
||||||
allocs = append(allocs, allocated)
|
allocs = append(allocs, allocated)
|
||||||
}
|
}
|
||||||
for i, v := range allocs {
|
for i, v := range allocs {
|
||||||
@ -111,6 +138,60 @@ func TestUserNsManagerAllocate(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMakeUserNsManager(t *testing.T) {
|
||||||
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)()
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
mappingFirstID uint32
|
||||||
|
mappingLen uint32
|
||||||
|
maxPods int
|
||||||
|
success bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "default",
|
||||||
|
success: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "firstID not multiple",
|
||||||
|
mappingFirstID: 65536 + 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "firstID is less than 65535",
|
||||||
|
mappingFirstID: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "mappingLen not multiple",
|
||||||
|
mappingFirstID: 65536,
|
||||||
|
mappingLen: 65536 + 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "range can't fit maxPods",
|
||||||
|
mappingFirstID: 65536,
|
||||||
|
mappingLen: 65536,
|
||||||
|
maxPods: 2,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
testUserNsPodsManager := &testUserNsPodsManager{
|
||||||
|
podDir: t.TempDir(),
|
||||||
|
mappingFirstID: tc.mappingFirstID,
|
||||||
|
mappingLen: tc.mappingLen,
|
||||||
|
maxPods: tc.maxPods,
|
||||||
|
}
|
||||||
|
_, err := MakeUserNsManager(testUserNsPodsManager)
|
||||||
|
|
||||||
|
if tc.success {
|
||||||
|
assert.NoError(t, err)
|
||||||
|
} else {
|
||||||
|
assert.Error(t, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestUserNsManagerParseUserNsFile(t *testing.T) {
|
func TestUserNsManagerParseUserNsFile(t *testing.T) {
|
||||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)()
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)()
|
||||||
|
|
||||||
@ -366,42 +447,6 @@ func TestCleanupOrphanedPodUsernsAllocations(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAllocateMaxPods(t *testing.T) {
|
|
||||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)()
|
|
||||||
|
|
||||||
testUserNsPodsManager := &testUserNsPodsManager{}
|
|
||||||
m, err := MakeUserNsManager(testUserNsPodsManager)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// The first maxPods allocations should succeed.
|
|
||||||
for i := 0; i < maxPods; i++ {
|
|
||||||
_, _, err = m.allocateOne(types.UID(fmt.Sprintf("%d", i)))
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The next allocation should fail, hitting maxPods.
|
|
||||||
_, _, err = m.allocateOne(types.UID(fmt.Sprintf("%d", maxPods+1)))
|
|
||||||
assert.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRecordMaxPods(t *testing.T) {
|
|
||||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)()
|
|
||||||
|
|
||||||
testUserNsPodsManager := &testUserNsPodsManager{}
|
|
||||||
m, err := MakeUserNsManager(testUserNsPodsManager)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// The first maxPods allocations should succeed.
|
|
||||||
for i := 0; i < maxPods; i++ {
|
|
||||||
err = m.record(types.UID(fmt.Sprintf("%d", i)), uint32((i+1)*65536), 65536)
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The next allocation should fail, hitting maxPods.
|
|
||||||
err = m.record(types.UID(fmt.Sprintf("%d", maxPods+1)), uint32((maxPods+1)*65536), 65536)
|
|
||||||
assert.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
type failingUserNsPodsManager struct {
|
type failingUserNsPodsManager struct {
|
||||||
testUserNsPodsManager
|
testUserNsPodsManager
|
||||||
}
|
}
|
||||||
@ -418,3 +463,25 @@ func TestMakeUserNsManagerFailsListPod(t *testing.T) {
|
|||||||
assert.Error(t, err)
|
assert.Error(t, err)
|
||||||
assert.ErrorContains(t, err, "read pods from disk")
|
assert.ErrorContains(t, err, "read pods from disk")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRecordBounds(t *testing.T) {
|
||||||
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.UserNamespacesSupport, true)()
|
||||||
|
|
||||||
|
// Allow exactly for 1 pod
|
||||||
|
testUserNsPodsManager := &testUserNsPodsManager{
|
||||||
|
mappingFirstID: 65536,
|
||||||
|
mappingLen: 65536,
|
||||||
|
maxPods: 1,
|
||||||
|
}
|
||||||
|
m, err := MakeUserNsManager(testUserNsPodsManager)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// The first pod allocation should succeed.
|
||||||
|
err = m.record(types.UID(fmt.Sprintf("%d", 0)), 65536, 65536)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// The next allocation should fail, as there is no space left.
|
||||||
|
err = m.record(types.UID(fmt.Sprintf("%d", 2)), uint32(2*65536), 65536)
|
||||||
|
assert.Error(t, err)
|
||||||
|
assert.ErrorContains(t, err, "out of range")
|
||||||
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user