mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-21 02:41:25 +00:00
Add Ephemeral Containers to the Kubernetes core API
This commit is contained in:
parent
c7ffc1cd8c
commit
013f049ce0
@ -125,6 +125,14 @@ API rule violation: list_type_missing,k8s.io/api/core/v1,EndpointSubset,NotReady
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,EndpointSubset,Ports
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,Endpoints,Subsets
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,EndpointsList,Items
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,Args
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,Command
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,Env
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,EnvFrom
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,Ports
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,VolumeDevices
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,VolumeMounts
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainers,EphemeralContainers
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,EventList,Items
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,ExecAction,Command
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,FCVolumeSource,TargetWWNs
|
||||
@ -173,6 +181,7 @@ API rule violation: list_type_missing,k8s.io/api/core/v1,PodPortForwardOptions,P
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,PodSecurityContext,SupplementalGroups
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,PodSecurityContext,Sysctls
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,PodSpec,Containers
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,PodSpec,EphemeralContainers
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,PodSpec,HostAliases
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,PodSpec,ImagePullSecrets
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,PodSpec,InitContainers
|
||||
@ -181,6 +190,7 @@ API rule violation: list_type_missing,k8s.io/api/core/v1,PodSpec,Tolerations
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,PodSpec,Volumes
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,PodStatus,Conditions
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,PodStatus,ContainerStatuses
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,PodStatus,EphemeralContainerStatuses
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,PodStatus,InitContainerStatuses
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,PodStatus,PodIPs
|
||||
API rule violation: list_type_missing,k8s.io/api/core/v1,PodTemplateList,Items
|
||||
|
@ -45,6 +45,13 @@ func VisitContainers(podSpec *api.PodSpec, visitor ContainerVisitor) bool {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if utilfeature.DefaultFeatureGate.Enabled(features.EphemeralContainers) {
|
||||
for i := range podSpec.EphemeralContainers {
|
||||
if !visitor((*api.Container)(&podSpec.EphemeralContainers[i].EphemeralContainerCommon)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@ -362,6 +369,9 @@ func dropDisabledFields(
|
||||
return true
|
||||
})
|
||||
}
|
||||
if !utilfeature.DefaultFeatureGate.Enabled(features.EphemeralContainers) {
|
||||
podSpec.EphemeralContainers = nil
|
||||
}
|
||||
|
||||
if (!utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpath) || !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpathEnvExpansion)) && !subpathExprInUse(oldPodSpec) {
|
||||
// drop subpath env expansion from the pod if either of the subpath features is disabled and the old spec did not specify subpath env expansion
|
||||
|
@ -35,6 +35,8 @@ import (
|
||||
)
|
||||
|
||||
func TestVisitContainers(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EphemeralContainers, true)()
|
||||
|
||||
testCases := []struct {
|
||||
description string
|
||||
haveSpec *api.PodSpec
|
||||
@ -79,6 +81,37 @@ func TestVisitContainers(t *testing.T) {
|
||||
},
|
||||
[]string{"i1", "i2", "c1", "c2"},
|
||||
},
|
||||
{
|
||||
"ephemeral containers",
|
||||
&api.PodSpec{
|
||||
Containers: []api.Container{
|
||||
{Name: "c1"},
|
||||
{Name: "c2"},
|
||||
},
|
||||
EphemeralContainers: []api.EphemeralContainer{
|
||||
{EphemeralContainerCommon: api.EphemeralContainerCommon{Name: "e1"}},
|
||||
},
|
||||
},
|
||||
[]string{"c1", "c2", "e1"},
|
||||
},
|
||||
{
|
||||
"all container types",
|
||||
&api.PodSpec{
|
||||
Containers: []api.Container{
|
||||
{Name: "c1"},
|
||||
{Name: "c2"},
|
||||
},
|
||||
InitContainers: []api.Container{
|
||||
{Name: "i1"},
|
||||
{Name: "i2"},
|
||||
},
|
||||
EphemeralContainers: []api.EphemeralContainer{
|
||||
{EphemeralContainerCommon: api.EphemeralContainerCommon{Name: "e1"}},
|
||||
{EphemeralContainerCommon: api.EphemeralContainerCommon{Name: "e2"}},
|
||||
},
|
||||
},
|
||||
[]string{"i1", "i2", "c1", "c2", "e1", "e2"},
|
||||
},
|
||||
{
|
||||
"dropping fields",
|
||||
&api.PodSpec{
|
||||
@ -90,8 +123,12 @@ func TestVisitContainers(t *testing.T) {
|
||||
{Name: "i1"},
|
||||
{Name: "i2", SecurityContext: &api.SecurityContext{}},
|
||||
},
|
||||
EphemeralContainers: []api.EphemeralContainer{
|
||||
{EphemeralContainerCommon: api.EphemeralContainerCommon{Name: "e1"}},
|
||||
{EphemeralContainerCommon: api.EphemeralContainerCommon{Name: "e2", SecurityContext: &api.SecurityContext{}}},
|
||||
},
|
||||
},
|
||||
[]string{"i1", "i2", "c1", "c2"},
|
||||
[]string{"i1", "i2", "c1", "c2", "e1", "e2"},
|
||||
},
|
||||
}
|
||||
|
||||
@ -117,10 +154,17 @@ func TestVisitContainers(t *testing.T) {
|
||||
t.Errorf("VisitContainers() for test case %q: got SecurityContext %#v for init container %v, wanted nil", tc.description, c.SecurityContext, c.Name)
|
||||
}
|
||||
}
|
||||
for _, c := range tc.haveSpec.EphemeralContainers {
|
||||
if c.SecurityContext != nil {
|
||||
t.Errorf("VisitContainers() for test case %q: got SecurityContext %#v for ephemeral container %v, wanted nil", tc.description, c.SecurityContext, c.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPodSecrets(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EphemeralContainers, true)()
|
||||
|
||||
// Stub containing all possible secret references in a pod.
|
||||
// The names of the referenced secrets match struct paths detected by reflection.
|
||||
pod := &api.Pod{
|
||||
@ -195,6 +239,17 @@ func TestPodSecrets(t *testing.T) {
|
||||
CSI: &api.CSIVolumeSource{
|
||||
NodePublishSecretRef: &api.LocalObjectReference{
|
||||
Name: "Spec.Volumes[*].VolumeSource.CSI.NodePublishSecretRef"}}}}},
|
||||
EphemeralContainers: []api.EphemeralContainer{{
|
||||
EphemeralContainerCommon: api.EphemeralContainerCommon{
|
||||
EnvFrom: []api.EnvFromSource{{
|
||||
SecretRef: &api.SecretEnvSource{
|
||||
LocalObjectReference: api.LocalObjectReference{
|
||||
Name: "Spec.EphemeralContainers[*].EphemeralContainerCommon.EnvFrom[*].SecretRef"}}}},
|
||||
Env: []api.EnvVar{{
|
||||
ValueFrom: &api.EnvVarSource{
|
||||
SecretKeyRef: &api.SecretKeySelector{
|
||||
LocalObjectReference: api.LocalObjectReference{
|
||||
Name: "Spec.EphemeralContainers[*].EphemeralContainerCommon.Env[*].ValueFrom.SecretKeyRef"}}}}}}}},
|
||||
},
|
||||
}
|
||||
extractedNames := sets.NewString()
|
||||
@ -212,6 +267,8 @@ func TestPodSecrets(t *testing.T) {
|
||||
expectedSecretPaths := sets.NewString(
|
||||
"Spec.Containers[*].EnvFrom[*].SecretRef",
|
||||
"Spec.Containers[*].Env[*].ValueFrom.SecretKeyRef",
|
||||
"Spec.EphemeralContainers[*].EphemeralContainerCommon.EnvFrom[*].SecretRef",
|
||||
"Spec.EphemeralContainers[*].EphemeralContainerCommon.Env[*].ValueFrom.SecretKeyRef",
|
||||
"Spec.ImagePullSecrets",
|
||||
"Spec.InitContainers[*].EnvFrom[*].SecretRef",
|
||||
"Spec.InitContainers[*].Env[*].ValueFrom.SecretKeyRef",
|
||||
@ -290,6 +347,8 @@ func collectResourcePaths(t *testing.T, resourcename string, path *field.Path, n
|
||||
}
|
||||
|
||||
func TestPodConfigmaps(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EphemeralContainers, true)()
|
||||
|
||||
// Stub containing all possible ConfigMap references in a pod.
|
||||
// The names of the referenced ConfigMaps match struct paths detected by reflection.
|
||||
pod := &api.Pod{
|
||||
@ -304,6 +363,17 @@ func TestPodConfigmaps(t *testing.T) {
|
||||
ConfigMapKeyRef: &api.ConfigMapKeySelector{
|
||||
LocalObjectReference: api.LocalObjectReference{
|
||||
Name: "Spec.Containers[*].Env[*].ValueFrom.ConfigMapKeyRef"}}}}}}},
|
||||
EphemeralContainers: []api.EphemeralContainer{{
|
||||
EphemeralContainerCommon: api.EphemeralContainerCommon{
|
||||
EnvFrom: []api.EnvFromSource{{
|
||||
ConfigMapRef: &api.ConfigMapEnvSource{
|
||||
LocalObjectReference: api.LocalObjectReference{
|
||||
Name: "Spec.EphemeralContainers[*].EphemeralContainerCommon.EnvFrom[*].ConfigMapRef"}}}},
|
||||
Env: []api.EnvVar{{
|
||||
ValueFrom: &api.EnvVarSource{
|
||||
ConfigMapKeyRef: &api.ConfigMapKeySelector{
|
||||
LocalObjectReference: api.LocalObjectReference{
|
||||
Name: "Spec.EphemeralContainers[*].EphemeralContainerCommon.Env[*].ValueFrom.ConfigMapKeyRef"}}}}}}}},
|
||||
InitContainers: []api.Container{{
|
||||
EnvFrom: []api.EnvFromSource{{
|
||||
ConfigMapRef: &api.ConfigMapEnvSource{
|
||||
@ -338,6 +408,8 @@ func TestPodConfigmaps(t *testing.T) {
|
||||
expectedPaths := sets.NewString(
|
||||
"Spec.Containers[*].EnvFrom[*].ConfigMapRef",
|
||||
"Spec.Containers[*].Env[*].ValueFrom.ConfigMapKeyRef",
|
||||
"Spec.EphemeralContainers[*].EphemeralContainerCommon.EnvFrom[*].ConfigMapRef",
|
||||
"Spec.EphemeralContainers[*].EphemeralContainerCommon.Env[*].ValueFrom.ConfigMapKeyRef",
|
||||
"Spec.InitContainers[*].EnvFrom[*].ConfigMapRef",
|
||||
"Spec.InitContainers[*].Env[*].ValueFrom.ConfigMapKeyRef",
|
||||
"Spec.Volumes[*].VolumeSource.Projected.Sources[*].ConfigMap",
|
||||
|
@ -50,6 +50,7 @@ func TestDefaulting(t *testing.T) {
|
||||
{Group: "", Version: "v1", Kind: "ConfigMapList"}: {},
|
||||
{Group: "", Version: "v1", Kind: "Endpoints"}: {},
|
||||
{Group: "", Version: "v1", Kind: "EndpointsList"}: {},
|
||||
{Group: "", Version: "v1", Kind: "EphemeralContainers"}: {},
|
||||
{Group: "", Version: "v1", Kind: "Namespace"}: {},
|
||||
{Group: "", Version: "v1", Kind: "NamespaceList"}: {},
|
||||
{Group: "", Version: "v1", Kind: "Node"}: {},
|
||||
|
@ -23,6 +23,8 @@ import (
|
||||
"k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
)
|
||||
|
||||
// FindPort locates the container port for the given pod and portName. If the
|
||||
@ -67,6 +69,13 @@ func VisitContainers(podSpec *v1.PodSpec, visitor ContainerVisitor) bool {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if utilfeature.DefaultFeatureGate.Enabled(features.EphemeralContainers) {
|
||||
for i := range podSpec.EphemeralContainers {
|
||||
if !visitor((*v1.Container)(&podSpec.EphemeralContainers[i].EphemeralContainerCommon)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
|
@ -28,6 +28,9 @@ import (
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apimachinery/pkg/util/validation/field"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
)
|
||||
|
||||
func TestFindPort(t *testing.T) {
|
||||
@ -199,6 +202,8 @@ func TestFindPort(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestVisitContainers(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EphemeralContainers, true)()
|
||||
|
||||
testCases := []struct {
|
||||
description string
|
||||
haveSpec *v1.PodSpec
|
||||
@ -243,6 +248,37 @@ func TestVisitContainers(t *testing.T) {
|
||||
},
|
||||
[]string{"i1", "i2", "c1", "c2"},
|
||||
},
|
||||
{
|
||||
"ephemeral containers",
|
||||
&v1.PodSpec{
|
||||
Containers: []v1.Container{
|
||||
{Name: "c1"},
|
||||
{Name: "c2"},
|
||||
},
|
||||
EphemeralContainers: []v1.EphemeralContainer{
|
||||
{EphemeralContainerCommon: v1.EphemeralContainerCommon{Name: "e1"}},
|
||||
},
|
||||
},
|
||||
[]string{"c1", "c2", "e1"},
|
||||
},
|
||||
{
|
||||
"all container types",
|
||||
&v1.PodSpec{
|
||||
Containers: []v1.Container{
|
||||
{Name: "c1"},
|
||||
{Name: "c2"},
|
||||
},
|
||||
InitContainers: []v1.Container{
|
||||
{Name: "i1"},
|
||||
{Name: "i2"},
|
||||
},
|
||||
EphemeralContainers: []v1.EphemeralContainer{
|
||||
{EphemeralContainerCommon: v1.EphemeralContainerCommon{Name: "e1"}},
|
||||
{EphemeralContainerCommon: v1.EphemeralContainerCommon{Name: "e2"}},
|
||||
},
|
||||
},
|
||||
[]string{"i1", "i2", "c1", "c2", "e1", "e2"},
|
||||
},
|
||||
{
|
||||
"dropping fields",
|
||||
&v1.PodSpec{
|
||||
@ -254,8 +290,12 @@ func TestVisitContainers(t *testing.T) {
|
||||
{Name: "i1"},
|
||||
{Name: "i2", SecurityContext: &v1.SecurityContext{}},
|
||||
},
|
||||
EphemeralContainers: []v1.EphemeralContainer{
|
||||
{EphemeralContainerCommon: v1.EphemeralContainerCommon{Name: "e1"}},
|
||||
{EphemeralContainerCommon: v1.EphemeralContainerCommon{Name: "e2", SecurityContext: &v1.SecurityContext{}}},
|
||||
},
|
||||
},
|
||||
[]string{"i1", "i2", "c1", "c2"},
|
||||
[]string{"i1", "i2", "c1", "c2", "e1", "e2"},
|
||||
},
|
||||
}
|
||||
|
||||
@ -281,10 +321,17 @@ func TestVisitContainers(t *testing.T) {
|
||||
t.Errorf("VisitContainers() for test case %q: got SecurityContext %#v for init container %v, wanted nil", tc.description, c.SecurityContext, c.Name)
|
||||
}
|
||||
}
|
||||
for _, c := range tc.haveSpec.EphemeralContainers {
|
||||
if c.SecurityContext != nil {
|
||||
t.Errorf("VisitContainers() for test case %q: got SecurityContext %#v for ephemeral container %v, wanted nil", tc.description, c.SecurityContext, c.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPodSecrets(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EphemeralContainers, true)()
|
||||
|
||||
// Stub containing all possible secret references in a pod.
|
||||
// The names of the referenced secrets match struct paths detected by reflection.
|
||||
pod := &v1.Pod{
|
||||
@ -359,6 +406,17 @@ func TestPodSecrets(t *testing.T) {
|
||||
CSI: &v1.CSIVolumeSource{
|
||||
NodePublishSecretRef: &v1.LocalObjectReference{
|
||||
Name: "Spec.Volumes[*].VolumeSource.CSI.NodePublishSecretRef"}}}}},
|
||||
EphemeralContainers: []v1.EphemeralContainer{{
|
||||
EphemeralContainerCommon: v1.EphemeralContainerCommon{
|
||||
EnvFrom: []v1.EnvFromSource{{
|
||||
SecretRef: &v1.SecretEnvSource{
|
||||
LocalObjectReference: v1.LocalObjectReference{
|
||||
Name: "Spec.EphemeralContainers[*].EphemeralContainerCommon.EnvFrom[*].SecretRef"}}}},
|
||||
Env: []v1.EnvVar{{
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
SecretKeyRef: &v1.SecretKeySelector{
|
||||
LocalObjectReference: v1.LocalObjectReference{
|
||||
Name: "Spec.EphemeralContainers[*].EphemeralContainerCommon.Env[*].ValueFrom.SecretKeyRef"}}}}}}}},
|
||||
},
|
||||
}
|
||||
extractedNames := sets.NewString()
|
||||
@ -376,6 +434,8 @@ func TestPodSecrets(t *testing.T) {
|
||||
expectedSecretPaths := sets.NewString(
|
||||
"Spec.Containers[*].EnvFrom[*].SecretRef",
|
||||
"Spec.Containers[*].Env[*].ValueFrom.SecretKeyRef",
|
||||
"Spec.EphemeralContainers[*].EphemeralContainerCommon.EnvFrom[*].SecretRef",
|
||||
"Spec.EphemeralContainers[*].EphemeralContainerCommon.Env[*].ValueFrom.SecretKeyRef",
|
||||
"Spec.ImagePullSecrets",
|
||||
"Spec.InitContainers[*].EnvFrom[*].SecretRef",
|
||||
"Spec.InitContainers[*].Env[*].ValueFrom.SecretKeyRef",
|
||||
@ -454,6 +514,8 @@ func collectResourcePaths(t *testing.T, resourcename string, path *field.Path, n
|
||||
}
|
||||
|
||||
func TestPodConfigmaps(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EphemeralContainers, true)()
|
||||
|
||||
// Stub containing all possible ConfigMap references in a pod.
|
||||
// The names of the referenced ConfigMaps match struct paths detected by reflection.
|
||||
pod := &v1.Pod{
|
||||
@ -468,6 +530,17 @@ func TestPodConfigmaps(t *testing.T) {
|
||||
ConfigMapKeyRef: &v1.ConfigMapKeySelector{
|
||||
LocalObjectReference: v1.LocalObjectReference{
|
||||
Name: "Spec.Containers[*].Env[*].ValueFrom.ConfigMapKeyRef"}}}}}}},
|
||||
EphemeralContainers: []v1.EphemeralContainer{{
|
||||
EphemeralContainerCommon: v1.EphemeralContainerCommon{
|
||||
EnvFrom: []v1.EnvFromSource{{
|
||||
ConfigMapRef: &v1.ConfigMapEnvSource{
|
||||
LocalObjectReference: v1.LocalObjectReference{
|
||||
Name: "Spec.EphemeralContainers[*].EphemeralContainerCommon.EnvFrom[*].ConfigMapRef"}}}},
|
||||
Env: []v1.EnvVar{{
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
ConfigMapKeyRef: &v1.ConfigMapKeySelector{
|
||||
LocalObjectReference: v1.LocalObjectReference{
|
||||
Name: "Spec.EphemeralContainers[*].EphemeralContainerCommon.Env[*].ValueFrom.ConfigMapKeyRef"}}}}}}}},
|
||||
InitContainers: []v1.Container{{
|
||||
EnvFrom: []v1.EnvFromSource{{
|
||||
ConfigMapRef: &v1.ConfigMapEnvSource{
|
||||
@ -502,6 +575,8 @@ func TestPodConfigmaps(t *testing.T) {
|
||||
expectedPaths := sets.NewString(
|
||||
"Spec.Containers[*].EnvFrom[*].ConfigMapRef",
|
||||
"Spec.Containers[*].Env[*].ValueFrom.ConfigMapKeyRef",
|
||||
"Spec.EphemeralContainers[*].EphemeralContainerCommon.EnvFrom[*].ConfigMapRef",
|
||||
"Spec.EphemeralContainers[*].EphemeralContainerCommon.Env[*].ValueFrom.ConfigMapKeyRef",
|
||||
"Spec.InitContainers[*].EnvFrom[*].ConfigMapRef",
|
||||
"Spec.InitContainers[*].Env[*].ValueFrom.ConfigMapKeyRef",
|
||||
"Spec.Volumes[*].VolumeSource.Projected.Sources[*].ConfigMap",
|
||||
|
@ -26,8 +26,11 @@ import (
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/apimachinery/pkg/util/validation/field"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||
"k8s.io/kubernetes/pkg/apis/apps"
|
||||
api "k8s.io/kubernetes/pkg/apis/core"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
)
|
||||
|
||||
func TestValidateStatefulSet(t *testing.T) {
|
||||
@ -1776,6 +1779,8 @@ func TestValidateDaemonSetUpdate(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestValidateDaemonSet(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EphemeralContainers, true)()
|
||||
|
||||
validSelector := map[string]string{"a": "b"}
|
||||
validPodTemplate := api.PodTemplate{
|
||||
Template: api.PodTemplateSpec{
|
||||
@ -1946,6 +1951,26 @@ func TestValidateDaemonSet(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
"template may not contain ephemeral containers": {
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault},
|
||||
Spec: apps.DaemonSetSpec{
|
||||
Selector: &metav1.LabelSelector{MatchLabels: validSelector},
|
||||
Template: api.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: validSelector,
|
||||
},
|
||||
Spec: api.PodSpec{
|
||||
RestartPolicy: api.RestartPolicyAlways,
|
||||
DNSPolicy: api.DNSClusterFirst,
|
||||
Containers: []api.Container{{Name: "abc", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: api.TerminationMessageReadFile}},
|
||||
EphemeralContainers: []api.EphemeralContainer{{EphemeralContainerCommon: api.EphemeralContainerCommon{Name: "debug", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}}},
|
||||
},
|
||||
},
|
||||
UpdateStrategy: apps.DaemonSetUpdateStrategy{
|
||||
Type: apps.OnDeleteDaemonSetStrategyType,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for k, v := range errorCases {
|
||||
errs := ValidateDaemonSet(&v)
|
||||
@ -2018,6 +2043,8 @@ func validDeployment() *apps.Deployment {
|
||||
}
|
||||
|
||||
func TestValidateDeployment(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EphemeralContainers, true)()
|
||||
|
||||
successCases := []*apps.Deployment{
|
||||
validDeployment(),
|
||||
}
|
||||
@ -2103,6 +2130,17 @@ func TestValidateDeployment(t *testing.T) {
|
||||
invalidProgressDeadlineDeployment.Spec.MinReadySeconds = seconds
|
||||
errorCases["must be greater than minReadySeconds"] = invalidProgressDeadlineDeployment
|
||||
|
||||
// Must not have ephemeral containers
|
||||
invalidEphemeralContainersDeployment := validDeployment()
|
||||
invalidEphemeralContainersDeployment.Spec.Template.Spec.EphemeralContainers = []api.EphemeralContainer{{
|
||||
EphemeralContainerCommon: api.EphemeralContainerCommon{
|
||||
Name: "ec",
|
||||
Image: "image",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File"},
|
||||
}}
|
||||
errorCases["ephemeral containers not allowed"] = invalidEphemeralContainersDeployment
|
||||
|
||||
for k, v := range errorCases {
|
||||
errs := ValidateDeployment(v)
|
||||
if len(errs) == 0 {
|
||||
|
@ -20,7 +20,9 @@ import (
|
||||
"fmt"
|
||||
|
||||
"k8s.io/apimachinery/pkg/util/validation/field"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
api "k8s.io/kubernetes/pkg/apis/core"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
"k8s.io/kubernetes/pkg/fieldpath"
|
||||
)
|
||||
|
||||
@ -45,6 +47,14 @@ func VisitContainersWithPath(podSpec *api.PodSpec, visitor ContainerVisitorWithP
|
||||
return false
|
||||
}
|
||||
}
|
||||
if utilfeature.DefaultFeatureGate.Enabled(features.EphemeralContainers) {
|
||||
path = field.NewPath("spec", "ephemeralContainers")
|
||||
for i := range podSpec.EphemeralContainers {
|
||||
if !visitor((*api.Container)(&podSpec.EphemeralContainers[i].EphemeralContainerCommon), path.Index(i)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
|
@ -21,10 +21,15 @@ import (
|
||||
"testing"
|
||||
|
||||
"k8s.io/apimachinery/pkg/util/validation/field"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||
api "k8s.io/kubernetes/pkg/apis/core"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
)
|
||||
|
||||
func TestVisitContainersWithPath(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EphemeralContainers, true)()
|
||||
|
||||
testCases := []struct {
|
||||
description string
|
||||
haveSpec *api.PodSpec
|
||||
@ -69,6 +74,37 @@ func TestVisitContainersWithPath(t *testing.T) {
|
||||
},
|
||||
[]string{"spec.initContainers[0]", "spec.initContainers[1]", "spec.containers[0]", "spec.containers[1]"},
|
||||
},
|
||||
{
|
||||
"ephemeral containers",
|
||||
&api.PodSpec{
|
||||
Containers: []api.Container{
|
||||
{Name: "c1"},
|
||||
{Name: "c2"},
|
||||
},
|
||||
EphemeralContainers: []api.EphemeralContainer{
|
||||
{EphemeralContainerCommon: api.EphemeralContainerCommon{Name: "e1"}},
|
||||
},
|
||||
},
|
||||
[]string{"spec.containers[0]", "spec.containers[1]", "spec.ephemeralContainers[0]"},
|
||||
},
|
||||
{
|
||||
"all container types",
|
||||
&api.PodSpec{
|
||||
Containers: []api.Container{
|
||||
{Name: "c1"},
|
||||
{Name: "c2"},
|
||||
},
|
||||
InitContainers: []api.Container{
|
||||
{Name: "i1"},
|
||||
{Name: "i2"},
|
||||
},
|
||||
EphemeralContainers: []api.EphemeralContainer{
|
||||
{EphemeralContainerCommon: api.EphemeralContainerCommon{Name: "e1"}},
|
||||
{EphemeralContainerCommon: api.EphemeralContainerCommon{Name: "e2"}},
|
||||
},
|
||||
},
|
||||
[]string{"spec.initContainers[0]", "spec.initContainers[1]", "spec.containers[0]", "spec.containers[1]", "spec.ephemeralContainers[0]", "spec.ephemeralContainers[1]"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
|
@ -92,6 +92,7 @@ func addKnownTypes(scheme *runtime.Scheme) error {
|
||||
&RangeAllocation{},
|
||||
&ConfigMap{},
|
||||
&ConfigMapList{},
|
||||
&EphemeralContainers{},
|
||||
)
|
||||
|
||||
return nil
|
||||
|
@ -2591,6 +2591,15 @@ type PodSpec struct {
|
||||
InitContainers []Container
|
||||
// List of containers belonging to the pod.
|
||||
Containers []Container
|
||||
// EphemeralContainers is the list of ephemeral containers that run in this pod. Ephemeral containers
|
||||
// are added to an existing pod as a result of a user-initiated action such as troubleshooting.
|
||||
// This list is read-only in the pod spec. It may not be specified in a create or modified in an
|
||||
// update of a pod or pod template.
|
||||
// To add an ephemeral container use the pod's ephemeralcontainers subresource, which allows update
|
||||
// using the EphemeralContainers kind.
|
||||
// This field is alpha-level and is only honored by servers that enable the EphemeralContainers feature.
|
||||
// +optional
|
||||
EphemeralContainers []EphemeralContainer
|
||||
// +optional
|
||||
RestartPolicy RestartPolicy
|
||||
// Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request.
|
||||
@ -2873,6 +2882,106 @@ type PodIP struct {
|
||||
IP string
|
||||
}
|
||||
|
||||
type EphemeralContainerCommon struct {
|
||||
// Required: This must be a DNS_LABEL. Each container in a pod must
|
||||
// have a unique name.
|
||||
Name string
|
||||
// Required.
|
||||
Image string
|
||||
// Optional: The docker image's entrypoint is used if this is not provided; cannot be updated.
|
||||
// Variable references $(VAR_NAME) are expanded using the container's environment. If a variable
|
||||
// cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax
|
||||
// can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded,
|
||||
// regardless of whether the variable exists or not.
|
||||
// +optional
|
||||
Command []string
|
||||
// Optional: The docker image's cmd is used if this is not provided; cannot be updated.
|
||||
// Variable references $(VAR_NAME) are expanded using the container's environment. If a variable
|
||||
// cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax
|
||||
// can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded,
|
||||
// regardless of whether the variable exists or not.
|
||||
// +optional
|
||||
Args []string
|
||||
// Optional: Defaults to Docker's default.
|
||||
// +optional
|
||||
WorkingDir string
|
||||
// Ports are not allowed for ephemeral containers.
|
||||
// +optional
|
||||
Ports []ContainerPort
|
||||
// List of sources to populate environment variables in the container.
|
||||
// The keys defined within a source must be a C_IDENTIFIER. All invalid keys
|
||||
// will be reported as an event when the container is starting. When a key exists in multiple
|
||||
// sources, the value associated with the last source will take precedence.
|
||||
// Values defined by an Env with a duplicate key will take precedence.
|
||||
// Cannot be updated.
|
||||
// +optional
|
||||
EnvFrom []EnvFromSource
|
||||
// +optional
|
||||
Env []EnvVar
|
||||
// Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources
|
||||
// already allocated to the pod.
|
||||
// +optional
|
||||
Resources ResourceRequirements
|
||||
// +optional
|
||||
VolumeMounts []VolumeMount
|
||||
// volumeDevices is the list of block devices to be used by the container.
|
||||
// This is a beta feature.
|
||||
// +optional
|
||||
VolumeDevices []VolumeDevice
|
||||
// Probes are not allowed for ephemeral containers.
|
||||
// +optional
|
||||
LivenessProbe *Probe
|
||||
// Probes are not allowed for ephemeral containers.
|
||||
// +optional
|
||||
ReadinessProbe *Probe
|
||||
// Lifecycle is not allowed for ephemeral containers.
|
||||
// +optional
|
||||
Lifecycle *Lifecycle
|
||||
// Required.
|
||||
// +optional
|
||||
TerminationMessagePath string
|
||||
// +optional
|
||||
TerminationMessagePolicy TerminationMessagePolicy
|
||||
// Required: Policy for pulling images for this container
|
||||
ImagePullPolicy PullPolicy
|
||||
// SecurityContext is not allowed for ephemeral containers.
|
||||
// +optional
|
||||
SecurityContext *SecurityContext
|
||||
|
||||
// Variables for interactive containers, these have very specialized use-cases (e.g. debugging)
|
||||
// and shouldn't be used for general purpose containers.
|
||||
// +optional
|
||||
Stdin bool
|
||||
// +optional
|
||||
StdinOnce bool
|
||||
// +optional
|
||||
TTY bool
|
||||
}
|
||||
|
||||
// EphemeralContainerCommon converts to Container. All fields must be kept in sync between
|
||||
// these two types.
|
||||
var _ = Container(EphemeralContainerCommon{})
|
||||
|
||||
// An EphemeralContainer is a special type of container which doesn't come with any resource
|
||||
// or scheduling guarantees but can be added to a pod that has already been created. They are
|
||||
// intended for user-initiated activities such as troubleshooting a running pod.
|
||||
// Ephemeral containers will not be restarted when they exit, and they will be killed if the
|
||||
// pod is removed or restarted. If an ephemeral container causes a pod to exceed its resource
|
||||
// allocation, the pod may be evicted.
|
||||
// Ephemeral containers are added via a pod's ephemeralcontainers subresource and will appear
|
||||
// in the pod spec once added.
|
||||
// This is an alpha feature enabled by the EphemeralContainers feature flag.
|
||||
type EphemeralContainer struct {
|
||||
EphemeralContainerCommon
|
||||
|
||||
// If set, the name of the container from PodSpec that this ephemeral container targets.
|
||||
// The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container.
|
||||
// If not set then the ephemeral container is run in whatever namespaces are shared
|
||||
// for the pod. Note that the container runtime must support this feature.
|
||||
// +optional
|
||||
TargetContainerName string
|
||||
}
|
||||
|
||||
// PodStatus represents information about the status of a pod. Status may trail the actual
|
||||
// state of a system.
|
||||
type PodStatus struct {
|
||||
@ -2920,6 +3029,11 @@ type PodStatus struct {
|
||||
// when we have done this.
|
||||
// +optional
|
||||
ContainerStatuses []ContainerStatus
|
||||
|
||||
// Status for any ephemeral containers that running in this pod.
|
||||
// This field is alpha-level and is only honored by servers that enable the EphemeralContainers feature.
|
||||
// +optional
|
||||
EphemeralContainerStatuses []ContainerStatus
|
||||
}
|
||||
|
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
@ -3926,6 +4040,18 @@ type Binding struct {
|
||||
Target ObjectReference
|
||||
}
|
||||
|
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
|
||||
// A list of ephemeral containers used in API operations
|
||||
type EphemeralContainers struct {
|
||||
metav1.TypeMeta
|
||||
// +optional
|
||||
metav1.ObjectMeta
|
||||
|
||||
// The new set of ephemeral containers to use for a pod.
|
||||
EphemeralContainers []EphemeralContainer
|
||||
}
|
||||
|
||||
// Preconditions must be fulfilled before an operation (update, delete, etc.) is carried out.
|
||||
type Preconditions struct {
|
||||
// Specifies the target UID.
|
||||
|
@ -70,27 +70,45 @@ func TestWorkloadDefaults(t *testing.T) {
|
||||
".Spec.Containers[0].TerminationMessagePath": `"/dev/termination-log"`,
|
||||
".Spec.Containers[0].TerminationMessagePolicy": `"File"`,
|
||||
".Spec.DNSPolicy": `"ClusterFirst"`,
|
||||
".Spec.InitContainers[0].Env[0].ValueFrom.FieldRef.APIVersion": `"v1"`,
|
||||
".Spec.InitContainers[0].ImagePullPolicy": `"IfNotPresent"`,
|
||||
".Spec.InitContainers[0].Lifecycle.PostStart.HTTPGet.Path": `"/"`,
|
||||
".Spec.InitContainers[0].Lifecycle.PostStart.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.InitContainers[0].Lifecycle.PreStop.HTTPGet.Path": `"/"`,
|
||||
".Spec.InitContainers[0].Lifecycle.PreStop.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.InitContainers[0].LivenessProbe.FailureThreshold": `3`,
|
||||
".Spec.InitContainers[0].LivenessProbe.Handler.HTTPGet.Path": `"/"`,
|
||||
".Spec.InitContainers[0].LivenessProbe.Handler.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.InitContainers[0].LivenessProbe.PeriodSeconds": `10`,
|
||||
".Spec.InitContainers[0].LivenessProbe.SuccessThreshold": `1`,
|
||||
".Spec.InitContainers[0].LivenessProbe.TimeoutSeconds": `1`,
|
||||
".Spec.InitContainers[0].Ports[0].Protocol": `"TCP"`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.FailureThreshold": `3`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.Handler.HTTPGet.Path": `"/"`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.Handler.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.PeriodSeconds": `10`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.SuccessThreshold": `1`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.TimeoutSeconds": `1`,
|
||||
".Spec.InitContainers[0].TerminationMessagePath": `"/dev/termination-log"`,
|
||||
".Spec.InitContainers[0].TerminationMessagePolicy": `"File"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.Env[0].ValueFrom.FieldRef.APIVersion": `"v1"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.Lifecycle.PostStart.HTTPGet.Path": `"/"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.Lifecycle.PostStart.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.Lifecycle.PreStop.HTTPGet.Path": `"/"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.Lifecycle.PreStop.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.LivenessProbe.FailureThreshold": "3",
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.LivenessProbe.Handler.HTTPGet.Path": `"/"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.LivenessProbe.Handler.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.LivenessProbe.PeriodSeconds": "10",
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.LivenessProbe.SuccessThreshold": "1",
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.LivenessProbe.TimeoutSeconds": "1",
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.Ports[0].Protocol": `"TCP"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.ReadinessProbe.FailureThreshold": "3",
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.ReadinessProbe.Handler.HTTPGet.Path": `"/"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.ReadinessProbe.Handler.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.ReadinessProbe.PeriodSeconds": "10",
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.ReadinessProbe.SuccessThreshold": "1",
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.ReadinessProbe.TimeoutSeconds": "1",
|
||||
".Spec.InitContainers[0].Env[0].ValueFrom.FieldRef.APIVersion": `"v1"`,
|
||||
".Spec.InitContainers[0].ImagePullPolicy": `"IfNotPresent"`,
|
||||
".Spec.InitContainers[0].Lifecycle.PostStart.HTTPGet.Path": `"/"`,
|
||||
".Spec.InitContainers[0].Lifecycle.PostStart.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.InitContainers[0].Lifecycle.PreStop.HTTPGet.Path": `"/"`,
|
||||
".Spec.InitContainers[0].Lifecycle.PreStop.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.InitContainers[0].LivenessProbe.FailureThreshold": `3`,
|
||||
".Spec.InitContainers[0].LivenessProbe.Handler.HTTPGet.Path": `"/"`,
|
||||
".Spec.InitContainers[0].LivenessProbe.Handler.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.InitContainers[0].LivenessProbe.PeriodSeconds": `10`,
|
||||
".Spec.InitContainers[0].LivenessProbe.SuccessThreshold": `1`,
|
||||
".Spec.InitContainers[0].LivenessProbe.TimeoutSeconds": `1`,
|
||||
".Spec.InitContainers[0].Ports[0].Protocol": `"TCP"`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.FailureThreshold": `3`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.Handler.HTTPGet.Path": `"/"`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.Handler.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.PeriodSeconds": `10`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.SuccessThreshold": `1`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.TimeoutSeconds": `1`,
|
||||
".Spec.InitContainers[0].TerminationMessagePath": `"/dev/termination-log"`,
|
||||
".Spec.InitContainers[0].TerminationMessagePolicy": `"File"`,
|
||||
".Spec.RestartPolicy": `"Always"`,
|
||||
".Spec.SchedulerName": `"default-scheduler"`,
|
||||
".Spec.SecurityContext": `{}`,
|
||||
@ -156,28 +174,46 @@ func TestPodDefaults(t *testing.T) {
|
||||
".Spec.Containers[0].TerminationMessagePolicy": `"File"`,
|
||||
".Spec.DNSPolicy": `"ClusterFirst"`,
|
||||
".Spec.EnableServiceLinks": `true`,
|
||||
".Spec.InitContainers[0].Env[0].ValueFrom.FieldRef.APIVersion": `"v1"`,
|
||||
".Spec.InitContainers[0].ImagePullPolicy": `"IfNotPresent"`,
|
||||
".Spec.InitContainers[0].Lifecycle.PostStart.HTTPGet.Path": `"/"`,
|
||||
".Spec.InitContainers[0].Lifecycle.PostStart.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.InitContainers[0].Lifecycle.PreStop.HTTPGet.Path": `"/"`,
|
||||
".Spec.InitContainers[0].Lifecycle.PreStop.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.InitContainers[0].LivenessProbe.FailureThreshold": `3`,
|
||||
".Spec.InitContainers[0].LivenessProbe.Handler.HTTPGet.Path": `"/"`,
|
||||
".Spec.InitContainers[0].LivenessProbe.Handler.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.InitContainers[0].LivenessProbe.PeriodSeconds": `10`,
|
||||
".Spec.InitContainers[0].LivenessProbe.SuccessThreshold": `1`,
|
||||
".Spec.InitContainers[0].LivenessProbe.TimeoutSeconds": `1`,
|
||||
".Spec.InitContainers[0].Ports[0].Protocol": `"TCP"`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.FailureThreshold": `3`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.Handler.HTTPGet.Path": `"/"`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.Handler.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.PeriodSeconds": `10`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.SuccessThreshold": `1`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.TimeoutSeconds": `1`,
|
||||
".Spec.InitContainers[0].Resources.Requests": `{"":"0"}`, // this gets defaulted from the limits field
|
||||
".Spec.InitContainers[0].TerminationMessagePath": `"/dev/termination-log"`,
|
||||
".Spec.InitContainers[0].TerminationMessagePolicy": `"File"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.Env[0].ValueFrom.FieldRef.APIVersion": `"v1"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.Lifecycle.PostStart.HTTPGet.Path": `"/"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.Lifecycle.PostStart.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.Lifecycle.PreStop.HTTPGet.Path": `"/"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.Lifecycle.PreStop.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.LivenessProbe.FailureThreshold": "3",
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.LivenessProbe.Handler.HTTPGet.Path": `"/"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.LivenessProbe.Handler.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.LivenessProbe.PeriodSeconds": "10",
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.LivenessProbe.SuccessThreshold": "1",
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.LivenessProbe.TimeoutSeconds": "1",
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.Ports[0].Protocol": `"TCP"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.ReadinessProbe.FailureThreshold": "3",
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.ReadinessProbe.Handler.HTTPGet.Path": `"/"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.ReadinessProbe.Handler.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.ReadinessProbe.PeriodSeconds": "10",
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.ReadinessProbe.SuccessThreshold": "1",
|
||||
".Spec.EphemeralContainers[0].EphemeralContainerCommon.ReadinessProbe.TimeoutSeconds": "1",
|
||||
".Spec.InitContainers[0].Env[0].ValueFrom.FieldRef.APIVersion": `"v1"`,
|
||||
".Spec.InitContainers[0].ImagePullPolicy": `"IfNotPresent"`,
|
||||
".Spec.InitContainers[0].Lifecycle.PostStart.HTTPGet.Path": `"/"`,
|
||||
".Spec.InitContainers[0].Lifecycle.PostStart.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.InitContainers[0].Lifecycle.PreStop.HTTPGet.Path": `"/"`,
|
||||
".Spec.InitContainers[0].Lifecycle.PreStop.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.InitContainers[0].LivenessProbe.FailureThreshold": `3`,
|
||||
".Spec.InitContainers[0].LivenessProbe.Handler.HTTPGet.Path": `"/"`,
|
||||
".Spec.InitContainers[0].LivenessProbe.Handler.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.InitContainers[0].LivenessProbe.PeriodSeconds": `10`,
|
||||
".Spec.InitContainers[0].LivenessProbe.SuccessThreshold": `1`,
|
||||
".Spec.InitContainers[0].LivenessProbe.TimeoutSeconds": `1`,
|
||||
".Spec.InitContainers[0].Ports[0].Protocol": `"TCP"`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.FailureThreshold": `3`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.Handler.HTTPGet.Path": `"/"`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.Handler.HTTPGet.Scheme": `"HTTP"`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.PeriodSeconds": `10`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.SuccessThreshold": `1`,
|
||||
".Spec.InitContainers[0].ReadinessProbe.TimeoutSeconds": `1`,
|
||||
".Spec.InitContainers[0].Resources.Requests": `{"":"0"}`, // this gets defaulted from the limits field
|
||||
".Spec.InitContainers[0].TerminationMessagePath": `"/dev/termination-log"`,
|
||||
".Spec.InitContainers[0].TerminationMessagePolicy": `"File"`,
|
||||
".Spec.RestartPolicy": `"Always"`,
|
||||
".Spec.SchedulerName": `"default-scheduler"`,
|
||||
".Spec.SecurityContext": `{}`,
|
||||
|
@ -26,6 +26,8 @@ import (
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"k8s.io/klog"
|
||||
|
||||
@ -73,6 +75,23 @@ var iscsiInitiatorIqnRegex = regexp.MustCompile(`iqn\.\d{4}-\d{2}\.([[:alnum:]-.
|
||||
var iscsiInitiatorEuiRegex = regexp.MustCompile(`^eui.[[:alnum:]]{16}$`)
|
||||
var iscsiInitiatorNaaRegex = regexp.MustCompile(`^naa.[[:alnum:]]{32}$`)
|
||||
|
||||
var allowedEphemeralContainerFields = map[string]bool{
|
||||
"Name": true,
|
||||
"Image": true,
|
||||
"Command": true,
|
||||
"Args": true,
|
||||
"WorkingDir": true,
|
||||
"EnvFrom": true,
|
||||
"Env": true,
|
||||
"VolumeMounts": true,
|
||||
"TerminationMessagePath": true,
|
||||
"TerminationMessagePolicy": true,
|
||||
"ImagePullPolicy": true,
|
||||
"Stdin": true,
|
||||
"StdinOnce": true,
|
||||
"TTY": true,
|
||||
}
|
||||
|
||||
// ValidateHasLabel requires that metav1.ObjectMeta has a Label with key and expectedValue
|
||||
func ValidateHasLabel(meta metav1.ObjectMeta, fldPath *field.Path, key, expectedValue string) field.ErrorList {
|
||||
allErrs := field.ErrorList{}
|
||||
@ -2588,6 +2607,75 @@ func validatePullPolicy(policy core.PullPolicy, fldPath *field.Path) field.Error
|
||||
return allErrors
|
||||
}
|
||||
|
||||
func validateEphemeralContainers(ephemeralContainers []core.EphemeralContainer, containers, initContainers []core.Container, volumes map[string]core.VolumeSource, fldPath *field.Path) field.ErrorList {
|
||||
allErrs := field.ErrorList{}
|
||||
|
||||
if len(ephemeralContainers) == 0 {
|
||||
return allErrs
|
||||
}
|
||||
|
||||
// Return early if EphemeralContainers disabled
|
||||
if !utilfeature.DefaultFeatureGate.Enabled(features.EphemeralContainers) {
|
||||
return append(allErrs, field.Forbidden(fldPath, "disabled by EphemeralContainers feature-gate"))
|
||||
}
|
||||
|
||||
allNames := sets.String{}
|
||||
for _, c := range containers {
|
||||
allNames.Insert(c.Name)
|
||||
}
|
||||
for _, c := range initContainers {
|
||||
allNames.Insert(c.Name)
|
||||
}
|
||||
|
||||
for i, ec := range ephemeralContainers {
|
||||
idxPath := fldPath.Index(i)
|
||||
|
||||
if ec.TargetContainerName != "" && !allNames.Has(ec.TargetContainerName) {
|
||||
allErrs = append(allErrs, field.NotFound(idxPath.Child("targetContainerName"), ec.TargetContainerName))
|
||||
}
|
||||
|
||||
if ec.Name == "" {
|
||||
allErrs = append(allErrs, field.Required(idxPath, "ephemeralContainer requires a name"))
|
||||
continue
|
||||
}
|
||||
|
||||
// Using validateContainers() here isn't ideal because it adds an index to the error message that
|
||||
// doesn't really exist for EphemeralContainers (i.e. ephemeralContainers[0].spec[0].name instead
|
||||
// of ephemeralContainers[0].spec.name)
|
||||
// TODO(verb): factor a validateContainer() out of validateContainers() to be used here
|
||||
c := core.Container(ec.EphemeralContainerCommon)
|
||||
allErrs = append(allErrs, validateContainers([]core.Container{c}, false, volumes, idxPath)...)
|
||||
// EphemeralContainers don't require the backwards-compatibility distinction between pod/podTemplate validation
|
||||
allErrs = append(allErrs, validateContainersOnlyForPod([]core.Container{c}, idxPath)...)
|
||||
|
||||
if allNames.Has(ec.Name) {
|
||||
allErrs = append(allErrs, field.Duplicate(idxPath.Child("name"), ec.Name))
|
||||
} else {
|
||||
allNames.Insert(ec.Name)
|
||||
}
|
||||
|
||||
// Ephemeral Containers should not be relied upon for fundamental pod services, so fields such as
|
||||
// Lifecycle, probes, resources and ports should be disallowed. This is implemented as a whitelist
|
||||
// so that new fields will be given consideration prior to inclusion in Ephemeral Containers.
|
||||
specType, specValue := reflect.TypeOf(ec.EphemeralContainerCommon), reflect.ValueOf(ec.EphemeralContainerCommon)
|
||||
for i := 0; i < specType.NumField(); i++ {
|
||||
f := specType.Field(i)
|
||||
if allowedEphemeralContainerFields[f.Name] {
|
||||
continue
|
||||
}
|
||||
|
||||
// Compare the value of this field to its zero value to determine if it has been set
|
||||
if !reflect.DeepEqual(specValue.Field(i).Interface(), reflect.Zero(f.Type).Interface()) {
|
||||
r, n := utf8.DecodeRuneInString(f.Name)
|
||||
lcName := string(unicode.ToLower(r)) + f.Name[n:]
|
||||
allErrs = append(allErrs, field.Forbidden(idxPath.Child(lcName), "cannot be set for an Ephemeral Container"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return allErrs
|
||||
}
|
||||
|
||||
func validateInitContainers(containers, otherContainers []core.Container, deviceVolumes map[string]core.VolumeSource, fldPath *field.Path) field.ErrorList {
|
||||
var allErrs field.ErrorList
|
||||
if len(containers) > 0 {
|
||||
@ -3083,6 +3171,7 @@ func ValidatePodSpec(spec *core.PodSpec, fldPath *field.Path) field.ErrorList {
|
||||
allErrs = append(allErrs, vErrs...)
|
||||
allErrs = append(allErrs, validateContainers(spec.Containers, false, vols, fldPath.Child("containers"))...)
|
||||
allErrs = append(allErrs, validateInitContainers(spec.InitContainers, spec.Containers, vols, fldPath.Child("initContainers"))...)
|
||||
allErrs = append(allErrs, validateEphemeralContainers(spec.EphemeralContainers, spec.Containers, spec.InitContainers, vols, fldPath.Child("ephemeralContainers"))...)
|
||||
allErrs = append(allErrs, validateRestartPolicy(&spec.RestartPolicy, fldPath.Child("restartPolicy"))...)
|
||||
allErrs = append(allErrs, validateDNSPolicy(&spec.DNSPolicy, fldPath.Child("dnsPolicy"))...)
|
||||
allErrs = append(allErrs, unversionedvalidation.ValidateLabels(spec.NodeSelector, fldPath.Child("nodeSelector"))...)
|
||||
@ -3584,6 +3673,19 @@ func ValidateContainerUpdates(newContainers, oldContainers []core.Container, fld
|
||||
return allErrs, false
|
||||
}
|
||||
|
||||
// ValidatePodCreate validates a pod in the context of its initial create
|
||||
func ValidatePodCreate(pod *core.Pod) field.ErrorList {
|
||||
allErrs := ValidatePod(pod)
|
||||
|
||||
fldPath := field.NewPath("spec")
|
||||
// EphemeralContainers can only be set on update using the ephemeralcontainers subresource
|
||||
if len(pod.Spec.EphemeralContainers) > 0 {
|
||||
allErrs = append(allErrs, field.Forbidden(fldPath.Child("ephemeralContainers"), "cannot be set on create"))
|
||||
}
|
||||
|
||||
return allErrs
|
||||
}
|
||||
|
||||
// ValidatePodUpdate tests to see if the update is legal for an end user to make. newPod is updated with fields
|
||||
// that cannot be changed.
|
||||
func ValidatePodUpdate(newPod, oldPod *core.Pod) field.ErrorList {
|
||||
@ -3735,6 +3837,35 @@ func validatePodConditions(conditions []core.PodCondition, fldPath *field.Path)
|
||||
return allErrs
|
||||
}
|
||||
|
||||
// ValidatePodEphemeralContainersUpdate tests that a user update to EphemeralContainers is valid.
|
||||
// newPod and oldPod must only differ in their EphemeralContainers.
|
||||
func ValidatePodEphemeralContainersUpdate(newPod, oldPod *core.Pod) field.ErrorList {
|
||||
spec := newPod.Spec
|
||||
specPath := field.NewPath("spec").Child("ephemeralContainers")
|
||||
|
||||
vols := make(map[string]core.VolumeSource)
|
||||
for _, vol := range spec.Volumes {
|
||||
vols[vol.Name] = vol.VolumeSource
|
||||
}
|
||||
allErrs := validateEphemeralContainers(spec.EphemeralContainers, spec.Containers, spec.InitContainers, vols, specPath)
|
||||
|
||||
// Existing EphemeralContainers may not be changed. Order isn't preserved by patch, so check each individually.
|
||||
newContainerIndex := make(map[string]*core.EphemeralContainer)
|
||||
for i := range newPod.Spec.EphemeralContainers {
|
||||
newContainerIndex[newPod.Spec.EphemeralContainers[i].Name] = &newPod.Spec.EphemeralContainers[i]
|
||||
}
|
||||
for _, old := range oldPod.Spec.EphemeralContainers {
|
||||
if new, ok := newContainerIndex[old.Name]; !ok {
|
||||
allErrs = append(allErrs, field.Forbidden(specPath, fmt.Sprintf("existing ephemeral containers %q may not be removed\n", old.Name)))
|
||||
} else if !apiequality.Semantic.DeepEqual(old, *new) {
|
||||
specDiff := diff.ObjectDiff(old, *new)
|
||||
allErrs = append(allErrs, field.Forbidden(specPath, fmt.Sprintf("existing ephemeral containers %q may not be changed\n%v", old.Name, specDiff)))
|
||||
}
|
||||
}
|
||||
|
||||
return allErrs
|
||||
}
|
||||
|
||||
// ValidatePodBinding tests if required fields in the pod binding are legal.
|
||||
func ValidatePodBinding(binding *core.Binding) field.ErrorList {
|
||||
allErrs := field.ErrorList{}
|
||||
@ -4149,6 +4280,11 @@ func ValidatePodTemplateSpec(spec *core.PodTemplateSpec, fldPath *field.Path) fi
|
||||
allErrs = append(allErrs, ValidateAnnotations(spec.Annotations, fldPath.Child("annotations"))...)
|
||||
allErrs = append(allErrs, ValidatePodSpecificAnnotations(spec.Annotations, &spec.Spec, fldPath.Child("annotations"))...)
|
||||
allErrs = append(allErrs, ValidatePodSpec(&spec.Spec, fldPath.Child("spec"))...)
|
||||
|
||||
if utilfeature.DefaultFeatureGate.Enabled(features.EphemeralContainers) && len(spec.Spec.EphemeralContainers) > 0 {
|
||||
allErrs = append(allErrs, field.Forbidden(fldPath.Child("spec", "ephemeralContainers"), "ephemeral containers not allowed in pod template"))
|
||||
}
|
||||
|
||||
return allErrs
|
||||
}
|
||||
|
||||
|
@ -5527,6 +5527,243 @@ func getResourceLimits(cpu, memory string) core.ResourceList {
|
||||
return res
|
||||
}
|
||||
|
||||
func TestValidateEphemeralContainers(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EphemeralContainers, true)()
|
||||
|
||||
containers := []core.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}}
|
||||
initContainers := []core.Container{{Name: "ictr", Image: "iimage", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}}
|
||||
vols := map[string]core.VolumeSource{"vol": {EmptyDir: &core.EmptyDirVolumeSource{}}}
|
||||
|
||||
// Success Cases
|
||||
for title, ephemeralContainers := range map[string][]core.EphemeralContainer{
|
||||
"Empty Ephemeral Containers": {},
|
||||
"Single Container": {
|
||||
{EphemeralContainerCommon: core.EphemeralContainerCommon{Name: "debug", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}},
|
||||
},
|
||||
"Multiple Containers": {
|
||||
{EphemeralContainerCommon: core.EphemeralContainerCommon{Name: "debug1", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}},
|
||||
{EphemeralContainerCommon: core.EphemeralContainerCommon{Name: "debug2", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}},
|
||||
},
|
||||
"Single Container with Target": {
|
||||
{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{Name: "debug", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"},
|
||||
TargetContainerName: "ctr",
|
||||
},
|
||||
},
|
||||
"All Whitelisted Fields": {
|
||||
{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
|
||||
Name: "debug",
|
||||
Image: "image",
|
||||
Command: []string{"bash"},
|
||||
Args: []string{"bash"},
|
||||
WorkingDir: "/",
|
||||
EnvFrom: []core.EnvFromSource{
|
||||
{
|
||||
ConfigMapRef: &core.ConfigMapEnvSource{
|
||||
LocalObjectReference: core.LocalObjectReference{Name: "dummy"},
|
||||
Optional: &[]bool{true}[0],
|
||||
},
|
||||
},
|
||||
},
|
||||
Env: []core.EnvVar{
|
||||
{Name: "TEST", Value: "TRUE"},
|
||||
},
|
||||
VolumeMounts: []core.VolumeMount{
|
||||
{Name: "vol", MountPath: "/vol"},
|
||||
},
|
||||
TerminationMessagePath: "/dev/termination-log",
|
||||
TerminationMessagePolicy: "File",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
Stdin: true,
|
||||
StdinOnce: true,
|
||||
TTY: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
} {
|
||||
if errs := validateEphemeralContainers(ephemeralContainers, containers, initContainers, vols, field.NewPath("ephemeralContainers")); len(errs) != 0 {
|
||||
t.Errorf("expected success for '%s' but got errors: %v", title, errs)
|
||||
}
|
||||
}
|
||||
|
||||
// Failure Cases
|
||||
tcs := []struct {
|
||||
title string
|
||||
ephemeralContainers []core.EphemeralContainer
|
||||
expectedError field.Error
|
||||
}{
|
||||
|
||||
{
|
||||
"Name Collision with Container.Containers",
|
||||
[]core.EphemeralContainer{
|
||||
{EphemeralContainerCommon: core.EphemeralContainerCommon{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}},
|
||||
{EphemeralContainerCommon: core.EphemeralContainerCommon{Name: "debug1", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}},
|
||||
},
|
||||
field.Error{Type: field.ErrorTypeDuplicate, Field: "ephemeralContainers[0].name"},
|
||||
},
|
||||
{
|
||||
"Name Collision with Container.InitContainers",
|
||||
[]core.EphemeralContainer{
|
||||
{EphemeralContainerCommon: core.EphemeralContainerCommon{Name: "ictr", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}},
|
||||
{EphemeralContainerCommon: core.EphemeralContainerCommon{Name: "debug1", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}},
|
||||
},
|
||||
field.Error{Type: field.ErrorTypeDuplicate, Field: "ephemeralContainers[0].name"},
|
||||
},
|
||||
{
|
||||
"Name Collision with EphemeralContainers",
|
||||
[]core.EphemeralContainer{
|
||||
{EphemeralContainerCommon: core.EphemeralContainerCommon{Name: "debug1", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}},
|
||||
{EphemeralContainerCommon: core.EphemeralContainerCommon{Name: "debug1", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}},
|
||||
},
|
||||
field.Error{Type: field.ErrorTypeDuplicate, Field: "ephemeralContainers[1].name"},
|
||||
},
|
||||
{
|
||||
"empty Container Container",
|
||||
[]core.EphemeralContainer{
|
||||
{EphemeralContainerCommon: core.EphemeralContainerCommon{}},
|
||||
},
|
||||
field.Error{Type: field.ErrorTypeRequired, Field: "ephemeralContainers[0]"},
|
||||
},
|
||||
{
|
||||
"empty Container Name",
|
||||
[]core.EphemeralContainer{
|
||||
{EphemeralContainerCommon: core.EphemeralContainerCommon{Name: "", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}},
|
||||
},
|
||||
field.Error{Type: field.ErrorTypeRequired, Field: "ephemeralContainers[0]"},
|
||||
},
|
||||
{
|
||||
"whitespace padded image name",
|
||||
[]core.EphemeralContainer{
|
||||
{EphemeralContainerCommon: core.EphemeralContainerCommon{Name: "debug", Image: " image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}},
|
||||
},
|
||||
field.Error{Type: field.ErrorTypeInvalid, Field: "ephemeralContainers[0][0].image"},
|
||||
},
|
||||
{
|
||||
"TargetContainerName doesn't exist",
|
||||
[]core.EphemeralContainer{
|
||||
{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{Name: "debug", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"},
|
||||
TargetContainerName: "bogus",
|
||||
},
|
||||
},
|
||||
field.Error{Type: field.ErrorTypeNotFound, Field: "ephemeralContainers[0].targetContainerName"},
|
||||
},
|
||||
{
|
||||
"Container uses non-whitelisted field: Lifecycle",
|
||||
[]core.EphemeralContainer{
|
||||
{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debug",
|
||||
Image: "image",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
Lifecycle: &core.Lifecycle{
|
||||
PreStop: &core.Handler{
|
||||
Exec: &core.ExecAction{Command: []string{"ls", "-l"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
field.Error{Type: field.ErrorTypeForbidden, Field: "ephemeralContainers[0].lifecycle"},
|
||||
},
|
||||
{
|
||||
"Container uses non-whitelisted field: LivenessProbe",
|
||||
[]core.EphemeralContainer{
|
||||
{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debug",
|
||||
Image: "image",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
LivenessProbe: &core.Probe{
|
||||
Handler: core.Handler{
|
||||
TCPSocket: &core.TCPSocketAction{Port: intstr.FromInt(80)},
|
||||
},
|
||||
SuccessThreshold: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
field.Error{Type: field.ErrorTypeForbidden, Field: "ephemeralContainers[0].livenessProbe"},
|
||||
},
|
||||
{
|
||||
"Container uses non-whitelisted field: Ports",
|
||||
[]core.EphemeralContainer{
|
||||
{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debug",
|
||||
Image: "image",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
Ports: []core.ContainerPort{
|
||||
{Protocol: "TCP", ContainerPort: 80},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
field.Error{Type: field.ErrorTypeForbidden, Field: "ephemeralContainers[0].ports"},
|
||||
},
|
||||
{
|
||||
"Container uses non-whitelisted field: ReadinessProbe",
|
||||
[]core.EphemeralContainer{
|
||||
{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debug",
|
||||
Image: "image",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
ReadinessProbe: &core.Probe{
|
||||
Handler: core.Handler{
|
||||
TCPSocket: &core.TCPSocketAction{Port: intstr.FromInt(80)},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
field.Error{Type: field.ErrorTypeForbidden, Field: "ephemeralContainers[0].readinessProbe"},
|
||||
},
|
||||
{
|
||||
"Container uses non-whitelisted field: Resources",
|
||||
[]core.EphemeralContainer{
|
||||
{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debug",
|
||||
Image: "image",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
Resources: core.ResourceRequirements{
|
||||
Limits: core.ResourceList{
|
||||
core.ResourceName(core.ResourceCPU): resource.MustParse("10"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
field.Error{Type: field.ErrorTypeForbidden, Field: "ephemeralContainers[0].resources"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tcs {
|
||||
errs := validateEphemeralContainers(tc.ephemeralContainers, containers, initContainers, vols, field.NewPath("ephemeralContainers"))
|
||||
|
||||
if len(errs) == 0 {
|
||||
t.Errorf("for test %q, expected error but received none", tc.title)
|
||||
} else if len(errs) > 1 {
|
||||
t.Errorf("for test %q, expected 1 error but received %d: %q", tc.title, len(errs), errs)
|
||||
} else {
|
||||
if errs[0].Type != tc.expectedError.Type {
|
||||
t.Errorf("for test %q, expected error type %q but received %q: %q", tc.title, string(tc.expectedError.Type), string(errs[0].Type), errs)
|
||||
}
|
||||
if errs[0].Field != tc.expectedError.Field {
|
||||
t.Errorf("for test %q, expected error for field %q but received error for field %q: %q", tc.title, tc.expectedError.Field, errs[0].Field, errs)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateContainers(t *testing.T) {
|
||||
volumeDevices := make(map[string]core.VolumeSource)
|
||||
capabilities.SetForTests(capabilities.Capabilities{
|
||||
@ -6330,6 +6567,7 @@ func TestValidatePodSpec(t *testing.T) {
|
||||
minGroupID := int64(0)
|
||||
maxGroupID := int64(2147483647)
|
||||
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EphemeralContainers, true)()
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RuntimeClass, true)()
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodOverhead, true)()
|
||||
|
||||
@ -6672,6 +6910,34 @@ func TestValidatePodSpec(t *testing.T) {
|
||||
t.Errorf("expected failure for %q", k)
|
||||
}
|
||||
}
|
||||
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EphemeralContainers, false)()
|
||||
|
||||
featuregatedCases := map[string]core.PodSpec{
|
||||
"disabled by EphemeralContainers feature-gate": {
|
||||
Containers: []core.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}},
|
||||
EphemeralContainers: []core.EphemeralContainer{
|
||||
{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debug",
|
||||
Image: "image",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
},
|
||||
},
|
||||
RestartPolicy: core.RestartPolicyAlways,
|
||||
DNSPolicy: core.DNSClusterFirst,
|
||||
},
|
||||
}
|
||||
for expectedErr, spec := range featuregatedCases {
|
||||
errs := ValidatePodSpec(&spec, field.NewPath("field"))
|
||||
if len(errs) == 0 {
|
||||
t.Errorf("expected failure due to gated feature: %s\n%+v", expectedErr, spec)
|
||||
} else if actualErr := errs.ToAggregate().Error(); !strings.Contains(actualErr, expectedErr) {
|
||||
t.Errorf("unexpected error message for gated feature. Expected error: %s\nActual error: %s", expectedErr, actualErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func extendPodSpecwithTolerations(in core.PodSpec, tolerations []core.Toleration) core.PodSpec {
|
||||
@ -8321,6 +8587,27 @@ func TestValidatePodUpdate(t *testing.T) {
|
||||
"spec.initContainers[0].image",
|
||||
"init container image change to empty",
|
||||
},
|
||||
{
|
||||
core.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
|
||||
Spec: core.PodSpec{
|
||||
EphemeralContainers: []core.EphemeralContainer{
|
||||
{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "ephemeral",
|
||||
Image: "busybox",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
core.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
|
||||
Spec: core.PodSpec{},
|
||||
},
|
||||
"Forbidden: pod updates may not change fields other than",
|
||||
"ephemeralContainer changes are not allowed via normal pod update",
|
||||
},
|
||||
{
|
||||
core.Pod{
|
||||
Spec: core.PodSpec{},
|
||||
@ -8902,6 +9189,272 @@ func makeValidService() core.Service {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatePodEphemeralContainersUpdate(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EphemeralContainers, true)()
|
||||
|
||||
tests := []struct {
|
||||
new []core.EphemeralContainer
|
||||
old []core.EphemeralContainer
|
||||
err string
|
||||
test string
|
||||
}{
|
||||
{[]core.EphemeralContainer{}, []core.EphemeralContainer{}, "", "nothing"},
|
||||
{
|
||||
[]core.EphemeralContainer{{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}, {
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger2",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}},
|
||||
[]core.EphemeralContainer{{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}, {
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger2",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}},
|
||||
"",
|
||||
"No change in Ephemeral Containers",
|
||||
},
|
||||
{
|
||||
[]core.EphemeralContainer{{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}, {
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger2",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}},
|
||||
[]core.EphemeralContainer{{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger2",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}, {
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}},
|
||||
"",
|
||||
"Ephemeral Container list order changes",
|
||||
},
|
||||
{
|
||||
[]core.EphemeralContainer{{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}},
|
||||
[]core.EphemeralContainer{},
|
||||
"",
|
||||
"Add an Ephemeral Container",
|
||||
},
|
||||
{
|
||||
[]core.EphemeralContainer{{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger1",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}, {
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger2",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}},
|
||||
[]core.EphemeralContainer{},
|
||||
"",
|
||||
"Add two Ephemeral Containers",
|
||||
},
|
||||
{
|
||||
[]core.EphemeralContainer{{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}, {
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger2",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}},
|
||||
[]core.EphemeralContainer{{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}},
|
||||
"",
|
||||
"Add to an existing Ephemeral Containers",
|
||||
},
|
||||
{
|
||||
[]core.EphemeralContainer{{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger3",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}, {
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger2",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}, {
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}},
|
||||
[]core.EphemeralContainer{{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}, {
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger2",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}},
|
||||
"",
|
||||
"Add to an existing Ephemeral Containers, list order changes",
|
||||
},
|
||||
{
|
||||
[]core.EphemeralContainer{},
|
||||
[]core.EphemeralContainer{{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}},
|
||||
"may not be removed",
|
||||
"Remove an Ephemeral Container",
|
||||
},
|
||||
{
|
||||
[]core.EphemeralContainer{{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "firstone",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}},
|
||||
[]core.EphemeralContainer{{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "thentheother",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}},
|
||||
"may not be removed",
|
||||
"Replace an Ephemeral Container",
|
||||
},
|
||||
{
|
||||
[]core.EphemeralContainer{{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger1",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}, {
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger2",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}},
|
||||
[]core.EphemeralContainer{{
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger1",
|
||||
Image: "debian",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}, {
|
||||
EphemeralContainerCommon: core.EphemeralContainerCommon{
|
||||
Name: "debugger2",
|
||||
Image: "busybox",
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
TerminationMessagePolicy: "File",
|
||||
},
|
||||
}},
|
||||
"may not be changed",
|
||||
"Change an Ephemeral Containers",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
new := core.Pod{Spec: core.PodSpec{EphemeralContainers: test.new}}
|
||||
old := core.Pod{Spec: core.PodSpec{EphemeralContainers: test.old}}
|
||||
errs := ValidatePodEphemeralContainersUpdate(&new, &old)
|
||||
if test.err == "" {
|
||||
if len(errs) != 0 {
|
||||
t.Errorf("unexpected invalid: %s (%+v)\nA: %+v\nB: %+v", test.test, errs, test.new, test.old)
|
||||
}
|
||||
} else {
|
||||
if len(errs) == 0 {
|
||||
t.Errorf("unexpected valid: %s\nA: %+v\nB: %+v", test.test, test.new, test.old)
|
||||
} else if actualErr := errs.ToAggregate().Error(); !strings.Contains(actualErr, test.err) {
|
||||
t.Errorf("unexpected error message: %s\nExpected error: %s\nActual error: %s", test.test, test.err, actualErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateService(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SCTPSupport, true)()
|
||||
|
||||
@ -10006,6 +10559,8 @@ func TestValidateReplicationControllerUpdate(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestValidateReplicationController(t *testing.T) {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EphemeralContainers, true)()
|
||||
|
||||
validSelector := map[string]string{"a": "b"}
|
||||
validPodTemplate := core.PodTemplate{
|
||||
Template: core.PodTemplateSpec{
|
||||
@ -10199,6 +10754,24 @@ func TestValidateReplicationController(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
"template may not contain ephemeral containers": {
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault},
|
||||
Spec: core.ReplicationControllerSpec{
|
||||
Replicas: 1,
|
||||
Selector: validSelector,
|
||||
Template: &core.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: validSelector,
|
||||
},
|
||||
Spec: core.PodSpec{
|
||||
RestartPolicy: core.RestartPolicyAlways,
|
||||
DNSPolicy: core.DNSClusterFirst,
|
||||
Containers: []core.Container{{Name: "abc", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}},
|
||||
EphemeralContainers: []core.EphemeralContainer{{EphemeralContainerCommon: core.EphemeralContainerCommon{Name: "debug", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for k, v := range errorCases {
|
||||
errs := ValidateReplicationController(&v)
|
||||
|
@ -31,10 +31,12 @@ import (
|
||||
"k8s.io/apiserver/pkg/storage"
|
||||
storeerr "k8s.io/apiserver/pkg/storage/errors"
|
||||
"k8s.io/apiserver/pkg/util/dryrun"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
policyclient "k8s.io/client-go/kubernetes/typed/policy/v1beta1"
|
||||
podutil "k8s.io/kubernetes/pkg/api/pod"
|
||||
api "k8s.io/kubernetes/pkg/apis/core"
|
||||
"k8s.io/kubernetes/pkg/apis/core/validation"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
"k8s.io/kubernetes/pkg/kubelet/client"
|
||||
"k8s.io/kubernetes/pkg/printers"
|
||||
printersinternal "k8s.io/kubernetes/pkg/printers/internalversion"
|
||||
@ -45,15 +47,16 @@ import (
|
||||
|
||||
// PodStorage includes storage for pods and all sub resources
|
||||
type PodStorage struct {
|
||||
Pod *REST
|
||||
Binding *BindingREST
|
||||
Eviction *EvictionREST
|
||||
Status *StatusREST
|
||||
Log *podrest.LogREST
|
||||
Proxy *podrest.ProxyREST
|
||||
Exec *podrest.ExecREST
|
||||
Attach *podrest.AttachREST
|
||||
PortForward *podrest.PortForwardREST
|
||||
Pod *REST
|
||||
Binding *BindingREST
|
||||
Eviction *EvictionREST
|
||||
Status *StatusREST
|
||||
EphemeralContainers *EphemeralContainersREST
|
||||
Log *podrest.LogREST
|
||||
Proxy *podrest.ProxyREST
|
||||
Exec *podrest.ExecREST
|
||||
Attach *podrest.AttachREST
|
||||
PortForward *podrest.PortForwardREST
|
||||
}
|
||||
|
||||
// REST implements a RESTStorage for pods
|
||||
@ -89,17 +92,20 @@ func NewStorage(optsGetter generic.RESTOptionsGetter, k client.ConnectionInfoGet
|
||||
|
||||
statusStore := *store
|
||||
statusStore.UpdateStrategy = pod.StatusStrategy
|
||||
ephemeralContainersStore := *store
|
||||
ephemeralContainersStore.UpdateStrategy = pod.EphemeralContainersStrategy
|
||||
|
||||
return PodStorage{
|
||||
Pod: &REST{store, proxyTransport},
|
||||
Binding: &BindingREST{store: store},
|
||||
Eviction: newEvictionStorage(store, podDisruptionBudgetClient),
|
||||
Status: &StatusREST{store: &statusStore},
|
||||
Log: &podrest.LogREST{Store: store, KubeletConn: k},
|
||||
Proxy: &podrest.ProxyREST{Store: store, ProxyTransport: proxyTransport},
|
||||
Exec: &podrest.ExecREST{Store: store, KubeletConn: k},
|
||||
Attach: &podrest.AttachREST{Store: store, KubeletConn: k},
|
||||
PortForward: &podrest.PortForwardREST{Store: store, KubeletConn: k},
|
||||
Pod: &REST{store, proxyTransport},
|
||||
Binding: &BindingREST{store: store},
|
||||
Eviction: newEvictionStorage(store, podDisruptionBudgetClient),
|
||||
Status: &StatusREST{store: &statusStore},
|
||||
EphemeralContainers: &EphemeralContainersREST{store: &ephemeralContainersStore},
|
||||
Log: &podrest.LogREST{Store: store, KubeletConn: k},
|
||||
Proxy: &podrest.ProxyREST{Store: store, ProxyTransport: proxyTransport},
|
||||
Exec: &podrest.ExecREST{Store: store, KubeletConn: k},
|
||||
Attach: &podrest.AttachREST{Store: store, KubeletConn: k},
|
||||
PortForward: &podrest.PortForwardREST{Store: store, KubeletConn: k},
|
||||
}
|
||||
}
|
||||
|
||||
@ -233,3 +239,96 @@ func (r *StatusREST) Update(ctx context.Context, name string, objInfo rest.Updat
|
||||
// subresources should never allow create on update.
|
||||
return r.store.Update(ctx, name, objInfo, createValidation, updateValidation, false, options)
|
||||
}
|
||||
|
||||
// EphemeralContainersREST implements the REST endpoint for adding EphemeralContainers
|
||||
type EphemeralContainersREST struct {
|
||||
store *genericregistry.Store
|
||||
}
|
||||
|
||||
var _ = rest.Patcher(&EphemeralContainersREST{})
|
||||
|
||||
// Get of this endpoint will return the list of ephemeral containers in this pod
|
||||
func (r *EphemeralContainersREST) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
|
||||
if !utilfeature.DefaultFeatureGate.Enabled(features.EphemeralContainers) {
|
||||
return nil, errors.NewBadRequest("feature EphemeralContainers disabled")
|
||||
}
|
||||
|
||||
obj, err := r.store.Get(ctx, name, options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ephemeralContainersInPod(obj.(*api.Pod)), nil
|
||||
}
|
||||
|
||||
// New creates a new EphemeralContainers resource
|
||||
func (r *EphemeralContainersREST) New() runtime.Object {
|
||||
return &api.EphemeralContainers{}
|
||||
}
|
||||
|
||||
// Update alters the EphemeralContainers field in PodSpec
|
||||
func (r *EphemeralContainersREST) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
|
||||
if !utilfeature.DefaultFeatureGate.Enabled(features.EphemeralContainers) {
|
||||
return nil, false, errors.NewBadRequest("feature EphemeralContainers disabled")
|
||||
}
|
||||
|
||||
obj, err := r.store.Get(ctx, name, &metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
pod := obj.(*api.Pod)
|
||||
|
||||
// Build an UpdatedObjectInfo to pass to the pod store.
|
||||
// It is given the currently stored v1.Pod and transforms it to the new pod that should be stored.
|
||||
updatedPodInfo := rest.DefaultUpdatedObjectInfo(pod, func(ctx context.Context, oldObject, _ runtime.Object) (newObject runtime.Object, err error) {
|
||||
oldPod, ok := oldObject.(*api.Pod)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unexpected type for Pod %T", oldObject)
|
||||
}
|
||||
|
||||
newEphemeralContainersObj, err := objInfo.UpdatedObject(ctx, ephemeralContainersInPod(oldPod))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
newEphemeralContainers, ok := newEphemeralContainersObj.(*api.EphemeralContainers)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unexpected type for EphemeralContainers %T", newEphemeralContainersObj)
|
||||
}
|
||||
|
||||
// avoid mutating
|
||||
newPod := oldPod.DeepCopy()
|
||||
// identity, version (make sure we're working with the right object, instance, and version)
|
||||
newPod.Name = newEphemeralContainers.Name
|
||||
newPod.Namespace = newEphemeralContainers.Namespace
|
||||
newPod.UID = newEphemeralContainers.UID
|
||||
newPod.ResourceVersion = newEphemeralContainers.ResourceVersion
|
||||
// ephemeral containers
|
||||
newPod.Spec.EphemeralContainers = newEphemeralContainers.EphemeralContainers
|
||||
|
||||
return newPod, nil
|
||||
})
|
||||
|
||||
obj, _, err = r.store.Update(ctx, name, updatedPodInfo, createValidation, updateValidation, false, options)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return ephemeralContainersInPod(obj.(*api.Pod)), false, err
|
||||
}
|
||||
|
||||
// Extract the list of Ephemeral Containers from a Pod
|
||||
func ephemeralContainersInPod(pod *api.Pod) *api.EphemeralContainers {
|
||||
ephemeralContainers := pod.Spec.EphemeralContainers
|
||||
if ephemeralContainers == nil {
|
||||
ephemeralContainers = []api.EphemeralContainer{}
|
||||
}
|
||||
return &api.EphemeralContainers{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: pod.Name,
|
||||
Namespace: pod.Namespace,
|
||||
UID: pod.UID,
|
||||
ResourceVersion: pod.ResourceVersion,
|
||||
CreationTimestamp: pod.CreationTimestamp,
|
||||
},
|
||||
EphemeralContainers: ephemeralContainers,
|
||||
}
|
||||
}
|
||||
|
@ -84,7 +84,7 @@ func (podStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object
|
||||
// Validate validates a new pod.
|
||||
func (podStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList {
|
||||
pod := obj.(*api.Pod)
|
||||
allErrs := validation.ValidatePod(pod)
|
||||
allErrs := validation.ValidatePodCreate(pod)
|
||||
allErrs = append(allErrs, validation.ValidateConditionalPod(pod, nil, field.NewPath(""))...)
|
||||
return allErrs
|
||||
}
|
||||
@ -174,6 +174,16 @@ func (podStatusStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Ob
|
||||
return validation.ValidatePodStatusUpdate(obj.(*api.Pod), old.(*api.Pod))
|
||||
}
|
||||
|
||||
type podEphemeralContainersStrategy struct {
|
||||
podStrategy
|
||||
}
|
||||
|
||||
var EphemeralContainersStrategy = podEphemeralContainersStrategy{Strategy}
|
||||
|
||||
func (podEphemeralContainersStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList {
|
||||
return validation.ValidatePodEphemeralContainersUpdate(obj.(*api.Pod), old.(*api.Pod))
|
||||
}
|
||||
|
||||
// GetAttrs returns labels and fields of a given object for filtering purposes.
|
||||
func GetAttrs(obj runtime.Object) (labels.Set, fields.Set, error) {
|
||||
pod, ok := obj.(*api.Pod)
|
||||
|
@ -238,6 +238,9 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi
|
||||
if serviceAccountStorage.Token != nil {
|
||||
restStorageMap["serviceaccounts/token"] = serviceAccountStorage.Token
|
||||
}
|
||||
if utilfeature.DefaultFeatureGate.Enabled(features.EphemeralContainers) {
|
||||
restStorageMap["pods/ephemeralcontainers"] = podStorage.EphemeralContainers
|
||||
}
|
||||
apiGroupInfo.VersionedResourcesStorageMap["v1"] = restStorageMap
|
||||
|
||||
return restStorage, apiGroupInfo, nil
|
||||
|
@ -88,6 +88,7 @@ func addKnownTypes(scheme *runtime.Scheme) error {
|
||||
&RangeAllocation{},
|
||||
&ConfigMap{},
|
||||
&ConfigMapList{},
|
||||
&EphemeralContainers{},
|
||||
)
|
||||
|
||||
// Add common types
|
||||
|
@ -2843,6 +2843,17 @@ type PodSpec struct {
|
||||
// +patchMergeKey=name
|
||||
// +patchStrategy=merge
|
||||
Containers []Container `json:"containers" patchStrategy:"merge" patchMergeKey:"name" protobuf:"bytes,2,rep,name=containers"`
|
||||
// EphemeralContainers is the list of ephemeral containers that run in this pod. Ephemeral containers
|
||||
// are added to an existing pod as a result of a user-initiated action such as troubleshooting.
|
||||
// This list is read-only in the pod spec. It may not be specified in a create or modified in an
|
||||
// update of a pod or pod template.
|
||||
// To add an ephemeral container use the pod's ephemeralcontainers subresource, which allows update
|
||||
// using the EphemeralContainers kind.
|
||||
// This field is alpha-level and is only honored by servers that enable the EphemeralContainers feature.
|
||||
// +optional
|
||||
// +patchMergeKey=name
|
||||
// +patchStrategy=merge
|
||||
EphemeralContainers []EphemeralContainer `json:"ephemeralContainers,omitempty" patchStrategy:"merge" patchMergeKey:"name" protobuf:"bytes,34,rep,name=ephemeralContainers"`
|
||||
// Restart policy for all containers within the pod.
|
||||
// One of Always, OnFailure, Never.
|
||||
// Default to Always.
|
||||
@ -3209,6 +3220,156 @@ type PodIP struct {
|
||||
IP string `json:"ip,omitempty" protobuf:"bytes,1,opt,name=ip"`
|
||||
}
|
||||
|
||||
type EphemeralContainerCommon struct {
|
||||
// Name of the ephemeral container specified as a DNS_LABEL.
|
||||
// This name must be unique among all containers, init containers and ephemeral containers.
|
||||
Name string `json:"name" protobuf:"bytes,1,opt,name=name"`
|
||||
// Docker image name.
|
||||
// More info: https://kubernetes.io/docs/concepts/containers/images
|
||||
Image string `json:"image,omitempty" protobuf:"bytes,2,opt,name=image"`
|
||||
// Entrypoint array. Not executed within a shell.
|
||||
// The docker image's ENTRYPOINT is used if this is not provided.
|
||||
// Variable references $(VAR_NAME) are expanded using the container's environment. If a variable
|
||||
// cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax
|
||||
// can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded,
|
||||
// regardless of whether the variable exists or not.
|
||||
// Cannot be updated.
|
||||
// More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell
|
||||
// +optional
|
||||
Command []string `json:"command,omitempty" protobuf:"bytes,3,rep,name=command"`
|
||||
// Arguments to the entrypoint.
|
||||
// The docker image's CMD is used if this is not provided.
|
||||
// Variable references $(VAR_NAME) are expanded using the container's environment. If a variable
|
||||
// cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax
|
||||
// can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded,
|
||||
// regardless of whether the variable exists or not.
|
||||
// Cannot be updated.
|
||||
// More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell
|
||||
// +optional
|
||||
Args []string `json:"args,omitempty" protobuf:"bytes,4,rep,name=args"`
|
||||
// Container's working directory.
|
||||
// If not specified, the container runtime's default will be used, which
|
||||
// might be configured in the container image.
|
||||
// Cannot be updated.
|
||||
// +optional
|
||||
WorkingDir string `json:"workingDir,omitempty" protobuf:"bytes,5,opt,name=workingDir"`
|
||||
// Ports are not allowed for ephemeral containers.
|
||||
Ports []ContainerPort `json:"ports,omitempty" protobuf:"bytes,6,rep,name=ports"`
|
||||
// List of sources to populate environment variables in the container.
|
||||
// The keys defined within a source must be a C_IDENTIFIER. All invalid keys
|
||||
// will be reported as an event when the container is starting. When a key exists in multiple
|
||||
// sources, the value associated with the last source will take precedence.
|
||||
// Values defined by an Env with a duplicate key will take precedence.
|
||||
// Cannot be updated.
|
||||
// +optional
|
||||
EnvFrom []EnvFromSource `json:"envFrom,omitempty" protobuf:"bytes,19,rep,name=envFrom"`
|
||||
// List of environment variables to set in the container.
|
||||
// Cannot be updated.
|
||||
// +optional
|
||||
// +patchMergeKey=name
|
||||
// +patchStrategy=merge
|
||||
Env []EnvVar `json:"env,omitempty" patchStrategy:"merge" patchMergeKey:"name" protobuf:"bytes,7,rep,name=env"`
|
||||
// Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources
|
||||
// already allocated to the pod.
|
||||
// +optional
|
||||
Resources ResourceRequirements `json:"resources,omitempty" protobuf:"bytes,8,opt,name=resources"`
|
||||
// Pod volumes to mount into the container's filesystem.
|
||||
// Cannot be updated.
|
||||
// +optional
|
||||
// +patchMergeKey=mountPath
|
||||
// +patchStrategy=merge
|
||||
VolumeMounts []VolumeMount `json:"volumeMounts,omitempty" patchStrategy:"merge" patchMergeKey:"mountPath" protobuf:"bytes,9,rep,name=volumeMounts"`
|
||||
// volumeDevices is the list of block devices to be used by the container.
|
||||
// This is a beta feature.
|
||||
// +patchMergeKey=devicePath
|
||||
// +patchStrategy=merge
|
||||
// +optional
|
||||
VolumeDevices []VolumeDevice `json:"volumeDevices,omitempty" patchStrategy:"merge" patchMergeKey:"devicePath" protobuf:"bytes,21,rep,name=volumeDevices"`
|
||||
// Probes are not allowed for ephemeral containers.
|
||||
// +optional
|
||||
LivenessProbe *Probe `json:"livenessProbe,omitempty" protobuf:"bytes,10,opt,name=livenessProbe"`
|
||||
// Probes are not allowed for ephemeral containers.
|
||||
// +optional
|
||||
ReadinessProbe *Probe `json:"readinessProbe,omitempty" protobuf:"bytes,11,opt,name=readinessProbe"`
|
||||
// Lifecycle is not allowed for ephemeral containers.
|
||||
// +optional
|
||||
Lifecycle *Lifecycle `json:"lifecycle,omitempty" protobuf:"bytes,12,opt,name=lifecycle"`
|
||||
// Optional: Path at which the file to which the container's termination message
|
||||
// will be written is mounted into the container's filesystem.
|
||||
// Message written is intended to be brief final status, such as an assertion failure message.
|
||||
// Will be truncated by the node if greater than 4096 bytes. The total message length across
|
||||
// all containers will be limited to 12kb.
|
||||
// Defaults to /dev/termination-log.
|
||||
// Cannot be updated.
|
||||
// +optional
|
||||
TerminationMessagePath string `json:"terminationMessagePath,omitempty" protobuf:"bytes,13,opt,name=terminationMessagePath"`
|
||||
// Indicate how the termination message should be populated. File will use the contents of
|
||||
// terminationMessagePath to populate the container status message on both success and failure.
|
||||
// FallbackToLogsOnError will use the last chunk of container log output if the termination
|
||||
// message file is empty and the container exited with an error.
|
||||
// The log output is limited to 2048 bytes or 80 lines, whichever is smaller.
|
||||
// Defaults to File.
|
||||
// Cannot be updated.
|
||||
// +optional
|
||||
TerminationMessagePolicy TerminationMessagePolicy `json:"terminationMessagePolicy,omitempty" protobuf:"bytes,20,opt,name=terminationMessagePolicy,casttype=TerminationMessagePolicy"`
|
||||
// Image pull policy.
|
||||
// One of Always, Never, IfNotPresent.
|
||||
// Defaults to Always if :latest tag is specified, or IfNotPresent otherwise.
|
||||
// Cannot be updated.
|
||||
// More info: https://kubernetes.io/docs/concepts/containers/images#updating-images
|
||||
// +optional
|
||||
ImagePullPolicy PullPolicy `json:"imagePullPolicy,omitempty" protobuf:"bytes,14,opt,name=imagePullPolicy,casttype=PullPolicy"`
|
||||
// SecurityContext is not allowed for ephemeral containers.
|
||||
// +optional
|
||||
SecurityContext *SecurityContext `json:"securityContext,omitempty" protobuf:"bytes,15,opt,name=securityContext"`
|
||||
|
||||
// Variables for interactive containers, these have very specialized use-cases (e.g. debugging)
|
||||
// and shouldn't be used for general purpose containers.
|
||||
|
||||
// Whether this container should allocate a buffer for stdin in the container runtime. If this
|
||||
// is not set, reads from stdin in the container will always result in EOF.
|
||||
// Default is false.
|
||||
// +optional
|
||||
Stdin bool `json:"stdin,omitempty" protobuf:"varint,16,opt,name=stdin"`
|
||||
// Whether the container runtime should close the stdin channel after it has been opened by
|
||||
// a single attach. When stdin is true the stdin stream will remain open across multiple attach
|
||||
// sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the
|
||||
// first client attaches to stdin, and then remains open and accepts data until the client disconnects,
|
||||
// at which time stdin is closed and remains closed until the container is restarted. If this
|
||||
// flag is false, a container processes that reads from stdin will never receive an EOF.
|
||||
// Default is false
|
||||
// +optional
|
||||
StdinOnce bool `json:"stdinOnce,omitempty" protobuf:"varint,17,opt,name=stdinOnce"`
|
||||
// Whether this container should allocate a TTY for itself, also requires 'stdin' to be true.
|
||||
// Default is false.
|
||||
// +optional
|
||||
TTY bool `json:"tty,omitempty" protobuf:"varint,18,opt,name=tty"`
|
||||
}
|
||||
|
||||
// EphemeralContainerCommon converts to Container. All fields must be kept in sync between
|
||||
// these two types.
|
||||
var _ = Container(EphemeralContainerCommon{})
|
||||
|
||||
// An EphemeralContainer is a special type of container which doesn't come with any resource
|
||||
// or scheduling guarantees but can be added to a pod that has already been created. They are
|
||||
// intended for user-initiated activities such as troubleshooting a running pod.
|
||||
// Ephemeral containers will not be restarted when they exit, and they will be killed if the
|
||||
// pod is removed or restarted. If an ephemeral container causes a pod to exceed its resource
|
||||
// allocation, the pod may be evicted.
|
||||
// Ephemeral containers are added via a pod's ephemeralcontainers subresource and will appear
|
||||
// in the pod spec once added. No fields in EphemeralContainer may be changed once added.
|
||||
// This is an alpha feature enabled by the EphemeralContainers feature flag.
|
||||
type EphemeralContainer struct {
|
||||
EphemeralContainerCommon `json:",inline" protobuf:"bytes,1,req"`
|
||||
|
||||
// If set, the name of the container from PodSpec that this ephemeral container targets.
|
||||
// The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container.
|
||||
// If not set then the ephemeral container is run in whatever namespaces are shared
|
||||
// for the pod. Note that the container runtime must support this feature.
|
||||
// +optional
|
||||
TargetContainerName string `json:"targetContainerName,omitempty" protobuf:"bytes,2,opt,name=targetContainerName"`
|
||||
}
|
||||
|
||||
// PodStatus represents information about the status of a pod. Status may trail the actual
|
||||
// state of a system, especially if the node that hosts the pod cannot contact the control
|
||||
// plane.
|
||||
@ -3293,6 +3454,10 @@ type PodStatus struct {
|
||||
// More info: https://git.k8s.io/community/contributors/design-proposals/node/resource-qos.md
|
||||
// +optional
|
||||
QOSClass PodQOSClass `json:"qosClass,omitempty" protobuf:"bytes,9,rep,name=qosClass"`
|
||||
// Status for any ephemeral containers that running in this pod.
|
||||
// This field is alpha-level and is only honored by servers that enable the EphemeralContainers feature.
|
||||
// +optional
|
||||
EphemeralContainerStatuses []ContainerStatus `json:"ephemeralContainerStatuses,omitempty" protobuf:"bytes,13,rep,name=ephemeralContainerStatuses"`
|
||||
}
|
||||
|
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
@ -3314,6 +3479,8 @@ type PodStatusResult struct {
|
||||
}
|
||||
|
||||
// +genclient
|
||||
// +genclient:method=GetEphemeralContainers,verb=get,subresource=ephemeralcontainers,result=EphemeralContainers
|
||||
// +genclient:method=UpdateEphemeralContainers,verb=update,subresource=ephemeralcontainers,input=EphemeralContainers,result=EphemeralContainers
|
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
|
||||
// Pod is a collection of containers that can run on a host. This resource is created
|
||||
@ -4494,6 +4661,20 @@ type Binding struct {
|
||||
Target ObjectReference `json:"target" protobuf:"bytes,2,opt,name=target"`
|
||||
}
|
||||
|
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
|
||||
// A list of ephemeral containers used in API operations
|
||||
type EphemeralContainers struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
// +optional
|
||||
metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
|
||||
|
||||
// The new set of ephemeral containers to use for a pod.
|
||||
// +patchMergeKey=name
|
||||
// +patchStrategy=merge
|
||||
EphemeralContainers []EphemeralContainer `json:"ephemeralContainers" patchStrategy:"merge" patchMergeKey:"name" protobuf:"bytes,2,rep,name=ephemeralContainers"`
|
||||
}
|
||||
|
||||
// Preconditions must be fulfilled before an operation (update, delete, etc.) is carried out.
|
||||
// +k8s:openapi-gen=false
|
||||
type Preconditions struct {
|
||||
|
@ -58,6 +58,7 @@ var kindWhiteList = sets.NewString(
|
||||
"APIVersions",
|
||||
"Binding",
|
||||
"DeleteOptions",
|
||||
"EphemeralContainers",
|
||||
"ExportOptions",
|
||||
"GetOptions",
|
||||
"ListOptions",
|
||||
|
Loading…
Reference in New Issue
Block a user