github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-31 07:20:13 +00:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into release-1.33

Browse Source
This commit is contained in:
Kubernetes Release Robot 2025-04-14 19:31:24 +00:00
commit 2f7a115a69
11 changed files with 866 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
pkg/apis/networking/validation/validation.go
View File

@ -765,28 +765,31 @@ var ValidateServiceCIDRName = apimachineryvalidation.NameIsDNSSubdomain
func ValidateServiceCIDR(cidrConfig *networking.ServiceCIDR) field.ErrorList {
allErrs := apivalidation.ValidateObjectMeta(&cidrConfig.ObjectMeta, false, ValidateServiceCIDRName, field.NewPath("metadata"))
fieldPath := field.NewPath("spec", "cidrs")
allErrs = append(allErrs, validateServiceCIDRSpec(&cidrConfig.Spec, field.NewPath("spec", "cidrs"))...)
return allErrs
}
if len(cidrConfig.Spec.CIDRs) == 0 {
func validateServiceCIDRSpec(cidrConfigSpec *networking.ServiceCIDRSpec, fieldPath *field.Path) field.ErrorList {
var allErrs field.ErrorList
if len(cidrConfigSpec.CIDRs) == 0 {
allErrs = append(allErrs, field.Required(fieldPath, "at least one CIDR required"))
return allErrs
}
if len(cidrConfig.Spec.CIDRs) > 2 {
allErrs = append(allErrs, field.Invalid(fieldPath, cidrConfig.Spec, "may only hold up to 2 values"))
if len(cidrConfigSpec.CIDRs) > 2 {
allErrs = append(allErrs, field.Invalid(fieldPath, cidrConfigSpec, "may only hold up to 2 values"))
return allErrs
}
// validate cidrs are dual stack, one of each IP family
if len(cidrConfig.Spec.CIDRs) == 2 {
isDual, err := netutils.IsDualStackCIDRStrings(cidrConfig.Spec.CIDRs)
if err != nil || !isDual {
allErrs = append(allErrs, field.Invalid(fieldPath, cidrConfig.Spec, "may specify no more than one IP for each IP family, i.e 192.168.0.0/24 and 2001:db8::/64"))
return allErrs
}
for i, cidr := range cidrConfigSpec.CIDRs {
allErrs = append(allErrs, validation.IsValidCIDR(fieldPath.Index(i), cidr)...)
}
for i, cidr := range cidrConfig.Spec.CIDRs {
allErrs = append(allErrs, validation.IsValidCIDR(fieldPath.Index(i), cidr)...)
// validate cidrs are dual stack, one of each IP family
if len(cidrConfigSpec.CIDRs) == 2 &&
netutils.IPFamilyOfCIDRString(cidrConfigSpec.CIDRs[0]) == netutils.IPFamilyOfCIDRString(cidrConfigSpec.CIDRs[1]) &&
netutils.IPFamilyOfCIDRString(cidrConfigSpec.CIDRs[0]) != netutils.IPFamilyUnknown {
allErrs = append(allErrs, field.Invalid(fieldPath, cidrConfigSpec.CIDRs, "may specify no more than one IP for each IP family, i.e 192.168.0.0/24 and 2001:db8::/64"))
}
return allErrs
@ -796,8 +799,28 @@ func ValidateServiceCIDR(cidrConfig *networking.ServiceCIDR) field.ErrorList {
func ValidateServiceCIDRUpdate(update, old *networking.ServiceCIDR) field.ErrorList {
var allErrs field.ErrorList
allErrs = append(allErrs, apivalidation.ValidateObjectMetaUpdate(&update.ObjectMeta, &old.ObjectMeta, field.NewPath("metadata"))...)
allErrs = append(allErrs, apivalidation.ValidateImmutableField(update.Spec.CIDRs, old.Spec.CIDRs, field.NewPath("spec").Child("cidrs"))...)
switch {
// no change in Spec.CIDRs lengths fields must not change
case len(old.Spec.CIDRs) == len(update.Spec.CIDRs):
for i, ip := range old.Spec.CIDRs {
if ip != update.Spec.CIDRs[i] {
allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("cidrs").Index(i), update.Spec.CIDRs[i], apimachineryvalidation.FieldImmutableErrorMsg))
}
}
// added a new CIDR is allowed to convert to Dual Stack
// ref: https://issues.k8s.io/131261
case len(old.Spec.CIDRs) == 1 && len(update.Spec.CIDRs) == 2:
// existing CIDR can not change
if update.Spec.CIDRs[0] != old.Spec.CIDRs[0] {
allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("cidrs").Index(0), update.Spec.CIDRs[0], apimachineryvalidation.FieldImmutableErrorMsg))
}
// validate the new added CIDR
allErrs = append(allErrs, validateServiceCIDRSpec(&update.Spec, field.NewPath("spec", "cidrs"))...)
// no other changes allowed
default:
allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("cidrs"), update.Spec.CIDRs, apimachineryvalidation.FieldImmutableErrorMsg))
}
return allErrs
}

236
pkg/apis/networking/validation/validation_test.go
View File

@ -2373,6 +2373,17 @@ func TestValidateServiceCIDR(t *testing.T) {
},
},
},
"bad-iprange-ipv6-bad-ipv4": {
expectedErrors: 2,
ipRange: &networking.ServiceCIDR{
ObjectMeta: metav1.ObjectMeta{
Name: "test-name",
},
Spec: networking.ServiceCIDRSpec{
CIDRs: []string{"192.168.007.0/24", "MN00:1234::/64"},
},
},
},
}
for name, testCase := range testCases {
@ -2386,9 +2397,27 @@ func TestValidateServiceCIDR(t *testing.T) {
}
func TestValidateServiceCIDRUpdate(t *testing.T) {
oldServiceCIDR := &networking.ServiceCIDR{
oldServiceCIDRv4 := &networking.ServiceCIDR{
ObjectMeta: metav1.ObjectMeta{
Name: "mysvc",
Name: "mysvc-v4",
ResourceVersion: "1",
},
Spec: networking.ServiceCIDRSpec{
CIDRs: []string{"192.168.0.0/24"},
},
}
oldServiceCIDRv6 := &networking.ServiceCIDR{
ObjectMeta: metav1.ObjectMeta{
Name: "mysvc-v6",
ResourceVersion: "1",
},
Spec: networking.ServiceCIDRSpec{
CIDRs: []string{"fd00:1234::/64"},
},
}
oldServiceCIDRDual := &networking.ServiceCIDR{
ObjectMeta: metav1.ObjectMeta{
Name: "mysvc-dual",
ResourceVersion: "1",
},
Spec: networking.ServiceCIDRSpec{
@ -2396,45 +2425,196 @@ func TestValidateServiceCIDRUpdate(t *testing.T) {
},
}
// Define expected immutable field error for convenience
cidrsPath := field.NewPath("spec").Child("cidrs")
cidr0Path := cidrsPath.Index(0)
cidr1Path := cidrsPath.Index(1)
testCases := []struct {
name string
svc func(svc *networking.ServiceCIDR) *networking.ServiceCIDR
expectErr bool
name string
old *networking.ServiceCIDR
new *networking.ServiceCIDR
expectedErrs field.ErrorList
}{
{
name: "Successful update, no changes",
svc: func(svc *networking.ServiceCIDR) *networking.ServiceCIDR {
out := svc.DeepCopy()
return out
},
expectErr: false,
name: "Successful update, no changes (dual)",
old: oldServiceCIDRDual,
new: oldServiceCIDRDual.DeepCopy(),
},
{
name: "Failed update, update spec.CIDRs single stack",
svc: func(svc *networking.ServiceCIDR) *networking.ServiceCIDR {
out := svc.DeepCopy()
name: "Successful update, no changes (v4)",
old: oldServiceCIDRv4,
new: oldServiceCIDRv4.DeepCopy(),
},
{
name: "Successful update, single IPv4 to dual stack upgrade",
old: oldServiceCIDRv4,
new: func() *networking.ServiceCIDR {
out := oldServiceCIDRv4.DeepCopy()
out.Spec.CIDRs = []string{"192.168.0.0/24", "fd00:1234::/64"} // Add IPv6
return out
}(),
},
{
name: "Successful update, single IPv6 to dual stack upgrade",
old: oldServiceCIDRv6,
new: func() *networking.ServiceCIDR {
out := oldServiceCIDRv6.DeepCopy()
out.Spec.CIDRs = []string{"fd00:1234::/64", "192.168.0.0/24"} // Add IPv4
return out
}(),
},
{
name: "Failed update, change CIDRs (dual)",
old: oldServiceCIDRDual,
new: func() *networking.ServiceCIDR {
out := oldServiceCIDRDual.DeepCopy()
out.Spec.CIDRs = []string{"10.0.0.0/16", "fd00:abcd::/64"}
return out
}(),
expectedErrs: field.ErrorList{
field.Invalid(cidr0Path, "10.0.0.0/16", apimachineryvalidation.FieldImmutableErrorMsg),
field.Invalid(cidr1Path, "fd00:abcd::/64", apimachineryvalidation.FieldImmutableErrorMsg),
},
},
{
name: "Failed update, change CIDRs (single)",
old: oldServiceCIDRv4,
new: func() *networking.ServiceCIDR {
out := oldServiceCIDRv4.DeepCopy()
out.Spec.CIDRs = []string{"10.0.0.0/16"}
return out
}, expectErr: true,
}(),
expectedErrs: field.ErrorList{field.Invalid(cidr0Path, "10.0.0.0/16", apimachineryvalidation.FieldImmutableErrorMsg)},
},
{
name: "Failed update, update spec.CIDRs dual stack",
svc: func(svc *networking.ServiceCIDR) *networking.ServiceCIDR {
out := svc.DeepCopy()
out.Spec.CIDRs = []string{"10.0.0.0/24", "fd00:1234::/64"}
name: "Failed update, single IPv4 to dual stack upgrade with primary change",
old: oldServiceCIDRv4,
new: func() *networking.ServiceCIDR {
out := oldServiceCIDRv4.DeepCopy()
// Change primary CIDR during upgrade
out.Spec.CIDRs = []string{"10.0.0.0/16", "fd00:1234::/64"}
return out
}, expectErr: true,
}(),
expectedErrs: field.ErrorList{field.Invalid(cidr0Path, "10.0.0.0/16", apimachineryvalidation.FieldImmutableErrorMsg)},
},
{
name: "Failed update, single IPv6 to dual stack upgrade with primary change",
old: oldServiceCIDRv6,
new: func() *networking.ServiceCIDR {
out := oldServiceCIDRv6.DeepCopy()
// Change primary CIDR during upgrade
out.Spec.CIDRs = []string{"fd00:abcd::/64", "192.168.0.0/24"}
return out
}(),
expectedErrs: field.ErrorList{field.Invalid(cidr0Path, "fd00:abcd::/64", apimachineryvalidation.FieldImmutableErrorMsg)},
},
{
name: "Failed update, dual stack downgrade to single",
old: oldServiceCIDRDual,
new: func() *networking.ServiceCIDR {
out := oldServiceCIDRDual.DeepCopy()
out.Spec.CIDRs = []string{"192.168.0.0/24"} // Remove IPv6
return out
}(),
expectedErrs: field.ErrorList{field.Invalid(cidrsPath, []string{"192.168.0.0/24"}, apimachineryvalidation.FieldImmutableErrorMsg)},
},
{
name: "Failed update, dual stack reorder",
old: oldServiceCIDRDual,
new: func() *networking.ServiceCIDR {
out := oldServiceCIDRDual.DeepCopy()
// Swap order
out.Spec.CIDRs = []string{"fd00:1234::/64", "192.168.0.0/24"}
return out
}(),
expectedErrs: field.ErrorList{
field.Invalid(cidr0Path, "fd00:1234::/64", apimachineryvalidation.FieldImmutableErrorMsg),
field.Invalid(cidr1Path, "192.168.0.0/24", apimachineryvalidation.FieldImmutableErrorMsg),
},
},
{
name: "Failed update, add invalid CIDR during upgrade",
old: oldServiceCIDRv4,
new: func() *networking.ServiceCIDR {
out := oldServiceCIDRv4.DeepCopy()
out.Spec.CIDRs = []string{"192.168.0.0/24", "invalid-cidr"}
return out
}(),
expectedErrs: field.ErrorList{field.Invalid(cidrsPath.Index(1), "invalid-cidr", "must be a valid CIDR value, (e.g. 10.9.8.0/24 or 2001:db8::/64)")},
},
{
name: "Failed update, add duplicate family CIDR during upgrade",
old: oldServiceCIDRv4,
new: func() *networking.ServiceCIDR {
out := oldServiceCIDRv4.DeepCopy()
out.Spec.CIDRs = []string{"192.168.0.0/24", "10.0.0.0/16"}
return out
}(),
expectedErrs: field.ErrorList{field.Invalid(cidrsPath, []string{"192.168.0.0/24", "10.0.0.0/16"}, "may specify no more than one IP for each IP family, i.e 192.168.0.0/24 and 2001:db8::/64")},
},
{
name: "Failed update, dual stack remove one cidr",
old: oldServiceCIDRDual,
new: func() *networking.ServiceCIDR {
out := oldServiceCIDRDual.DeepCopy()
out.Spec.CIDRs = out.Spec.CIDRs[0:1]
return out
}(),
expectedErrs: field.ErrorList{
field.Invalid(cidrsPath, []string{"192.168.0.0/24"}, apimachineryvalidation.FieldImmutableErrorMsg),
},
},
{
name: "Failed update, dual stack remove all cidrs",
old: oldServiceCIDRDual,
new: func() *networking.ServiceCIDR {
out := oldServiceCIDRDual.DeepCopy()
out.Spec.CIDRs = []string{}
return out
}(),
expectedErrs: field.ErrorList{
field.Invalid(cidrsPath, []string{}, apimachineryvalidation.FieldImmutableErrorMsg),
},
},
{
name: "Failed update, single stack remove cidr",
old: oldServiceCIDRv4,
new: func() *networking.ServiceCIDR {
out := oldServiceCIDRv4.DeepCopy()
out.Spec.CIDRs = []string{}
return out
}(),
expectedErrs: field.ErrorList{
field.Invalid(cidrsPath, []string{}, apimachineryvalidation.FieldImmutableErrorMsg),
},
},
{
name: "Failed update, add additional cidrs",
old: oldServiceCIDRDual,
new: func() *networking.ServiceCIDR {
out := oldServiceCIDRDual.DeepCopy()
out.Spec.CIDRs = append(out.Spec.CIDRs, "172.16.0.0/24")
return out
}(),
expectedErrs: field.ErrorList{
field.Invalid(cidrsPath, []string{"192.168.0.0/24", "fd00:1234::/64", "172.16.0.0/24"}, apimachineryvalidation.FieldImmutableErrorMsg),
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
err := ValidateServiceCIDRUpdate(testCase.svc(oldServiceCIDR), oldServiceCIDR)
if !testCase.expectErr && err != nil {
t.Errorf("ValidateServiceCIDRUpdate must be successful for test '%s', got %v", testCase.name, err)
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
// Ensure ResourceVersion is set for update validation
tc.new.ResourceVersion = tc.old.ResourceVersion
errs := ValidateServiceCIDRUpdate(tc.new, tc.old)
if len(errs) != len(tc.expectedErrs) {
t.Fatalf("Expected %d errors, got %d errors: %v", len(tc.expectedErrs), len(errs), errs)
}
if testCase.expectErr && err == nil {
t.Errorf("ValidateServiceCIDRUpdate must return error for test: %s, but got nil", testCase.name)
for i, expectedErr := range tc.expectedErrs {
if errs[i].Error() != expectedErr.Error() {
t.Errorf("Expected error %d: %v, got: %v", i, expectedErr, errs[i])
}
}
})
}

42
pkg/controlplane/controller/defaultservicecidr/default_servicecidr_controller.go
View File

@ -91,7 +91,9 @@ type Controller struct {
serviceCIDRLister networkingv1listers.ServiceCIDRLister
serviceCIDRsSynced cache.InformerSynced
interval time.Duration
interval time.Duration
reportedMismatchedCIDRs bool
reportedNotReadyCondition bool
}
// Start will not return until the default ServiceCIDR exists or stopCh is closed.
@ -138,7 +140,19 @@ func (c *Controller) sync() error {
serviceCIDR, err := c.serviceCIDRLister.Get(DefaultServiceCIDRName)
// if exists
if err == nil {
c.syncStatus(serviceCIDR)
// single to dual stack upgrade
if len(c.cidrs) == 2 && len(serviceCIDR.Spec.CIDRs) == 1 && c.cidrs[0] == serviceCIDR.Spec.CIDRs[0] {
klog.Infof("Updating default ServiceCIDR from single-stack (%v) to dual-stack (%v)", serviceCIDR.Spec.CIDRs, c.cidrs)
serviceCIDRcopy := serviceCIDR.DeepCopy()
serviceCIDRcopy.Spec.CIDRs = c.cidrs
_, err := c.client.NetworkingV1().ServiceCIDRs().Update(context.Background(), serviceCIDRcopy, metav1.UpdateOptions{})
if err != nil {
klog.Infof("The default ServiceCIDR can not be updated from %s to dual stack %v : %v", c.cidrs[0], c.cidrs, err)
c.eventRecorder.Eventf(serviceCIDR, v1.EventTypeWarning, "KubernetesDefaultServiceCIDRError", "The default ServiceCIDR can not be upgraded from %s to dual stack %v : %v", c.cidrs[0], c.cidrs, err)
}
} else {
c.syncStatus(serviceCIDR)
}
return nil
}
@ -175,18 +189,28 @@ func (c *Controller) syncStatus(serviceCIDR *networkingapiv1.ServiceCIDR) {
// This controller will set the Ready condition to true if the Ready condition
// does not exist and the CIDR values match this controller CIDR values.
sameConfig := reflect.DeepEqual(c.cidrs, serviceCIDR.Spec.CIDRs)
for _, condition := range serviceCIDR.Status.Conditions {
if condition.Type == networkingapiv1.ServiceCIDRConditionReady {
if condition.Status == metav1.ConditionTrue {
// ServiceCIDR is Ready and config matches this apiserver
// nothing else is required
if sameConfig {
return
}
} else {
if !c.reportedNotReadyCondition {
klog.InfoS("default ServiceCIDR condition Ready is not True, please validate your cluster's network configuration for this ServiceCIDR", "status", condition.Status, "reason", condition.Reason, "message", condition.Message)
c.eventRecorder.Eventf(serviceCIDR, v1.EventTypeWarning, condition.Reason, condition.Message)
c.reportedNotReadyCondition = true
}
return
}
klog.Infof("default ServiceCIDR condition Ready is not True: %v", condition.Status)
c.eventRecorder.Eventf(serviceCIDR, v1.EventTypeWarning, condition.Reason, condition.Message)
return
}
}
// set status to ready if the ServiceCIDR matches this configuration
if reflect.DeepEqual(c.cidrs, serviceCIDR.Spec.CIDRs) {
// No condition set, set status to ready if the ServiceCIDR matches this configuration
// otherwise, warn about it since the network configuration of the cluster is inconsistent
if sameConfig {
klog.Infof("Setting default ServiceCIDR condition Ready to True")
svcApplyStatus := networkingapiv1apply.ServiceCIDRStatus().WithConditions(
metav1apply.Condition().
@ -199,5 +223,9 @@ func (c *Controller) syncStatus(serviceCIDR *networkingapiv1.ServiceCIDR) {
klog.Infof("error updating default ServiceCIDR status: %v", errApply)
c.eventRecorder.Eventf(serviceCIDR, v1.EventTypeWarning, "KubernetesDefaultServiceCIDRError", "The default ServiceCIDR Status can not be set to Ready=True")
}
} else if !c.reportedMismatchedCIDRs {
klog.Infof("inconsistent ServiceCIDR status, global configuration: %v local configuration: %v, configure the flags to match current ServiceCIDR or manually delete the default ServiceCIDR", serviceCIDR.Spec.CIDRs, c.cidrs)
c.eventRecorder.Eventf(serviceCIDR, v1.EventTypeWarning, "KubernetesDefaultServiceCIDRInconsistent", "The default ServiceCIDR %v does not match the flag configurations %s", serviceCIDR.Spec.CIDRs, c.cidrs)
c.reportedMismatchedCIDRs = true
}
}

197
pkg/controlplane/controller/defaultservicecidr/default_servicecidr_controller_test.go
View File

@ -23,6 +23,7 @@ import (
"github.com/google/go-cmp/cmp"
networkingapiv1 "k8s.io/api/networking/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake"
k8stesting "k8s.io/client-go/testing"
@ -35,8 +36,13 @@ const (
defaultIPv6CIDR = "2001:db8::/64"
)
func newController(t *testing.T, objects []*networkingapiv1.ServiceCIDR) (*fake.Clientset, *Controller) {
client := fake.NewSimpleClientset()
func newController(t *testing.T, cidrsFromFlags []string, objects ...*networkingapiv1.ServiceCIDR) (*fake.Clientset, *Controller) {
var runtimeObjects []runtime.Object
for _, cidr := range objects {
runtimeObjects = append(runtimeObjects, cidr)
}
client := fake.NewSimpleClientset(runtimeObjects...)
informerFactory := informers.NewSharedInformerFactory(client, 0)
serviceCIDRInformer := informerFactory.Networking().V1().ServiceCIDRs()
@ -47,12 +53,11 @@ func newController(t *testing.T, objects []*networkingapiv1.ServiceCIDR) (*fake.
if err != nil {
t.Fatal(err)
}
}
c := &Controller{
client: client,
interval: time.Second,
cidrs: []string{defaultIPv4CIDR, defaultIPv6CIDR},
cidrs: cidrsFromFlags,
eventRecorder: record.NewFakeRecorder(100),
serviceCIDRLister: serviceCIDRInformer.Lister(),
serviceCIDRsSynced: func() bool { return true },
@ -151,13 +156,195 @@ func TestControllerSync(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
client, controller := newController(t, tc.cidrs)
client, controller := newController(t, []string{defaultIPv4CIDR, defaultIPv6CIDR}, tc.cidrs...)
controller.sync()
expectAction(t, client.Actions(), tc.actions)
})
}
}
func TestControllerSyncConversions(t *testing.T) {
testCases := []struct {
name string
controllerCIDRs []string
existingCIDR *networkingapiv1.ServiceCIDR
expectedAction [][]string // verb, resource, [subresource]
}{
{
name: "flags match ServiceCIDRs",
controllerCIDRs: []string{defaultIPv4CIDR, defaultIPv6CIDR},
existingCIDR: &networkingapiv1.ServiceCIDR{
ObjectMeta: metav1.ObjectMeta{
Name: DefaultServiceCIDRName,
},
Spec: networkingapiv1.ServiceCIDRSpec{
CIDRs: []string{defaultIPv4CIDR, defaultIPv6CIDR},
},
Status: networkingapiv1.ServiceCIDRStatus{}, // No conditions
},
expectedAction: [][]string{{"patch", "servicecidrs", "status"}},
},
{
name: "existing Ready=False condition, cidrs match -> no patch",
controllerCIDRs: []string{defaultIPv4CIDR, defaultIPv6CIDR},
existingCIDR: &networkingapiv1.ServiceCIDR{
ObjectMeta: metav1.ObjectMeta{
Name: DefaultServiceCIDRName,
},
Spec: networkingapiv1.ServiceCIDRSpec{
CIDRs: []string{defaultIPv4CIDR, defaultIPv6CIDR},
},
Status: networkingapiv1.ServiceCIDRStatus{
Conditions: []metav1.Condition{
{
Type: networkingapiv1.ServiceCIDRConditionReady,
Status: metav1.ConditionFalse,
Reason: "SomeReason",
},
},
},
},
expectedAction: [][]string{}, // No patch expected, just logs/events
},
{
name: "existing Ready=True condition -> no patch",
controllerCIDRs: []string{defaultIPv4CIDR, defaultIPv6CIDR},
existingCIDR: &networkingapiv1.ServiceCIDR{
ObjectMeta: metav1.ObjectMeta{
Name: DefaultServiceCIDRName,
},
Spec: networkingapiv1.ServiceCIDRSpec{
CIDRs: []string{defaultIPv4CIDR, defaultIPv6CIDR},
},
Status: networkingapiv1.ServiceCIDRStatus{
Conditions: []metav1.Condition{
{
Type: networkingapiv1.ServiceCIDRConditionReady,
Status: metav1.ConditionTrue,
},
},
},
},
expectedAction: [][]string{},
},
{
name: "ServiceCIDR being deleted -> no patch",
controllerCIDRs: []string{defaultIPv4CIDR, defaultIPv6CIDR},
existingCIDR: &networkingapiv1.ServiceCIDR{
ObjectMeta: metav1.ObjectMeta{
Name: DefaultServiceCIDRName,
DeletionTimestamp: ptr.To(metav1.Now()),
},
Spec: networkingapiv1.ServiceCIDRSpec{
CIDRs: []string{defaultIPv4CIDR, defaultIPv6CIDR},
},
Status: networkingapiv1.ServiceCIDRStatus{},
},
expectedAction: [][]string{},
},
{
name: "IPv4 to IPv4 IPv6 is ok",
controllerCIDRs: []string{defaultIPv4CIDR, defaultIPv6CIDR},
existingCIDR: &networkingapiv1.ServiceCIDR{
ObjectMeta: metav1.ObjectMeta{
Name: DefaultServiceCIDRName,
},
Spec: networkingapiv1.ServiceCIDRSpec{
CIDRs: []string{defaultIPv4CIDR}, // Existing has both
},
Status: networkingapiv1.ServiceCIDRStatus{},
},
expectedAction: [][]string{{"update", "servicecidrs"}},
},
{
name: "IPv4 to IPv6 IPv4 - switching primary IP family leaves in inconsistent state",
controllerCIDRs: []string{defaultIPv6CIDR, defaultIPv4CIDR},
existingCIDR: &networkingapiv1.ServiceCIDR{
ObjectMeta: metav1.ObjectMeta{
Name: DefaultServiceCIDRName,
},
Spec: networkingapiv1.ServiceCIDRSpec{
CIDRs: []string{defaultIPv4CIDR}, // Existing has both
},
Status: networkingapiv1.ServiceCIDRStatus{},
},
expectedAction: [][]string{},
},
{
name: "IPv6 to IPv6 IPv4",
controllerCIDRs: []string{defaultIPv6CIDR, defaultIPv4CIDR},
existingCIDR: &networkingapiv1.ServiceCIDR{
ObjectMeta: metav1.ObjectMeta{
Name: DefaultServiceCIDRName,
},
Spec: networkingapiv1.ServiceCIDRSpec{
CIDRs: []string{defaultIPv6CIDR}, // Existing has both
},
Status: networkingapiv1.ServiceCIDRStatus{},
},
expectedAction: [][]string{{"update", "servicecidrs"}},
},
{
name: "IPv6 to IPv4 IPv6 - switching primary IP family leaves in inconsistent state",
controllerCIDRs: []string{defaultIPv4CIDR, defaultIPv6CIDR},
existingCIDR: &networkingapiv1.ServiceCIDR{
ObjectMeta: metav1.ObjectMeta{
Name: DefaultServiceCIDRName,
},
Spec: networkingapiv1.ServiceCIDRSpec{
CIDRs: []string{defaultIPv6CIDR}, // Existing has both
},
Status: networkingapiv1.ServiceCIDRStatus{},
},
expectedAction: [][]string{},
},
{
name: "IPv6 IPv4 to IPv4 IPv6 - switching primary IP family leaves in inconsistent state",
controllerCIDRs: []string{defaultIPv4CIDR, defaultIPv6CIDR},
existingCIDR: &networkingapiv1.ServiceCIDR{
ObjectMeta: metav1.ObjectMeta{
Name: DefaultServiceCIDRName,
},
Spec: networkingapiv1.ServiceCIDRSpec{
CIDRs: []string{defaultIPv6CIDR, defaultIPv4CIDR}, // Existing has both
},
Status: networkingapiv1.ServiceCIDRStatus{},
},
expectedAction: [][]string{},
},
{
name: "IPv4 IPv6 to IPv4 - needs operator attention for the IPv6 remaining Services",
controllerCIDRs: []string{defaultIPv4CIDR},
existingCIDR: &networkingapiv1.ServiceCIDR{
ObjectMeta: metav1.ObjectMeta{
Name: DefaultServiceCIDRName,
},
Spec: networkingapiv1.ServiceCIDRSpec{
CIDRs: []string{defaultIPv4CIDR, defaultIPv6CIDR},
},
Status: networkingapiv1.ServiceCIDRStatus{},
},
expectedAction: [][]string{},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
// Initialize controller and client with the existing ServiceCIDR
client, controller := newController(t, tc.controllerCIDRs, tc.existingCIDR)
// Call the syncStatus method directly
err := controller.sync()
if err != nil {
t.Errorf("unexpected error: %v", err)
}
// Verify the expected actions
expectAction(t, client.Actions(), tc.expectedAction)
})
}
}
func expectAction(t *testing.T, actions []k8stesting.Action, expected [][]string) {
t.Helper()
if len(actions) != len(expected) {

3
pkg/registry/networking/servicecidr/strategy.go
View File

@ -106,8 +106,7 @@ func (serviceCIDRStrategy) WarningsOnCreate(ctx context.Context, obj runtime.Obj
func (serviceCIDRStrategy) ValidateUpdate(ctx context.Context, new, old runtime.Object) field.ErrorList {
newServiceCIDR := new.(*networking.ServiceCIDR)
oldServiceCIDR := old.(*networking.ServiceCIDR)
errList := validation.ValidateServiceCIDR(newServiceCIDR)
errList = append(errList, validation.ValidateServiceCIDRUpdate(newServiceCIDR, oldServiceCIDR)...)
errList := validation.ValidateServiceCIDRUpdate(newServiceCIDR, oldServiceCIDR)
return errList
}

6
pkg/registry/networking/servicecidr/strategy_test.go
View File

@ -47,15 +47,15 @@ func TestServiceCIDRStrategy(t *testing.T) {
errors := Strategy.Validate(context.TODO(), obj)
if len(errors) != 2 {
t.Errorf("Expected 2 validation errors for invalid object, got %d", len(errors))
t.Errorf("Expected 2 validation errors for invalid object, got %d : %v", len(errors), errors)
}
oldObj := newServiceCIDR()
newObj := oldObj.DeepCopy()
newObj.Spec.CIDRs = []string{"bad cidr"}
errors = Strategy.ValidateUpdate(context.TODO(), newObj, oldObj)
if len(errors) != 2 {
t.Errorf("Expected 2 validation errors for invalid update, got %d", len(errors))
if len(errors) != 1 {
t.Errorf("Expected 1 validation error for invalid update, got %d : %v", len(errors), errors)
}
}

7
test/conformance/testdata/conformance.yaml vendored
View File

@ -511,7 +511,7 @@
file: test/e2e/apimachinery/garbage_collector.go
- testname: Garbage Collector, delete replication controller, after owned pods
codename: '[sig-api-machinery] Garbage collector should keep the rc around until
all its pods are deleted if the deleteOptions says so [Conformance]'
all its pods are deleted if the deleteOptions says so [Serial] [Conformance]'
description: Create a replication controller with maximum allocatable Pods between
10 and 100 replicas. Once RC is created and the all Pods are created, delete RC
with deleteOptions.PropagationPolicy set to Foreground. Deleting the Replication
@ -528,7 +528,8 @@
file: test/e2e/apimachinery/garbage_collector.go
- testname: Garbage Collector, multiple owners
codename: '[sig-api-machinery] Garbage collector should not delete dependents that
have both valid owner and owner that''s waiting for dependents to be deleted [Conformance]'
have both valid owner and owner that''s waiting for dependents to be deleted [Serial]
[Conformance]'
description: Create a replication controller RC1, with maximum allocatable Pods
between 10 and 100 replicas. Create second replication controller RC2 and set
RC2 as owner for half of those replicas. Once RC1 is created and the all Pods
@ -549,7 +550,7 @@
file: test/e2e/apimachinery/garbage_collector.go
- testname: Garbage Collector, delete replication controller, propagation policy orphan
codename: '[sig-api-machinery] Garbage collector should orphan pods created by rc
if delete options say so [Conformance]'
if delete options say so [Serial] [Conformance]'
description: Create a replication controller with maximum allocatable Pods between
10 and 100 replicas. Once RC is created and the all Pods are created, delete RC
with deleteOptions.PropagationPolicy set to Orphan. Deleting the Replication Controller

20
test/e2e/apimachinery/garbage_collector.go
View File

@ -54,23 +54,31 @@ import (
// estimateMaximumPods estimates how many pods the cluster can handle
// with some wiggle room, to prevent pods being unable to schedule due
// to max pod constraints.
//
// Tests that call this should use framework.WithSerial() because they're not
// safe to run concurrently as they consume a large number of pods.
func estimateMaximumPods(ctx context.Context, c clientset.Interface, min, max int32) int32 {
nodes, err := e2enode.GetReadySchedulableNodes(ctx, c)
framework.ExpectNoError(err)
availablePods := int32(0)
// estimate some reasonable overhead per-node for pods that are non-test
const daemonSetReservedPods = 10
for _, node := range nodes.Items {
if q, ok := node.Status.Allocatable["pods"]; ok {
if num, ok := q.AsInt64(); ok {
availablePods += int32(num)
if num > daemonSetReservedPods {
availablePods += int32(num - daemonSetReservedPods)
}
continue
}
}
// best guess per node, since default maxPerCore is 10 and most nodes have at least
// Only when we fail to obtain the number, we fall back to a best guess
// per node. Since default maxPerCore is 10 and most nodes have at least
// one core.
availablePods += 10
}
//avoid creating exactly max pods
// avoid creating exactly max pods
availablePods = int32(float32(availablePods) * 0.5)
// bound the top and bottom
if availablePods > max {
@ -377,7 +385,7 @@ var _ = SIGDescribe("Garbage collector", func() {
Testname: Garbage Collector, delete replication controller, propagation policy orphan
Description: Create a replication controller with maximum allocatable Pods between 10 and 100 replicas. Once RC is created and the all Pods are created, delete RC with deleteOptions.PropagationPolicy set to Orphan. Deleting the Replication Controller MUST cause pods created by that RC to be orphaned.
*/
framework.ConformanceIt("should orphan pods created by rc if delete options say so", func(ctx context.Context) {
framework.ConformanceIt("should orphan pods created by rc if delete options say so", framework.WithSerial(), func(ctx context.Context) {
clientSet := f.ClientSet
rcClient := clientSet.CoreV1().ReplicationControllers(f.Namespace.Name)
podClient := clientSet.CoreV1().Pods(f.Namespace.Name)
@ -636,7 +644,7 @@ var _ = SIGDescribe("Garbage collector", func() {
Testname: Garbage Collector, delete replication controller, after owned pods
Description: Create a replication controller with maximum allocatable Pods between 10 and 100 replicas. Once RC is created and the all Pods are created, delete RC with deleteOptions.PropagationPolicy set to Foreground. Deleting the Replication Controller MUST cause pods created by that RC to be deleted before the RC is deleted.
*/
framework.ConformanceIt("should keep the rc around until all its pods are deleted if the deleteOptions says so", func(ctx context.Context) {
framework.ConformanceIt("should keep the rc around until all its pods are deleted if the deleteOptions says so", framework.WithSerial(), func(ctx context.Context) {
clientSet := f.ClientSet
rcClient := clientSet.CoreV1().ReplicationControllers(f.Namespace.Name)
podClient := clientSet.CoreV1().Pods(f.Namespace.Name)
@ -711,7 +719,7 @@ var _ = SIGDescribe("Garbage collector", func() {
Testname: Garbage Collector, multiple owners
Description: Create a replication controller RC1, with maximum allocatable Pods between 10 and 100 replicas. Create second replication controller RC2 and set RC2 as owner for half of those replicas. Once RC1 is created and the all Pods are created, delete RC1 with deleteOptions.PropagationPolicy set to Foreground. Half of the Pods that has RC2 as owner MUST not be deleted or have a deletion timestamp. Deleting the Replication Controller MUST not delete Pods that are owned by multiple replication controllers.
*/
framework.ConformanceIt("should not delete dependents that have both valid owner and owner that's waiting for dependents to be deleted", func(ctx context.Context) {
framework.ConformanceIt("should not delete dependents that have both valid owner and owner that's waiting for dependents to be deleted", framework.WithSerial(), func(ctx context.Context) {
clientSet := f.ClientSet
rcClient := clientSet.CoreV1().ReplicationControllers(f.Namespace.Name)
podClient := clientSet.CoreV1().Pods(f.Namespace.Name)

13
test/e2e/framework/node/helper.go
View File

@ -188,10 +188,15 @@ func AddExtendedResource(ctx context.Context, clientSet clientset.Interface, nod
extendedResourceList := v1.ResourceList{
extendedResource: extendedResourceQuantity,
}
patchPayload, err := json.Marshal(v1.Node{
Status: v1.NodeStatus{
Capacity: extendedResourceList,
Allocatable: extendedResourceList,
// This is a workaround for the fact that we shouldn't marshal a Node struct to JSON
// because it wipes out some fields from node status like the daemonEndpoints and
// nodeInfo which should not be changed at this time. We need to use a map instead.
// See https://github.com/kubernetes/kubernetes/issues/131229
patchPayload, err := json.Marshal(map[string]any{
"status": map[string]any{
"capacity": extendedResourceList,
"allocatable": extendedResourceList,
},
})
framework.ExpectNoError(err, "Failed to marshal node JSON")

46
test/integration/dualstack/dualstack_test.go
View File

@ -1484,7 +1484,7 @@ func TestUpgradeServicePreferToDualStack(t *testing.T) {
sharedEtcd := framework.SharedEtcd()
tCtx := ktesting.Init(t)
// Create an IPv4 only dual stack control-plane
// Create an IPv4 only control-plane
apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions()
s := kubeapiservertesting.StartTestServerOrDie(t,
apiServerOptions,
@ -1532,6 +1532,10 @@ func TestUpgradeServicePreferToDualStack(t *testing.T) {
},
}
// create a copy of the service so we can test creating it again after reconfiguring the control plane
svcDual := svc.DeepCopy()
svcDual.Name = "svc-dual"
// create the service
_, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{})
if err != nil {
@ -1582,9 +1586,25 @@ func TestUpgradeServicePreferToDualStack(t *testing.T) {
if err = validateServiceAndClusterIPFamily(svc, []v1.IPFamily{v1.IPv4Protocol}); err != nil {
t.Fatalf("Unexpected error validating the service %s %v", svc.Name, err)
}
// validate that new services created with prefer dual are now dual stack
// create the service
_, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svcDual, metav1.CreateOptions{})
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
// validate the service was created correctly
svcDual, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svcDual.Name, metav1.GetOptions{})
if err != nil {
t.Fatalf("Unexpected error to get the service %s %v", svcDual.Name, err)
}
// service should be dual stack
if err = validateServiceAndClusterIPFamily(svcDual, []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}); err != nil {
t.Fatalf("Unexpected error validating the service %s %v", svcDual.Name, err)
}
}
func TestDowngradeServicePreferToDualStack(t *testing.T) {
func TestDowngradeServicePreferFromDualStack(t *testing.T) {
tCtx := ktesting.Init(t)
// Create a dual stack control-plane
@ -1634,6 +1654,11 @@ func TestDowngradeServicePreferToDualStack(t *testing.T) {
},
},
}
// create a copy of the service so we can test creating it again after reconfiguring the control plane
svcSingle := svc.DeepCopy()
svcSingle.Name = "svc-single"
// create the service
_, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{})
if err != nil {
@ -1684,6 +1709,23 @@ func TestDowngradeServicePreferToDualStack(t *testing.T) {
if err = validateServiceAndClusterIPFamily(svc, []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}); err != nil {
t.Fatalf("Unexpected error validating the service %s %v", svc.Name, err)
}
// validate that new services created with prefer dual are now single stack
// create the service
_, err = client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svcSingle, metav1.CreateOptions{})
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
// validate the service was created correctly
svcSingle, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svcSingle.Name, metav1.GetOptions{})
if err != nil {
t.Fatalf("Unexpected error to get the service %s %v", svcSingle.Name, err)
}
// service should be single stack
if err = validateServiceAndClusterIPFamily(svcSingle, []v1.IPFamily{v1.IPv4Protocol}); err != nil {
t.Fatalf("Unexpected error validating the service %s %v", svcSingle.Name, err)
}
}
type serviceMergePatch struct {

319
test/integration/servicecidr/migration_test.go
View File

@ -19,6 +19,8 @@ package servicecidr
import (
"context"
"fmt"
"reflect"
"strings"
"testing"
"time"
@ -29,6 +31,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
"k8s.io/kubernetes/pkg/controller/servicecidrs"
"k8s.io/kubernetes/pkg/controlplane/controller/defaultservicecidr"
@ -293,3 +296,319 @@ func TestMigrateServiceCIDR(t *testing.T) {
}
}
// TestServiceCIDRMigrationScenarios tests various migration paths for ServiceCIDRs.
func TestServiceCIDRMigrationScenarios(t *testing.T) {
ipv4CIDRSmall := "10.0.0.0/29" // 6 IPs
ipv4CIDRBig := "10.1.0.0/16"
ipv6CIDRSmall := "2001:db8:1::/125" // 6 IPs
ipv6CIDRBig := "2001:db8:2::/112"
testCases := []struct {
name string
initialCIDRs []string
migratedCIDRs []string
preMigrationSvcName string
postMigrationSvcName string
expectedPostMigrationSvcError bool // Changing the primary IP family and retaining the old allocator
expectInconsistentState bool // New Service CIDR configured by flags are not applied
}{
// --- No Change ---
{
name: "IPv4 -> IPv4 (no change)",
initialCIDRs: []string{ipv4CIDRSmall},
migratedCIDRs: []string{ipv4CIDRSmall},
preMigrationSvcName: "svc-pre-v4-v4",
postMigrationSvcName: "svc-post-v4-v4",
},
{
name: "IPv6 -> IPv6 (no change)",
initialCIDRs: []string{ipv6CIDRSmall},
migratedCIDRs: []string{ipv6CIDRSmall},
preMigrationSvcName: "svc-pre-v6-v6",
postMigrationSvcName: "svc-post-v6-v6",
},
{
name: "IPv4,IPv6 -> IPv4,IPv6 (no change)",
initialCIDRs: []string{ipv4CIDRSmall, ipv6CIDRSmall},
migratedCIDRs: []string{ipv4CIDRSmall, ipv6CIDRSmall},
preMigrationSvcName: "svc-pre-v4v6-v4v6",
postMigrationSvcName: "svc-post-v4v6-v4v6",
},
{
name: "IPv6,IPv4 -> IPv6,IPv4 (no change)",
initialCIDRs: []string{ipv6CIDRSmall, ipv4CIDRSmall},
migratedCIDRs: []string{ipv6CIDRSmall, ipv4CIDRSmall},
preMigrationSvcName: "svc-pre-v6v4-v6v4",
postMigrationSvcName: "svc-post-v6v4-v6v4",
},
// --- Valid Upgrades ---
{
name: "IPv4 -> IPv4,IPv6 (upgrade)",
initialCIDRs: []string{ipv4CIDRSmall},
migratedCIDRs: []string{ipv4CIDRSmall, ipv6CIDRBig},
preMigrationSvcName: "svc-pre-v4-v4v6",
postMigrationSvcName: "svc-post-v4-v4v6",
},
{
name: "IPv6 -> IPv6,IPv4 (upgrade)",
initialCIDRs: []string{ipv6CIDRSmall},
migratedCIDRs: []string{ipv6CIDRSmall, ipv4CIDRBig},
preMigrationSvcName: "svc-pre-v6-v6v4",
postMigrationSvcName: "svc-post-v6-v6v4",
},
// --- Invalid Migrations (Require manual intervention) ---
{
name: "IPv4,IPv6 -> IPv6,IPv4 (change primary)",
initialCIDRs: []string{ipv4CIDRSmall, ipv6CIDRSmall},
migratedCIDRs: []string{ipv6CIDRSmall, ipv4CIDRSmall},
preMigrationSvcName: "svc-pre-v4v6-v6v4",
postMigrationSvcName: "svc-post-v4v6-v6v4",
expectedPostMigrationSvcError: true,
expectInconsistentState: true,
},
{
name: "IPv6,IPv4 -> IPv4,IPv6 (change primary)",
initialCIDRs: []string{ipv6CIDRSmall, ipv4CIDRSmall},
migratedCIDRs: []string{ipv4CIDRSmall, ipv6CIDRSmall},
preMigrationSvcName: "svc-pre-v6v4-v4v6",
postMigrationSvcName: "svc-post-v6v4-v4v6",
expectedPostMigrationSvcError: true,
expectInconsistentState: true,
},
{
name: "IPv4,IPv6 -> IPv4 (downgrade)",
initialCIDRs: []string{ipv4CIDRSmall, ipv6CIDRSmall},
migratedCIDRs: []string{ipv4CIDRSmall},
preMigrationSvcName: "svc-pre-v4v6-v4",
postMigrationSvcName: "svc-post-v4v6-v4",
expectInconsistentState: true,
},
{
name: "IPv4,IPv6 -> IPv6 (downgrade)",
initialCIDRs: []string{ipv4CIDRSmall, ipv6CIDRSmall},
migratedCIDRs: []string{ipv6CIDRSmall},
preMigrationSvcName: "svc-pre-v4v6-v6",
postMigrationSvcName: "svc-post-v4v6-v6",
expectedPostMigrationSvcError: true,
expectInconsistentState: true,
},
{
name: "IPv4 -> IPv6 (change family)",
initialCIDRs: []string{ipv4CIDRSmall},
migratedCIDRs: []string{ipv6CIDRSmall},
preMigrationSvcName: "svc-pre-v4-v6",
postMigrationSvcName: "svc-post-v4-v6",
expectedPostMigrationSvcError: true,
expectInconsistentState: true,
},
{
name: "IPv6 -> IPv4 (change family)",
initialCIDRs: []string{ipv6CIDRSmall},
migratedCIDRs: []string{ipv4CIDRSmall},
preMigrationSvcName: "svc-pre-v6-v4",
postMigrationSvcName: "svc-post-v6-v4",
expectedPostMigrationSvcError: true,
expectInconsistentState: true,
},
{
name: "IPv4 -> IPv6,IPv4 (upgrade, change primary)",
initialCIDRs: []string{ipv4CIDRSmall},
migratedCIDRs: []string{ipv6CIDRBig, ipv4CIDRSmall}, // Change primary during upgrade
preMigrationSvcName: "svc-pre-v4-v6v4",
postMigrationSvcName: "svc-post-v4-v6v4",
expectedPostMigrationSvcError: true,
expectInconsistentState: true,
},
{
name: "IPv6 -> IPv4,IPv6 (upgrade, change primary)",
initialCIDRs: []string{ipv6CIDRSmall},
migratedCIDRs: []string{ipv4CIDRBig, ipv6CIDRSmall}, // Change primary during upgrade
preMigrationSvcName: "svc-pre-v6-v4v6",
postMigrationSvcName: "svc-post-v6-v4v6",
expectedPostMigrationSvcError: true,
expectInconsistentState: true,
},
}
for i, tc := range testCases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
tCtx := ktesting.Init(t)
etcdOptions := framework.SharedEtcd()
apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions()
resyncPeriod := 12 * time.Hour
// --- Initial Setup ---
initialFlags := []string{
"--service-cluster-ip-range=" + strings.Join(tc.initialCIDRs, ","),
"--advertise-address=" + strings.Split(tc.initialCIDRs[0], "/")[0], // the advertise address MUST match the cluster primary ip family
"--disable-admission-plugins=ServiceAccount",
// fmt.Sprintf("--feature-gates=%s=true,%s=true", features.MultiCIDRServiceAllocator, features.DisableAllocatorDualWrite),
}
t.Logf("Starting API server with CIDRs: %v", tc.initialCIDRs)
s1 := kubeapiservertesting.StartTestServerOrDie(t, apiServerOptions, initialFlags, etcdOptions)
client1, err := clientset.NewForConfig(s1.ClientConfig)
if err != nil {
s1.TearDownFn()
t.Fatalf("Failed to create client for initial server: %v", err)
}
ns := framework.CreateNamespaceOrDie(client1, fmt.Sprintf("migrate-%d", i), t)
informers1 := informers.NewSharedInformerFactory(client1, resyncPeriod)
controllerCtx1, cancelController1 := context.WithCancel(tCtx)
go servicecidrs.NewController(
controllerCtx1,
informers1.Networking().V1().ServiceCIDRs(),
informers1.Networking().V1().IPAddresses(),
client1,
).Run(controllerCtx1, 5)
informers1.Start(controllerCtx1.Done())
informers1.WaitForCacheSync(controllerCtx1.Done())
// Wait for default ServiceCIDR to be ready
if err := waitForServiceCIDRState(tCtx, client1, tc.initialCIDRs, true); err != nil {
s1.TearDownFn()
cancelController1()
t.Fatalf("Initial default ServiceCIDR did not become ready: %v", err)
}
// Create pre-migration service
preSvc, err := client1.CoreV1().Services(ns.Name).Create(tCtx, makeService(tc.preMigrationSvcName), metav1.CreateOptions{})
if err != nil {
s1.TearDownFn()
cancelController1()
t.Fatalf("Failed to create pre-migration service: %v", err)
}
t.Logf("Pre-migration service %s created with ClusterIPs: %v", preSvc.Name, preSvc.Spec.ClusterIPs)
// Basic verification of pre-migration service IP
if len(preSvc.Spec.ClusterIPs) == 0 {
s1.TearDownFn()
cancelController1()
t.Fatalf("Pre-migration service %s has no ClusterIPs", preSvc.Name)
}
if !cidrContainsIP(tc.initialCIDRs[0], preSvc.Spec.ClusterIPs[0]) {
s1.TearDownFn()
cancelController1()
t.Fatalf("Pre-migration service %s primary IP %s not in expected range %s", preSvc.Name, preSvc.Spec.ClusterIPs[0], tc.initialCIDRs[0])
}
// --- Migration ---
t.Logf("Shutting down initial API server and controller")
cancelController1()
s1.TearDownFn()
t.Logf("Starting migrated API server with CIDRs: %v", tc.migratedCIDRs)
migratedFlags := []string{
"--service-cluster-ip-range=" + strings.Join(tc.migratedCIDRs, ","),
"--advertise-address=" + strings.Split(tc.migratedCIDRs[0], "/")[0], // the advertise address MUST match the cluster configured primary ip family
"--disable-admission-plugins=ServiceAccount",
// fmt.Sprintf("--feature-gates=%s=true,%s=true", features.MultiCIDRServiceAllocator, features.DisableAllocatorDualWrite),
}
s2 := kubeapiservertesting.StartTestServerOrDie(t, apiServerOptions, migratedFlags, etcdOptions)
defer s2.TearDownFn() // Ensure cleanup even on test failure
client2, err := clientset.NewForConfig(s2.ClientConfig)
if err != nil {
t.Fatalf("Failed to create client for migrated server: %v", err)
}
defer framework.DeleteNamespaceOrDie(client2, ns, t)
informers2 := informers.NewSharedInformerFactory(client2, resyncPeriod)
controllerCtx2, cancelController2 := context.WithCancel(tCtx)
defer cancelController2() // Ensure controller context is cancelled
go servicecidrs.NewController(
controllerCtx2,
informers2.Networking().V1().ServiceCIDRs(),
informers2.Networking().V1().IPAddresses(),
client2,
).Run(controllerCtx2, 5)
informers2.Start(controllerCtx2.Done())
informers2.WaitForCacheSync(controllerCtx2.Done())
// Wait for default ServiceCIDR to reflect migrated state
// For inconsistent states, we expect to keep existing CIDRs.
expectedCIDRs := tc.migratedCIDRs
if tc.expectInconsistentState {
expectedCIDRs = tc.initialCIDRs
}
if err := waitForServiceCIDRState(tCtx, client2, expectedCIDRs, true); err != nil {
t.Fatalf("Migrated default ServiceCIDR did not reach expected state : %v", err)
}
// --- Post-Migration Verification ---
// Verify pre-migration service still exists and retains its IP(s)
preSvcMigrated, err := client2.CoreV1().Services(ns.Name).Get(tCtx, tc.preMigrationSvcName, metav1.GetOptions{})
if err != nil {
t.Fatalf("Failed to get pre-migration service after migration: %v", err)
}
if !reflect.DeepEqual(preSvcMigrated.Spec.ClusterIPs, preSvc.Spec.ClusterIPs) {
t.Errorf("Pre-migration service %s ClusterIPs changed after migration. Before: %v, After: %v",
preSvcMigrated.Name, preSvc.Spec.ClusterIPs, preSvcMigrated.Spec.ClusterIPs)
}
// Create post-migration service
postSvc, err := client2.CoreV1().Services(ns.Name).Create(tCtx, makeService(tc.postMigrationSvcName), metav1.CreateOptions{})
if err != nil && !tc.expectedPostMigrationSvcError {
t.Fatalf("Failed to create post-migration service: %v", err)
} else if err == nil && tc.expectedPostMigrationSvcError {
return
}
t.Logf("Post-migration service %s created with ClusterIPs: %v, Families: %v", postSvc.Name, postSvc.Spec.ClusterIPs, postSvc.Spec.IPFamilies)
// Check if IPs are within the migrated CIDR ranges
if len(postSvc.Spec.ClusterIPs) > 0 && !cidrContainsIP(expectedCIDRs[0], postSvc.Spec.ClusterIPs[0]) {
t.Errorf("Post-migration service %s primary IP %s not in expected range %s", postSvc.Name, postSvc.Spec.ClusterIPs[0], expectedCIDRs[0])
}
})
}
}
// waitForServiceCIDRState waits for the named ServiceCIDR to exist, match the expected CIDRs,
// and have the specified Ready condition status.
func waitForServiceCIDRState(ctx context.Context, client clientset.Interface, expectedCIDRs []string, expectReady bool) error {
pollCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
return wait.PollUntilContextCancel(pollCtx, 500*time.Millisecond, true, func(ctx context.Context) (bool, error) {
cidr, err := client.NetworkingV1().ServiceCIDRs().Get(ctx, defaultservicecidr.DefaultServiceCIDRName, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
return true, fmt.Errorf("default ServiceCIDR must exist")
}
return false, nil
}
// Check CIDRs match
if !reflect.DeepEqual(cidr.Spec.CIDRs, expectedCIDRs) {
klog.Infof("Waiting for ServiceCIDR %s CIDRs to match %v, current: %v", defaultservicecidr.DefaultServiceCIDRName, expectedCIDRs, cidr.Spec.CIDRs)
return false, nil
}
// Check Ready condition
isReady := false
foundReadyCondition := false
for _, condition := range cidr.Status.Conditions {
if condition.Type == networkingv1.ServiceCIDRConditionReady {
foundReadyCondition = true
isReady = condition.Status == metav1.ConditionTrue
break
}
}
if !foundReadyCondition && expectReady {
klog.Infof("Waiting for ServiceCIDR %s Ready condition to be set...", defaultservicecidr.DefaultServiceCIDRName)
return false, nil // Ready condition not found yet
}
if isReady != expectReady {
klog.Infof("Waiting for ServiceCIDR %s Ready condition to be %v, current: %v", defaultservicecidr.DefaultServiceCIDRName, expectReady, isReady)
return false, nil // Ready condition doesn't match expectation
}
klog.Infof("ServiceCIDR %s reached desired state (Ready: %v, CIDRs: %v)", defaultservicecidr.DefaultServiceCIDRName, expectReady, expectedCIDRs)
return true, nil // All conditions met
})
}