Merge pull request #129081 from stlaz/fg_remote_uid

featuregate UID in RequestHeader authenticator
This commit is contained in:
Kubernetes Prow Robot 2024-12-04 23:43:06 +00:00 committed by GitHub
commit 1504f10e79
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 345 additions and 65 deletions

View File

@ -35,7 +35,9 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/authentication/request/headerrequest"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
utilfeature "k8s.io/apiserver/pkg/util/feature"
corev1informers "k8s.io/client-go/informers/core/v1"
"k8s.io/client-go/kubernetes"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
@ -262,10 +264,13 @@ func getConfigMapDataFor(authenticationInfo ClusterAuthenticationInfo) (map[stri
if err != nil {
return nil, err
}
data["requestheader-uid-headers"], err = jsonSerializeStringSlice(authenticationInfo.RequestHeaderUIDHeaders.Value())
if err != nil {
return nil, err
if utilfeature.DefaultFeatureGate.Enabled(features.RemoteRequestHeaderUID) && len(authenticationInfo.RequestHeaderUIDHeaders.Value()) > 0 {
data["requestheader-uid-headers"], err = jsonSerializeStringSlice(authenticationInfo.RequestHeaderUIDHeaders.Value())
if err != nil {
return nil, err
}
}
data["requestheader-group-headers"], err = jsonSerializeStringSlice(authenticationInfo.RequestHeaderGroupHeaders.Value())
if err != nil {
return nil, err
@ -305,9 +310,12 @@ func getClusterAuthenticationInfoFor(data map[string]string) (ClusterAuthenticat
if err != nil {
return ClusterAuthenticationInfo{}, err
}
ret.RequestHeaderUIDHeaders, err = jsonDeserializeStringSlice(data["requestheader-uid-headers"])
if err != nil {
return ClusterAuthenticationInfo{}, err
if utilfeature.DefaultFeatureGate.Enabled(features.RemoteRequestHeaderUID) {
ret.RequestHeaderUIDHeaders, err = jsonDeserializeStringSlice(data["requestheader-uid-headers"])
if err != nil {
return ClusterAuthenticationInfo{}, err
}
}
if caBundle := data["requestheader-client-ca-file"]; len(caBundle) > 0 {

View File

@ -30,11 +30,14 @@ import (
"k8s.io/apimachinery/pkg/util/dump"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apiserver/pkg/authentication/request/headerrequest"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/kubernetes/fake"
corev1listers "k8s.io/client-go/listers/core/v1"
clienttesting "k8s.io/client-go/testing"
"k8s.io/client-go/tools/cache"
featuregatetesting "k8s.io/component-base/featuregate/testing"
)
var (
@ -95,6 +98,7 @@ func TestWriteClientCAs(t *testing.T) {
preexistingObjs []runtime.Object
expectedConfigMaps map[string]*corev1.ConfigMap
expectCreate bool
uidGate bool
}{
{
name: "basic",
@ -107,6 +111,32 @@ func TestWriteClientCAs(t *testing.T) {
RequestHeaderCA: anotherRandomCAProvider,
RequestHeaderAllowedNames: headerrequest.StaticStringSlice{"first", "second"},
},
expectedConfigMaps: map[string]*corev1.ConfigMap{
"extension-apiserver-authentication": {
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
Data: map[string]string{
"client-ca-file": string(someRandomCA),
"requestheader-username-headers": `["alfa","bravo","charlie"]`,
"requestheader-group-headers": `["delta"]`,
"requestheader-extra-headers-prefix": `["echo","foxtrot"]`,
"requestheader-client-ca-file": string(anotherRandomCA),
"requestheader-allowed-names": `["first","second"]`,
},
},
},
expectCreate: true,
},
{
name: "basic with feature gate",
clusterAuthInfo: ClusterAuthenticationInfo{
ClientCA: someRandomCAProvider,
RequestHeaderUsernameHeaders: headerrequest.StaticStringSlice{"alfa", "bravo", "charlie"},
RequestHeaderUIDHeaders: headerrequest.StaticStringSlice{"golf", "hotel", "india"},
RequestHeaderGroupHeaders: headerrequest.StaticStringSlice{"delta"},
RequestHeaderExtraHeaderPrefixes: headerrequest.StaticStringSlice{"echo", "foxtrot"},
RequestHeaderCA: anotherRandomCAProvider,
RequestHeaderAllowedNames: headerrequest.StaticStringSlice{"first", "second"},
},
expectedConfigMaps: map[string]*corev1.ConfigMap{
"extension-apiserver-authentication": {
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
@ -122,6 +152,7 @@ func TestWriteClientCAs(t *testing.T) {
},
},
expectCreate: true,
uidGate: true,
},
{
name: "skip extension-apiserver-authentication",
@ -134,7 +165,6 @@ func TestWriteClientCAs(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
Data: map[string]string{
"requestheader-username-headers": `[]`,
"requestheader-uid-headers": `[]`,
"requestheader-group-headers": `[]`,
"requestheader-extra-headers-prefix": `[]`,
"requestheader-client-ca-file": string(anotherRandomCA),
@ -169,7 +199,6 @@ func TestWriteClientCAs(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
Data: map[string]string{
"requestheader-username-headers": `[]`,
"requestheader-uid-headers": `[]`,
"requestheader-group-headers": `[]`,
"requestheader-extra-headers-prefix": `[]`,
"requestheader-client-ca-file": string(anotherRandomCA),
@ -205,7 +234,6 @@ func TestWriteClientCAs(t *testing.T) {
name: "overwrite extension-apiserver-authentication requestheader",
clusterAuthInfo: ClusterAuthenticationInfo{
RequestHeaderUsernameHeaders: headerrequest.StaticStringSlice{},
RequestHeaderUIDHeaders: headerrequest.StaticStringSlice{},
RequestHeaderGroupHeaders: headerrequest.StaticStringSlice{},
RequestHeaderExtraHeaderPrefixes: headerrequest.StaticStringSlice{},
RequestHeaderCA: anotherRandomCAProvider,
@ -216,7 +244,6 @@ func TestWriteClientCAs(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
Data: map[string]string{
"requestheader-username-headers": `[]`,
"requestheader-uid-headers": `[]`,
"requestheader-group-headers": `[]`,
"requestheader-extra-headers-prefix": `[]`,
"requestheader-client-ca-file": string(someRandomCA),
@ -229,7 +256,6 @@ func TestWriteClientCAs(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
Data: map[string]string{
"requestheader-username-headers": `[]`,
"requestheader-uid-headers": `[]`,
"requestheader-group-headers": `[]`,
"requestheader-extra-headers-prefix": `[]`,
"requestheader-client-ca-file": string(someRandomCA) + string(anotherRandomCA),
@ -260,7 +286,6 @@ func TestWriteClientCAs(t *testing.T) {
name: "skip on no change",
clusterAuthInfo: ClusterAuthenticationInfo{
RequestHeaderUsernameHeaders: headerrequest.StaticStringSlice{},
RequestHeaderUIDHeaders: headerrequest.StaticStringSlice{},
RequestHeaderGroupHeaders: headerrequest.StaticStringSlice{},
RequestHeaderExtraHeaderPrefixes: headerrequest.StaticStringSlice{},
RequestHeaderCA: anotherRandomCAProvider,
@ -271,7 +296,6 @@ func TestWriteClientCAs(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
Data: map[string]string{
"requestheader-username-headers": `[]`,
"requestheader-uid-headers": `[]`,
"requestheader-group-headers": `[]`,
"requestheader-extra-headers-prefix": `[]`,
"requestheader-client-ca-file": string(anotherRandomCA),
@ -282,10 +306,126 @@ func TestWriteClientCAs(t *testing.T) {
expectedConfigMaps: map[string]*corev1.ConfigMap{},
expectCreate: false,
},
{
name: "drop uid without feature gate",
clusterAuthInfo: ClusterAuthenticationInfo{
RequestHeaderUsernameHeaders: headerrequest.StaticStringSlice{},
RequestHeaderUIDHeaders: headerrequest.StaticStringSlice{"panda"},
RequestHeaderGroupHeaders: headerrequest.StaticStringSlice{},
RequestHeaderExtraHeaderPrefixes: headerrequest.StaticStringSlice{},
RequestHeaderCA: anotherRandomCAProvider,
RequestHeaderAllowedNames: headerrequest.StaticStringSlice{},
},
preexistingObjs: []runtime.Object{
&corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
Data: map[string]string{
"requestheader-username-headers": `[]`,
"requestheader-uid-headers": `["snorlax"]`,
"requestheader-group-headers": `[]`,
"requestheader-extra-headers-prefix": `[]`,
"requestheader-client-ca-file": string(anotherRandomCA),
"requestheader-allowed-names": `[]`,
},
},
},
expectedConfigMaps: map[string]*corev1.ConfigMap{
"extension-apiserver-authentication": {
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
Data: map[string]string{
"requestheader-username-headers": `[]`,
"requestheader-group-headers": `[]`,
"requestheader-extra-headers-prefix": `[]`,
"requestheader-client-ca-file": string(anotherRandomCA),
"requestheader-allowed-names": `[]`,
},
},
},
expectCreate: false,
},
{
name: "add uid with feature gate",
clusterAuthInfo: ClusterAuthenticationInfo{
RequestHeaderUsernameHeaders: headerrequest.StaticStringSlice{},
RequestHeaderUIDHeaders: headerrequest.StaticStringSlice{"panda"},
RequestHeaderGroupHeaders: headerrequest.StaticStringSlice{},
RequestHeaderExtraHeaderPrefixes: headerrequest.StaticStringSlice{},
RequestHeaderCA: anotherRandomCAProvider,
RequestHeaderAllowedNames: headerrequest.StaticStringSlice{},
},
preexistingObjs: []runtime.Object{
&corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
Data: map[string]string{
"requestheader-username-headers": `[]`,
"requestheader-group-headers": `[]`,
"requestheader-extra-headers-prefix": `[]`,
"requestheader-client-ca-file": string(anotherRandomCA),
"requestheader-allowed-names": `[]`,
},
},
},
expectedConfigMaps: map[string]*corev1.ConfigMap{
"extension-apiserver-authentication": {
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
Data: map[string]string{
"requestheader-username-headers": `[]`,
"requestheader-uid-headers": `["panda"]`,
"requestheader-group-headers": `[]`,
"requestheader-extra-headers-prefix": `[]`,
"requestheader-client-ca-file": string(anotherRandomCA),
"requestheader-allowed-names": `[]`,
},
},
},
expectCreate: false,
uidGate: true,
},
{
name: "append uid with feature gate",
clusterAuthInfo: ClusterAuthenticationInfo{
RequestHeaderUsernameHeaders: headerrequest.StaticStringSlice{},
RequestHeaderUIDHeaders: headerrequest.StaticStringSlice{"panda"},
RequestHeaderGroupHeaders: headerrequest.StaticStringSlice{},
RequestHeaderExtraHeaderPrefixes: headerrequest.StaticStringSlice{},
RequestHeaderCA: anotherRandomCAProvider,
RequestHeaderAllowedNames: headerrequest.StaticStringSlice{},
},
preexistingObjs: []runtime.Object{
&corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
Data: map[string]string{
"requestheader-username-headers": `[]`,
"requestheader-uid-headers": `["snorlax"]`,
"requestheader-group-headers": `[]`,
"requestheader-extra-headers-prefix": `[]`,
"requestheader-client-ca-file": string(anotherRandomCA),
"requestheader-allowed-names": `[]`,
},
},
},
expectedConfigMaps: map[string]*corev1.ConfigMap{
"extension-apiserver-authentication": {
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
Data: map[string]string{
"requestheader-username-headers": `[]`,
"requestheader-uid-headers": `["snorlax","panda"]`,
"requestheader-group-headers": `[]`,
"requestheader-extra-headers-prefix": `[]`,
"requestheader-client-ca-file": string(anotherRandomCA),
"requestheader-allowed-names": `[]`,
},
},
},
expectCreate: false,
uidGate: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RemoteRequestHeaderUID, test.uidGate)
client := fake.NewSimpleClientset(test.preexistingObjs...)
configMapIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
for _, obj := range test.preexistingObjs {
@ -341,7 +481,6 @@ func TestWriteConfigMapDeleted(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"},
Data: map[string]string{
"requestheader-username-headers": `[]`,
"requestheader-uid-headers": `[]`,
"requestheader-group-headers": `[]`,
"requestheader-extra-headers-prefix": `[]`,
"requestheader-client-ca-file": string(anotherRandomCA),

View File

@ -306,6 +306,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
{Version: version.MustParse("1.29"), Default: true, PreRelease: featuregate.GA, LockToDefault: true},
},
genericfeatures.RemoteRequestHeaderUID: {
{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha},
},
genericfeatures.ResilientWatchCacheInitialization: {
{Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta},
},

View File

@ -149,6 +149,13 @@ const (
// to a chunking list request.
RemainingItemCount featuregate.Feature = "RemainingItemCount"
// owner: @stlaz
//
// Enable kube-apiserver to accept UIDs via request header authentication.
// This will also make the kube-apiserver's API aggregator add UIDs via standard
// headers when forwarding requests to the servers serving the aggregated API.
RemoteRequestHeaderUID featuregate.Feature = "RemoteRequestHeaderUID"
// owner: @wojtek-t
//
// Enables resilient watchcache initialization to avoid controlplane
@ -359,6 +366,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
{Version: version.MustParse("1.29"), Default: true, PreRelease: featuregate.GA, LockToDefault: true},
},
RemoteRequestHeaderUID: {
{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha},
},
ResilientWatchCacheInitialization: {
{Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.Beta},
},

View File

@ -29,8 +29,10 @@ import (
"k8s.io/apiserver/pkg/apis/apiserver"
"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
"k8s.io/apiserver/pkg/authentication/request/headerrequest"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
@ -68,9 +70,6 @@ func (s *RequestHeaderAuthenticationOptions) Validate() []error {
if err := checkForWhiteSpaceOnly("requestheader-username-headers", s.UsernameHeaders...); err != nil {
allErrors = append(allErrors, err)
}
if err := checkForWhiteSpaceOnly("requestheader-uid-headers", s.UIDHeaders...); err != nil {
allErrors = append(allErrors, err)
}
if err := checkForWhiteSpaceOnly("requestheader-group-headers", s.GroupHeaders...); err != nil {
allErrors = append(allErrors, err)
}
@ -84,10 +83,6 @@ func (s *RequestHeaderAuthenticationOptions) Validate() []error {
if len(s.UsernameHeaders) > 0 && !caseInsensitiveHas(s.UsernameHeaders, "X-Remote-User") {
klog.Warningf("--requestheader-username-headers is set without specifying the standard X-Remote-User header - API aggregation will not work")
}
if len(s.UIDHeaders) > 0 && !caseInsensitiveHas(s.UIDHeaders, "X-Remote-Uid") {
// this was added later and so we are able to error out
allErrors = append(allErrors, fmt.Errorf("--requestheader-uid-headers is set without specifying the standard X-Remote-Uid header - API aggregation will not work"))
}
if len(s.GroupHeaders) > 0 && !caseInsensitiveHas(s.GroupHeaders, "X-Remote-Group") {
klog.Warningf("--requestheader-group-headers is set without specifying the standard X-Remote-Group header - API aggregation will not work")
}
@ -95,6 +90,20 @@ func (s *RequestHeaderAuthenticationOptions) Validate() []error {
klog.Warningf("--requestheader-extra-headers-prefix is set without specifying the standard X-Remote-Extra- header prefix - API aggregation will not work")
}
if !utilfeature.DefaultFeatureGate.Enabled(features.RemoteRequestHeaderUID) {
if len(s.UIDHeaders) > 0 {
allErrors = append(allErrors, fmt.Errorf("--requestheader-uid-headers requires the %q feature to be enabled", features.RemoteRequestHeaderUID))
}
} else {
if err := checkForWhiteSpaceOnly("requestheader-uid-headers", s.UIDHeaders...); err != nil {
allErrors = append(allErrors, err)
}
if len(s.UIDHeaders) > 0 && !caseInsensitiveHas(s.UIDHeaders, "X-Remote-Uid") {
// this was added later and so we are able to error out
allErrors = append(allErrors, fmt.Errorf("--requestheader-uid-headers is set without specifying the standard X-Remote-Uid header - API aggregation will not work"))
}
}
return allErrors
}
@ -126,7 +135,7 @@ func (s *RequestHeaderAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
"List of request headers to inspect for usernames. X-Remote-User is common.")
fs.StringSliceVar(&s.UIDHeaders, "requestheader-uid-headers", s.UIDHeaders, ""+
"List of request headers to inspect for UIDs. X-Remote-Uid is suggested.")
"List of request headers to inspect for UIDs. X-Remote-Uid is suggested. Requires the RemoteRequestHeaderUID feature to be enabled.")
fs.StringSliceVar(&s.GroupHeaders, "requestheader-group-headers", s.GroupHeaders, ""+
"List of request headers to inspect for groups. X-Remote-Group is suggested.")

View File

@ -99,7 +99,7 @@ func SetFeatureGateEmulationVersionDuringTest(tb TB, gate featuregate.FeatureGat
detectParallelOverrideCleanup := detectParallelOverrideEmulationVersion(tb, ver)
originalEmuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion()
if err := gate.(featuregate.MutableVersionedFeatureGate).SetEmulationVersion(ver); err != nil {
tb.Fatalf("failed to set emulation version to %s during test", ver.String())
tb.Fatalf("failed to set emulation version to %s during test: %v", ver.String(), err)
}
tb.Cleanup(func() {
tb.Helper()

View File

@ -159,7 +159,12 @@ func (r *proxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
proxyRoundTripper := handlingInfo.proxyRoundTripper
upgrade := httpstream.IsUpgradeRequest(req)
proxyRoundTripper = transport.NewAuthProxyRoundTripper(user.GetName(), user.GetUID(), user.GetGroups(), user.GetExtra(), proxyRoundTripper)
var userUID string
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.RemoteRequestHeaderUID) {
userUID = user.GetUID()
}
proxyRoundTripper = transport.NewAuthProxyRoundTripper(user.GetName(), userUID, user.GetGroups(), user.GetExtra(), proxyRoundTripper)
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) && !upgrade {
tracingWrapper := tracing.WrapperFor(r.tracerProvider)
@ -170,7 +175,7 @@ func (r *proxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// NOT use the proxyRoundTripper. It's a direct dial that bypasses the proxyRoundTripper. This means that we have to
// attach the "correct" user headers to the request ahead of time.
if upgrade {
transport.SetAuthProxyHeaders(newReq, user.GetName(), user.GetUID(), user.GetGroups(), user.GetExtra())
transport.SetAuthProxyHeaders(newReq, user.GetName(), userUID, user.GetGroups(), user.GetExtra())
}
handler := proxy.NewUpgradeAwareHandler(location, proxyRoundTripper, true, upgrade, &responder{w: w})

View File

@ -34,31 +34,34 @@ import (
"sync/atomic"
"testing"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
"k8s.io/client-go/transport"
"golang.org/x/net/websocket"
"github.com/google/go-cmp/cmp"
"go.opentelemetry.io/otel/propagation"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
"go.opentelemetry.io/otel/trace"
"golang.org/x/net/websocket"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/proxy"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/endpoints/filters"
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
"k8s.io/apiserver/pkg/server/egressselector"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
apiserverproxyutil "k8s.io/apiserver/pkg/util/proxy"
"k8s.io/client-go/transport"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
apiregistration "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
"k8s.io/utils/pointer"
"k8s.io/utils/ptr"
)
@ -130,6 +133,8 @@ func TestProxyHandler(t *testing.T) {
expectedBody string
expectedCalled bool
expectedHeaders map[string][]string
enableFeatureGates []featuregate.Feature
}{
"no target": {
expectedStatusCode: http.StatusNotFound,
@ -138,7 +143,7 @@ func TestProxyHandler(t *testing.T) {
apiService: &apiregistration.APIService{
ObjectMeta: metav1.ObjectMeta{Name: "v1.foo"},
Spec: apiregistration.APIServiceSpec{
Service: &apiregistration.ServiceReference{Port: pointer.Int32Ptr(443)},
Service: &apiregistration.ServiceReference{Port: ptr.To[int32](443)},
Group: "foo",
Version: "v1",
},
@ -161,7 +166,7 @@ func TestProxyHandler(t *testing.T) {
apiService: &apiregistration.APIService{
ObjectMeta: metav1.ObjectMeta{Name: "v1.foo"},
Spec: apiregistration.APIServiceSpec{
Service: &apiregistration.ServiceReference{Port: pointer.Int32Ptr(443)},
Service: &apiregistration.ServiceReference{Port: ptr.To[int32](443)},
Group: "foo",
Version: "v1",
InsecureSkipTLSVerify: true,
@ -174,6 +179,40 @@ func TestProxyHandler(t *testing.T) {
},
expectedStatusCode: http.StatusOK,
expectedCalled: true,
expectedHeaders: map[string][]string{
"X-Forwarded-Proto": {"https"},
"X-Forwarded-Uri": {"/request/path"},
"X-Forwarded-For": {"127.0.0.1"},
"X-Remote-User": {"username"},
"User-Agent": {"Go-http-client/1.1"},
"Accept-Encoding": {"gzip"},
"X-Remote-Group": {"one", "two"},
},
},
"[RemoteRequestHeaderUID] proxy with user, insecure": {
user: &user.DefaultInfo{
Name: "username",
UID: "6b60d791-1af9-4513-92e5-e4252a1e0a78",
Groups: []string{"one", "two"},
},
path: "/request/path",
apiService: &apiregistration.APIService{
ObjectMeta: metav1.ObjectMeta{Name: "v1.foo"},
Spec: apiregistration.APIServiceSpec{
Service: &apiregistration.ServiceReference{Port: ptr.To[int32](443)},
Group: "foo",
Version: "v1",
InsecureSkipTLSVerify: true,
},
Status: apiregistration.APIServiceStatus{
Conditions: []apiregistration.APIServiceCondition{
{Type: apiregistration.Available, Status: apiregistration.ConditionTrue},
},
},
},
enableFeatureGates: []featuregate.Feature{features.RemoteRequestHeaderUID},
expectedStatusCode: http.StatusOK,
expectedCalled: true,
expectedHeaders: map[string][]string{
"X-Forwarded-Proto": {"https"},
"X-Forwarded-Uri": {"/request/path"},
@ -195,7 +234,7 @@ func TestProxyHandler(t *testing.T) {
apiService: &apiregistration.APIService{
ObjectMeta: metav1.ObjectMeta{Name: "v1.foo"},
Spec: apiregistration.APIServiceSpec{
Service: &apiregistration.ServiceReference{Name: "test-service", Namespace: "test-ns", Port: pointer.Int32Ptr(443)},
Service: &apiregistration.ServiceReference{Name: "test-service", Namespace: "test-ns", Port: ptr.To[int32](443)},
Group: "foo",
Version: "v1",
CABundle: testCACrt,
@ -208,6 +247,40 @@ func TestProxyHandler(t *testing.T) {
},
expectedStatusCode: http.StatusOK,
expectedCalled: true,
expectedHeaders: map[string][]string{
"X-Forwarded-Proto": {"https"},
"X-Forwarded-Uri": {"/request/path"},
"X-Forwarded-For": {"127.0.0.1"},
"X-Remote-User": {"username"},
"User-Agent": {"Go-http-client/1.1"},
"Accept-Encoding": {"gzip"},
"X-Remote-Group": {"one", "two"},
},
},
"[RemoteRequestHeaderUID] proxy with user, cabundle": {
user: &user.DefaultInfo{
Name: "username",
UID: "6b60d791-1af9-4513-92e5-e4252a1e0a78",
Groups: []string{"one", "two"},
},
path: "/request/path",
apiService: &apiregistration.APIService{
ObjectMeta: metav1.ObjectMeta{Name: "v1.foo"},
Spec: apiregistration.APIServiceSpec{
Service: &apiregistration.ServiceReference{Name: "test-service", Namespace: "test-ns", Port: ptr.To[int32](443)},
Group: "foo",
Version: "v1",
CABundle: testCACrt,
},
Status: apiregistration.APIServiceStatus{
Conditions: []apiregistration.APIServiceCondition{
{Type: apiregistration.Available, Status: apiregistration.ConditionTrue},
},
},
},
enableFeatureGates: []featuregate.Feature{features.RemoteRequestHeaderUID},
expectedStatusCode: http.StatusOK,
expectedCalled: true,
expectedHeaders: map[string][]string{
"X-Forwarded-Proto": {"https"},
"X-Forwarded-Uri": {"/request/path"},
@ -229,7 +302,7 @@ func TestProxyHandler(t *testing.T) {
apiService: &apiregistration.APIService{
ObjectMeta: metav1.ObjectMeta{Name: "v1.foo"},
Spec: apiregistration.APIServiceSpec{
Service: &apiregistration.ServiceReference{Name: "test-service", Namespace: "test-ns", Port: pointer.Int32Ptr(443)},
Service: &apiregistration.ServiceReference{Name: "test-service", Namespace: "test-ns", Port: ptr.To[int32](443)},
Group: "foo",
Version: "v1",
CABundle: testCACrt,
@ -253,7 +326,7 @@ func TestProxyHandler(t *testing.T) {
apiService: &apiregistration.APIService{
ObjectMeta: metav1.ObjectMeta{Name: "v1.foo"},
Spec: apiregistration.APIServiceSpec{
Service: &apiregistration.ServiceReference{Name: "bad-service", Namespace: "test-ns", Port: pointer.Int32Ptr(443)},
Service: &apiregistration.ServiceReference{Name: "bad-service", Namespace: "test-ns", Port: ptr.To[int32](443)},
Group: "foo",
Version: "v1",
CABundle: testCACrt,
@ -276,7 +349,7 @@ func TestProxyHandler(t *testing.T) {
apiService: &apiregistration.APIService{
ObjectMeta: metav1.ObjectMeta{Name: "v1.foo"},
Spec: apiregistration.APIServiceSpec{
Service: &apiregistration.ServiceReference{Port: pointer.Int32Ptr(443)},
Service: &apiregistration.ServiceReference{Port: ptr.To[int32](443)},
Group: "foo",
Version: "v1",
},
@ -298,7 +371,7 @@ func TestProxyHandler(t *testing.T) {
apiService: &apiregistration.APIService{
ObjectMeta: metav1.ObjectMeta{Name: "v1.foo"},
Spec: apiregistration.APIServiceSpec{
Service: &apiregistration.ServiceReference{Name: "test-service", Namespace: "test-ns", Port: pointer.Int32Ptr(443)},
Service: &apiregistration.ServiceReference{Name: "test-service", Namespace: "test-ns", Port: ptr.To[int32](443)},
Group: "foo",
Version: "v1",
CABundle: testCACrt,
@ -320,7 +393,11 @@ func TestProxyHandler(t *testing.T) {
target.Reset()
legacyregistry.Reset()
func() {
t.Run(name, func(t *testing.T) {
for _, f := range tc.enableFeatureGates {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, f, true)
}
targetServer := httptest.NewUnstartedServer(target)
serviceCert := tc.serviceCertOverride
if serviceCert == nil {
@ -354,37 +431,37 @@ func TestProxyHandler(t *testing.T) {
resp, err := http.Get(server.URL + tc.path)
if err != nil {
t.Errorf("%s: %v", name, err)
t.Errorf("%v", err)
return
}
if e, a := tc.expectedStatusCode, resp.StatusCode; e != a {
body, _ := httputil.DumpResponse(resp, true)
t.Logf("%s: %v", name, string(body))
t.Errorf("%s: expected %v, got %v", name, e, a)
t.Logf("%v", string(body))
t.Errorf("expected %v, got %v", e, a)
return
}
bytes, err := io.ReadAll(resp.Body)
if err != nil {
t.Errorf("%s: %v", name, err)
t.Errorf("%v", err)
return
}
if !strings.Contains(string(bytes), tc.expectedBody) {
t.Errorf("%s: expected %q, got %q", name, tc.expectedBody, string(bytes))
t.Errorf("expected %q, got %q", tc.expectedBody, string(bytes))
return
}
if e, a := tc.expectedCalled, target.called; e != a {
t.Errorf("%s: expected %v, got %v", name, e, a)
t.Errorf("expected %v, got %v", e, a)
return
}
// this varies every test
delete(target.headers, "X-Forwarded-Host")
if e, a := tc.expectedHeaders, target.headers; !reflect.DeepEqual(e, a) {
t.Errorf("%s: expected %v, got %v", name, e, a)
t.Errorf("expected != got %v", cmp.Diff(e, a))
return
}
if e, a := targetServer.Listener.Addr().String(), target.host; tc.expectedCalled && !reflect.DeepEqual(e, a) {
t.Errorf("%s: expected %v, got %v", name, e, a)
t.Errorf("expected %v, got %v", e, a)
return
}
@ -397,7 +474,7 @@ func TestProxyHandler(t *testing.T) {
t.Errorf("expected the x509_missing_san_total to be 1, but it's %d", errorCounter)
}
}
}()
})
}
}
@ -432,6 +509,8 @@ func newBrokenDialerAndSelector() (*mockEgressDialer, *egressselector.EgressSele
}
func TestProxyUpgrade(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RemoteRequestHeaderUID, true)
upgradeUser := "upgradeUser"
upgradeUID := "upgradeUser-UID"
testcases := map[string]struct {
@ -446,7 +525,7 @@ func TestProxyUpgrade(t *testing.T) {
CABundle: testCACrt,
Group: "mygroup",
Version: "v1",
Service: &apiregistration.ServiceReference{Name: "test-service", Namespace: "test-ns", Port: pointer.Int32Ptr(443)},
Service: &apiregistration.ServiceReference{Name: "test-service", Namespace: "test-ns", Port: ptr.To[int32](443)},
},
Status: apiregistration.APIServiceStatus{
Conditions: []apiregistration.APIServiceCondition{
@ -463,7 +542,7 @@ func TestProxyUpgrade(t *testing.T) {
InsecureSkipTLSVerify: true,
Group: "mygroup",
Version: "v1",
Service: &apiregistration.ServiceReference{Name: "invalid-service", Namespace: "invalid-ns", Port: pointer.Int32Ptr(443)},
Service: &apiregistration.ServiceReference{Name: "invalid-service", Namespace: "invalid-ns", Port: ptr.To[int32](443)},
},
Status: apiregistration.APIServiceStatus{
Conditions: []apiregistration.APIServiceCondition{
@ -480,7 +559,7 @@ func TestProxyUpgrade(t *testing.T) {
CABundle: testCACrt,
Group: "mygroup",
Version: "v1",
Service: &apiregistration.ServiceReference{Name: "invalid-service", Namespace: "invalid-ns", Port: pointer.Int32Ptr(443)},
Service: &apiregistration.ServiceReference{Name: "invalid-service", Namespace: "invalid-ns", Port: ptr.To[int32](443)},
},
Status: apiregistration.APIServiceStatus{
Conditions: []apiregistration.APIServiceCondition{
@ -497,7 +576,7 @@ func TestProxyUpgrade(t *testing.T) {
CABundle: testCACrt,
Group: "mygroup",
Version: "v1",
Service: &apiregistration.ServiceReference{Name: "test-service", Namespace: "test-ns", Port: pointer.Int32Ptr(443)},
Service: &apiregistration.ServiceReference{Name: "test-service", Namespace: "test-ns", Port: ptr.To[int32](443)},
},
Status: apiregistration.APIServiceStatus{
Conditions: []apiregistration.APIServiceCondition{
@ -515,7 +594,7 @@ func TestProxyUpgrade(t *testing.T) {
CABundle: testCACrt,
Group: "mygroup",
Version: "v1",
Service: &apiregistration.ServiceReference{Name: "test-service", Namespace: "test-ns", Port: pointer.Int32Ptr(443)},
Service: &apiregistration.ServiceReference{Name: "test-service", Namespace: "test-ns", Port: ptr.To[int32](443)},
},
Status: apiregistration.APIServiceStatus{
Conditions: []apiregistration.APIServiceCondition{
@ -996,7 +1075,7 @@ func TestProxyCertReload(t *testing.T) {
apiService := &apiregistration.APIService{
ObjectMeta: metav1.ObjectMeta{Name: "v1.foo"},
Spec: apiregistration.APIServiceSpec{
Service: &apiregistration.ServiceReference{Name: "test-service2", Namespace: "test-ns", Port: pointer.Int32Ptr(443)},
Service: &apiregistration.ServiceReference{Name: "test-service2", Namespace: "test-ns", Port: ptr.To[int32](443)},
Group: "foo",
Version: "v1",
CABundle: backendCaCertificate(), // used to validate backendCertificate()

View File

@ -27,7 +27,7 @@ import (
func main() {
ctx := genericapiserver.SetupSignalContext()
options := server.NewWardleServerOptions(os.Stdout, os.Stderr)
cmd := server.NewCommandStartWardleServer(ctx, options)
cmd := server.NewCommandStartWardleServer(ctx, options, false)
code := cli.Run(cmd)
os.Exit(code)
}

View File

@ -90,12 +90,15 @@ func NewWardleServerOptions(out, errOut io.Writer) *WardleServerOptions {
// NewCommandStartWardleServer provides a CLI handler for 'start master' command
// with a default WardleServerOptions.
func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOptions) *cobra.Command {
func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOptions, skipDefaultComponentGlobalsRegistrySet bool) *cobra.Command {
o := *defaults
cmd := &cobra.Command{
Short: "Launch a wardle API server",
Long: "Launch a wardle API server",
PersistentPreRunE: func(*cobra.Command, []string) error {
if skipDefaultComponentGlobalsRegistrySet {
return nil
}
return featuregate.DefaultComponentGlobalsRegistry.Set()
},
RunE: func(c *cobra.Command, args []string) error {

View File

@ -1058,6 +1058,12 @@
lockToDefault: true
preRelease: GA
version: "1.29"
- name: RemoteRequestHeaderUID
versionedSpecs:
- default: false
lockToDefault: false
preRelease: Alpha
version: "1.32"
- name: ResilientWatchCacheInitialization
versionedSpecs:
- default: true

View File

@ -27,11 +27,14 @@ import (
authnv1 "k8s.io/api/authentication/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/features"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
certutil "k8s.io/client-go/util/cert"
"k8s.io/client-go/util/keyutil"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
"k8s.io/kubernetes/test/integration/framework"
testutils "k8s.io/kubernetes/test/utils"
@ -39,6 +42,8 @@ import (
)
func TestAuthnToKAS(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RemoteRequestHeaderUID, true)
tCtx := ktesting.Init(t)
frontProxyCA, frontProxyClient, frontProxyKey, err := newTestCAWithClient(

View File

@ -45,8 +45,10 @@ import (
"k8s.io/apiserver/pkg/authentication/serviceaccount"
"k8s.io/apiserver/pkg/authentication/user"
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
genericapiserveroptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
client "k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
@ -54,6 +56,7 @@ import (
"k8s.io/client-go/transport"
"k8s.io/client-go/util/cert"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
utilversion "k8s.io/component-base/version"
apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
@ -260,6 +263,7 @@ func TestFrontProxyConfig(t *testing.T) {
testFrontProxyConfig(t, false)
})
t.Run("WithUID", func(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RemoteRequestHeaderUID, true)
testFrontProxyConfig(t, true)
})
}
@ -282,7 +286,7 @@ func testFrontProxyConfig(t *testing.T, withUID bool) {
kubeBinaryVersion := sampleserver.WardleVersionToKubeVersion(version.MustParse(wardleBinaryVersion)).String()
// start up the KAS and prepare the options for the wardle API server
testKAS, wardleOptions, wardlePort := prepareAggregatedWardleAPIServer(ctx, t, testNamespace, kubeBinaryVersion, wardleBinaryVersion, extraKASFlags)
testKAS, wardleOptions, wardlePort := prepareAggregatedWardleAPIServer(ctx, t, testNamespace, kubeBinaryVersion, wardleBinaryVersion, extraKASFlags, withUID)
kubeConfig := getKubeConfig(testKAS)
// create the SA that we will use to query the aggregated API
@ -337,12 +341,12 @@ func testFrontProxyConfig(t *testing.T, withUID bool) {
transport.WrapperFunc(func(rt http.RoundTripper) http.RoundTripper {
return roundTripperFunc(func(req *http.Request) (*http.Response, error) {
gotUser, ok := genericapirequest.UserFrom(req.Context())
if !ok || gotUser.GetName() == "system:anonymous" {
if !ok {
return nil, fmt.Errorf("got an unauthenticated request")
}
// this is likely the KAS checking the OpenAPI endpoints
if gotUser.GetName() == "system:anonymous" || gotUser.GetName() == "system:aggregator" {
if gotUser.GetName() == "system:anonymous" || gotUser.GetName() == "system:aggregator" || gotUser.GetName() == "system:kube-aggregator" {
return rt.RoundTrip(req)
}
@ -368,7 +372,7 @@ func testFrontProxyConfig(t *testing.T, withUID bool) {
wardleCertDir, _ := os.MkdirTemp("", "test-integration-wardle-server")
defer os.RemoveAll(wardleCertDir)
runPreparedWardleServer(ctx, t, wardleOptions, wardleCertDir, wardlePort, false, true, wardleBinaryVersion, kubeConfig)
runPreparedWardleServer(ctx, t, wardleOptions, wardleCertDir, wardlePort, false, true, wardleBinaryVersion, kubeConfig, withUID)
waitForWardleAPIServiceReady(ctx, t, kubeConfig, wardleCertDir, testNamespace)
// get the wardle API client using our SA token
@ -401,13 +405,13 @@ func testAggregatedAPIServer(t *testing.T, setWardleFeatureGate, banFlunder bool
// each wardle binary is bundled with a specific kube binary.
kubeBinaryVersion := sampleserver.WardleVersionToKubeVersion(version.MustParse(wardleBinaryVersion)).String()
testKAS, wardleOptions, wardlePort := prepareAggregatedWardleAPIServer(ctx, t, testNamespace, kubeBinaryVersion, wardleBinaryVersion, nil)
testKAS, wardleOptions, wardlePort := prepareAggregatedWardleAPIServer(ctx, t, testNamespace, kubeBinaryVersion, wardleBinaryVersion, nil, false)
kubeClientConfig := getKubeConfig(testKAS)
wardleCertDir, _ := os.MkdirTemp("", "test-integration-wardle-server")
defer os.RemoveAll(wardleCertDir)
directWardleClientConfig := runPreparedWardleServer(ctx, t, wardleOptions, wardleCertDir, wardlePort, setWardleFeatureGate, banFlunder, wardleEmulationVersion, kubeClientConfig)
directWardleClientConfig := runPreparedWardleServer(ctx, t, wardleOptions, wardleCertDir, wardlePort, setWardleFeatureGate, banFlunder, wardleEmulationVersion, kubeClientConfig, false)
// now we're finally ready to test. These are what's run by default now
wardleDirectClient := client.NewForConfigOrDie(directWardleClientConfig)
@ -681,7 +685,7 @@ func TestAggregatedAPIServerRejectRedirectResponse(t *testing.T) {
}
}
func prepareAggregatedWardleAPIServer(ctx context.Context, t *testing.T, namespace, kubebinaryVersion, wardleBinaryVersion string, kubeAPIServerFlags []string) (*kastesting.TestServer, *sampleserver.WardleServerOptions, int) {
func prepareAggregatedWardleAPIServer(ctx context.Context, t *testing.T, namespace, kubebinaryVersion, wardleBinaryVersion string, kubeAPIServerFlags []string, withUID bool) (*kastesting.TestServer, *sampleserver.WardleServerOptions, int) {
// makes the kube-apiserver very responsive. it's normally a minute
dynamiccertificates.FileRefreshDuration = 1 * time.Second
@ -693,6 +697,11 @@ func prepareAggregatedWardleAPIServer(ctx context.Context, t *testing.T, namespa
// endpoints cannot have loopback IPs so we need to override the resolver itself
t.Cleanup(app.SetServiceResolverForTests(staticURLServiceResolver(fmt.Sprintf("https://127.0.0.1:%d", wardlePort))))
// TODO figure out how to actually make BinaryVersion/EmulationVersion work with Wardle and KAS at the same time when Alpha FG are being set
if withUID {
kubebinaryVersion = ""
}
testServer := kastesting.StartTestServerOrDie(t,
&kastesting.TestServerInstanceOptions{
EnableCertAuth: true,
@ -751,6 +760,7 @@ func runPreparedWardleServer(
banFlunder bool,
emulationVersion string,
kubeConfig *rest.Config,
withUID bool,
) *rest.Config {
// start the wardle server to prove we can aggregate it
@ -769,7 +779,8 @@ func runPreparedWardleServer(
if flunderBanningFeatureGate {
args = append(args, "--feature-gates", fmt.Sprintf("wardle:BanFlunder=%v", banFlunder))
}
wardleCmd := sampleserver.NewCommandStartWardleServer(ctx, wardleOptions)
// TODO figure out how to actually make BinaryVersion/EmulationVersion work with Wardle and KAS at the same time when Alpha FG are being set
wardleCmd := sampleserver.NewCommandStartWardleServer(ctx, wardleOptions, withUID)
wardleCmd.SetArgs(args)
if err := wardleCmd.Execute(); err != nil {
t.Error(err)