mirror of
https://github.com/rancher/steve.git
synced 2025-09-04 00:44:55 +00:00
237 lines
9.9 KiB
Go
237 lines
9.9 KiB
Go
package clustercache_test
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rancher/apiserver/pkg/types"
|
|
"github.com/rancher/steve/pkg/accesscontrol"
|
|
"github.com/rancher/steve/pkg/clustercache"
|
|
"github.com/rancher/steve/pkg/schema"
|
|
"github.com/rancher/wrangler/v3/pkg/schemas"
|
|
"github.com/stretchr/testify/assert"
|
|
"golang.org/x/time/rate"
|
|
"k8s.io/apiserver/pkg/authentication/user"
|
|
"k8s.io/client-go/dynamic"
|
|
"k8s.io/client-go/rest"
|
|
)
|
|
|
|
var testSchema = map[string]*types.APISchema{
|
|
"rbac.authorization.k8s.io.clusterrole": &types.APISchema{
|
|
Schema: &schemas.Schema{
|
|
ID: "rbac.authorization.k8s.io.clusterrole",
|
|
PluralName: "rbac.authorization.k8s.io.clusterroles",
|
|
Attributes: map[string]any{
|
|
"group": "rbac.authorization.k8s.io",
|
|
"kind": "ClusterRole",
|
|
"namespaced": false,
|
|
"resource": "clusterroles",
|
|
"verbs": []string{"create", "delete", "deletecollection", "get", "list", "patch", "update", "watch"},
|
|
"version": "v1",
|
|
},
|
|
},
|
|
},
|
|
"configmap": &types.APISchema{
|
|
Schema: &schemas.Schema{
|
|
ID: "configmap",
|
|
PluralName: "configmaps",
|
|
Attributes: map[string]any{
|
|
"kind": "ConfigMap",
|
|
"namespaced": true,
|
|
"resource": "configmaps",
|
|
"verbs": []string{"create", "delete", "deletecollection", "get", "list", "patch", "update", "watch"},
|
|
"version": "v1",
|
|
},
|
|
},
|
|
},
|
|
"secret": &types.APISchema{
|
|
Schema: &schemas.Schema{
|
|
ID: "secret",
|
|
PluralName: "secrets",
|
|
Attributes: map[string]any{
|
|
"kind": "Secret",
|
|
"namespaced": true,
|
|
"resource": "secrets",
|
|
"verbs": []string{"create", "delete", "deletecollection", "get", "list", "patch", "update", "watch"},
|
|
"version": "v1",
|
|
},
|
|
},
|
|
},
|
|
"service": &types.APISchema{
|
|
Schema: &schemas.Schema{
|
|
ID: "service",
|
|
PluralName: "services",
|
|
Attributes: map[string]any{
|
|
"kind": "Service",
|
|
"namespaced": true,
|
|
"resource": "services",
|
|
"verbs": []string{"create", "delete", "deletecollection", "get", "list", "patch", "update", "watch"},
|
|
"version": "v1"},
|
|
},
|
|
},
|
|
}
|
|
|
|
func TestClusterCacheRateLimitingNotEnabled(t *testing.T) {
|
|
var errorCount int32
|
|
ctx, cancel := context.WithCancel(context.TODO())
|
|
defer cancel()
|
|
// The cache makes spawns a Go routine that makes 2 requests for each schema
|
|
// (list and watch).
|
|
// This configures the test-server to rate-limit responses.
|
|
requestCount := len(testSchema) * 2
|
|
// The cache spawns a Go routine that makes 2 requests for each schema
|
|
// (list and watch).
|
|
tstSrv := startTestServer(t, rate.NewLimiter(rate.Limit(1000),
|
|
requestCount-2), &errorCount)
|
|
|
|
sf := schema.NewCollection(ctx, &types.APISchemas{}, fakeAccessSetLookup{})
|
|
sf.Reset(testSchema)
|
|
dc, err := dynamic.NewForConfig(&rest.Config{Host: tstSrv.URL})
|
|
assert.NoError(t, err)
|
|
cache := clustercache.NewClusterCache(ctx, dc)
|
|
|
|
err = cache.OnSchemas(sf)
|
|
assert.NoError(t, err)
|
|
|
|
// We should make requestCount requests in a burst, but the server is
|
|
// limited to requestCount-2 in a burst.
|
|
//
|
|
// We can't really know how many failed requests there will be as this will
|
|
// be impacted by how quickly we start the Go routines (and thus the load on
|
|
// the machine), but as the server is rate-limited to fewer than requests we
|
|
// make, we get 429s.
|
|
assert.NotZero(t, errorCount)
|
|
}
|
|
|
|
func TestClusterCacheRateLimitingEnabled(t *testing.T) {
|
|
var errorCount int32
|
|
ctx, cancel := context.WithCancel(context.TODO())
|
|
defer cancel()
|
|
t.Setenv("RANCHER_CACHE_CLIENT_LIMIT", "3")
|
|
requestCount := len(testSchema) * 2
|
|
// The cache spawns a Go routine that makes 2 requests for each schema
|
|
// (list and watch).
|
|
tstSrv := startTestServer(t, rate.NewLimiter(rate.Limit(1000),
|
|
requestCount-2), &errorCount)
|
|
|
|
sf := schema.NewCollection(ctx, &types.APISchemas{}, fakeAccessSetLookup{})
|
|
sf.Reset(testSchema)
|
|
dc, err := dynamic.NewForConfig(&rest.Config{Host: tstSrv.URL})
|
|
assert.NoError(t, err)
|
|
cache := clustercache.NewClusterCache(ctx, dc)
|
|
|
|
err = cache.OnSchemas(sf)
|
|
assert.NoError(t, err)
|
|
|
|
// The cache client is rate-limited to less than the number of requests to
|
|
// be made.
|
|
// The server will allow all requests in a burst, so we should get no 429
|
|
// responses.
|
|
assert.Zero(t, errorCount)
|
|
}
|
|
|
|
// This provides a fake K8s API server that uses the provided rate.Limit to
|
|
// rate-limit requests, responding with 429 if the rate-limiter is limiting
|
|
// requests.
|
|
//
|
|
// This only handles list and watch requests for secrets, services, configmaps
|
|
// and clusterroles.
|
|
//
|
|
// The errors value passed in will be incremented every time a 429 response is
|
|
// returned to the client (client-go will consume some 429 responses).
|
|
func startTestServer(t *testing.T, rl *rate.Limiter, errors *int32) *httptest.Server {
|
|
start := time.Now()
|
|
|
|
writeJSON := func(w http.ResponseWriter, s string) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Write([]byte(s))
|
|
}
|
|
|
|
writeChunks := func(w http.ResponseWriter, chunks []string) {
|
|
w.Header().Set("Transfer-Encoding", "chunked")
|
|
w.Header().Set("Content-Type", "application/json")
|
|
for _, chunk := range chunks {
|
|
time.Sleep(1 * time.Second)
|
|
fmt.Fprintf(w, chunk)
|
|
w.(http.Flusher).Flush()
|
|
}
|
|
}
|
|
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
values := r.URL.Query()
|
|
watched := values.Get("watch") == "true"
|
|
t.Logf("Faking response to %s watch=%v", r.URL.Path, watched)
|
|
|
|
if !rl.Allow() {
|
|
w.WriteHeader(http.StatusTooManyRequests)
|
|
w.Header().Set("Retry-After", "1")
|
|
atomic.AddInt32(errors, 1)
|
|
return
|
|
}
|
|
|
|
switch {
|
|
case r.URL.Path == "/api/v1/configmaps" && !watched:
|
|
writeJSON(w, `{"kind":"ConfigMapList","apiVersion":"v1","metadata":{"resourceVersion":"839768"},"items":[{"metadata":{"name":"my-test-configmap","namespace":"default","uid":"05f64c34-711e-4bef-8655-c486fbd6761e","resourceVersion":"629061","creationTimestamp":"2025-06-16T07:41:19Z","managedFields":[{"manager":"kubectl-create","operation":"Update","apiVersion":"v1","time":"2025-06-16T07:41:19Z","fieldsType":"FieldsV1","fieldsV1":{"f:data":{".":{},"f:testing":{}}}}]},"data":{"testing":"tested"}}]}`)
|
|
case r.URL.Path == "/api/v1/configmaps" && watched:
|
|
chunks := []string{
|
|
`{"type":"ADDED","object":{"apiVersion":"v1","data":{"value":"nothing"},"kind":"ConfigMap","metadata":{"creationTimestamp":"2025-06-18T15:47:32Z","name":"other-test-configmap","namespace":"default","resourceVersion":"855779","uid": "83b9f7c9-05aa-4a8d-bd01-adf1e1089c8d"}}}`,
|
|
}
|
|
writeChunks(w, chunks)
|
|
return
|
|
|
|
case r.URL.Path == "/apis/rbac.authorization.k8s.io/v1/clusterroles" && !watched:
|
|
writeJSON(w, `{"kind":"ClusterRoleList","apiVersion":"rbac.authorization.k8s.io/v1","metadata":{"resourceVersion":"840190"},"items":[{"metadata":{"name":"admin","uid":"dfa4f720-857f-4a9a-96ae-2cebdd5b4166","resourceVersion":"1413","creationTimestamp":"2025-05-30T09:33:38Z","labels":{"kubernetes.io/bootstrapping":"rbac-defaults"},"annotations":{"rbac.authorization.kubernetes.io/autoupdate":"true"},"managedFields":[{"manager":"clusterrole-aggregation-controller","operation":"Apply","apiVersion":"rbac.authorization.k8s.io/v1","time":"2025-05-30T10:12:19Z","fieldsType":"FieldsV1","fieldsV1":{"f:rules":{}}},{"manager":"k3s","operation":"Update","apiVersion":"rbac.authorization.k8s.io/v1","time":"2025-05-30T09:33:38Z","fieldsType":"FieldsV1","fieldsV1":{"f:aggregationRule":{".":{},"f:clusterRoleSelectors":{}},"f:metadata":{"f:annotations":{".":{},"f:rbac.authorization.kubernetes.io/autoupdate":{}},"f:labels":{".":{},"f:kubernetes.io/bootstrapping":{}}}}}]},"rules":[{"verbs":["create","delete","deletecollection","patch","update"],"apiGroups":["cert-manager.io"],"resources":["certificates","certificaterequests","issuers"]}]}]}`)
|
|
case r.URL.Path == "/apis/rbac.authorization.k8s.io/v1/clusterroles" && watched:
|
|
chunks := []string{
|
|
`{"type":"ADDED","object":{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRole","metadata":{"creationTimestamp":"2025-05-30T10:12:19Z","name":"cert-manager-view","resourceVersion":"1399","uid":"06c404a0-895c-45db-a506-9ab6c7336d0f"}}}`,
|
|
}
|
|
writeChunks(w, chunks)
|
|
return
|
|
case r.URL.Path == "/api/v1/secrets" && !watched:
|
|
writeJSON(w, `{"kind":"SecretList","apiVersion":"v1","metadata":{"resourceVersion":"839768"},"items":[{"metadata":{"name":"my-test-secret","namespace":"default","uid":"975dab7f-0b57-4d57-aaf8-f95851fac6e5","resourceVersion":"629061","creationTimestamp":"2025-06-16T07:41:19Z","managedFields":[]},"data":{"testing":"tested"}}]}`)
|
|
return
|
|
case r.URL.Path == "/api/v1/secrets" && watched:
|
|
chunks := []string{
|
|
`{"type":"ADDED","object":{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"Secret","metadata":{"creationTimestamp":"2025-05-30T10:12:19Z","name":"new-test-secret","resourceVersion":"629062","uid":"338ca29a-44ef-45ce-9e21-6479766a008f"}}}`,
|
|
}
|
|
writeChunks(w, chunks)
|
|
return
|
|
case r.URL.Path == "/api/v1/services" && !watched:
|
|
writeJSON(w, `{"kind":"ServiceList","apiVersion":"v1","metadata":{"resourceVersion":"839768"},"items":[{"metadata":{"name":"my-test-service","namespace":"default","uid":"8c0ab3eb-a5b7-428f-82f3-a1e77d11b596","resourceVersion":"629061","creationTimestamp":"2025-06-16T07:41:19Z","managedFields":[]}}]}`)
|
|
return
|
|
case r.URL.Path == "/api/v1/services" && watched:
|
|
chunks := []string{
|
|
`{"type":"ADDED","object":{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"Service","metadata":{"creationTimestamp":"2025-06-19T08:12:19Z","name":"new-test-service","resourceVersion":"629063","uid":"e5d5f856-c961-4df8-b1eb-234195b5556a"}}}`,
|
|
}
|
|
writeChunks(w, chunks)
|
|
return
|
|
|
|
default:
|
|
w.WriteHeader(http.StatusNotFound)
|
|
return
|
|
}
|
|
}))
|
|
|
|
t.Cleanup(func() {
|
|
t.Logf("%v errors in %v", *errors, time.Since(start))
|
|
ts.Close()
|
|
})
|
|
|
|
return ts
|
|
}
|
|
|
|
type fakeAccessSetLookup struct {
|
|
}
|
|
|
|
func (a fakeAccessSetLookup) AccessFor(user user.Info) *accesscontrol.AccessSet {
|
|
return nil
|
|
}
|
|
|
|
func (a fakeAccessSetLookup) PurgeUserData(id string) {
|
|
}
|