From 3eba71d06b1c242931bba9b2f515456b2c04fba9 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Mon, 1 Jun 2020 15:59:38 -0700 Subject: [PATCH] Add kubectl shell support --- pkg/client/factory.go | 36 ++- pkg/schemaserver/handlers/list.go | 30 +- pkg/schemaserver/urlbuilder/url.go | 5 +- pkg/schemaserver/writer/encoding.go | 9 + pkg/server/resources/clusters/clusters.go | 13 +- pkg/server/resources/clusters/shell.go | 343 ++++++++++++++++++++++ pkg/server/resources/schema.go | 2 +- pkg/server/router/router.go | 7 +- pkg/server/store/proxy/proxy_store.go | 4 + 9 files changed, 426 insertions(+), 23 deletions(-) create mode 100644 pkg/server/resources/clusters/shell.go diff --git a/pkg/client/factory.go b/pkg/client/factory.go index 650bfd7..7ec6eca 100644 --- a/pkg/client/factory.go +++ b/pkg/client/factory.go @@ -9,6 +9,7 @@ import ( "github.com/rancher/steve/pkg/schemaserver/types" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/metadata" "k8s.io/client-go/rest" ) @@ -43,7 +44,6 @@ func NewFactory(cfg *rest.Config, impersonate bool) (*Factory, error) { clientCfg := rest.CopyConfig(cfg) clientCfg.QPS = 10000 clientCfg.Burst = 100 - clientCfg.AcceptContentTypes = "application/json;as=Table;v=v1;g=meta.k8s.io" watchClientCfg := rest.CopyConfig(clientCfg) watchClientCfg.Timeout = 30 * time.Minute @@ -59,8 +59,10 @@ func NewFactory(cfg *rest.Config, impersonate bool) (*Factory, error) { tableClientCfg := rest.CopyConfig(clientCfg) tableClientCfg.Wrap(setTable) + tableClientCfg.AcceptContentTypes = "application/json;as=Table;v=v1;g=meta.k8s.io" tableWatchClientCfg := rest.CopyConfig(watchClientCfg) tableWatchClientCfg.Wrap(setTable) + tableWatchClientCfg.AcceptContentTypes = "application/json;as=Table;v=v1;g=meta.k8s.io" md, err := metadata.NewForConfig(cfg) if err != nil { @@ -92,6 +94,28 @@ func (p *Factory) DynamicClient() dynamic.Interface { return p.dynamic } +func (p *Factory) IsImpersonating() bool { + return p.impersonate +} + +func (p *Factory) K8sInterface(ctx *types.APIRequest) (kubernetes.Interface, error) { + cfg, err := setupConfig(ctx, p.clientCfg, p.impersonate) + if err != nil { + return nil, err + } + + return kubernetes.NewForConfig(cfg) +} + +func (p *Factory) AdminK8sInterface(ctx *types.APIRequest) (kubernetes.Interface, error) { + cfg, err := setupConfig(ctx, p.clientCfg, false) + if err != nil { + return nil, err + } + + return kubernetes.NewForConfig(cfg) +} + func (p *Factory) Client(ctx *types.APIRequest, s *types.APISchema, namespace string) (dynamic.ResourceInterface, error) { return newClient(ctx, p.clientCfg, s, namespace, p.impersonate) } @@ -136,7 +160,7 @@ func (p *Factory) TableAdminClientForWatch(ctx *types.APIRequest, s *types.APISc return p.AdminClientForWatch(ctx, s, namespace) } -func newClient(ctx *types.APIRequest, cfg *rest.Config, s *types.APISchema, namespace string, impersonate bool) (dynamic.ResourceInterface, error) { +func setupConfig(ctx *types.APIRequest, cfg *rest.Config, impersonate bool) (*rest.Config, error) { if impersonate { user, ok := request.UserFrom(ctx.Context()) if !ok { @@ -147,6 +171,14 @@ func newClient(ctx *types.APIRequest, cfg *rest.Config, s *types.APISchema, name cfg.Impersonate.Groups = user.GetGroups() cfg.Impersonate.Extra = user.GetExtra() } + return cfg, nil +} + +func newClient(ctx *types.APIRequest, cfg *rest.Config, s *types.APISchema, namespace string, impersonate bool) (dynamic.ResourceInterface, error) { + cfg, err := setupConfig(ctx, cfg, impersonate) + if err != nil { + return nil, err + } client, err := dynamic.NewForConfig(cfg) if err != nil { diff --git a/pkg/schemaserver/handlers/list.go b/pkg/schemaserver/handlers/list.go index f0097d3..fc4c6b6 100644 --- a/pkg/schemaserver/handlers/list.go +++ b/pkg/schemaserver/handlers/list.go @@ -16,7 +16,19 @@ func ByIDHandler(request *types.APIRequest) (types.APIObject, error) { return types.APIObject{}, httperror.NewAPIError(validation.NotFound, "no store found") } - return store.ByID(request, request.Schema, request.Name) + resp, err := store.ByID(request, request.Schema, request.Name) + if err != nil { + return resp, err + } + + if request.Link != "" { + if handler, ok := request.Schema.LinkHandlers[request.Link]; ok { + handler.ServeHTTP(request.Response, request.Request) + return types.APIObject{}, validation.ErrComplete + } + } + + return resp, nil } func ListHandler(request *types.APIRequest) (types.APIObjectList, error) { @@ -35,19 +47,5 @@ func ListHandler(request *types.APIRequest) (types.APIObjectList, error) { return types.APIObjectList{}, httperror.NewAPIError(validation.NotFound, "no store found") } - if request.Link == "" { - return store.List(request, request.Schema) - } - - _, err := store.ByID(request, request.Schema, request.Name) - if err != nil { - return types.APIObjectList{}, err - } - - if handler, ok := request.Schema.LinkHandlers[request.Link]; ok { - handler.ServeHTTP(request.Response, request.Request) - return types.APIObjectList{}, validation.ErrComplete - } - - return types.APIObjectList{}, validation.NotFound + return store.List(request, request.Schema) } diff --git a/pkg/schemaserver/urlbuilder/url.go b/pkg/schemaserver/urlbuilder/url.go index 880e2bc..d552f9a 100644 --- a/pkg/schemaserver/urlbuilder/url.go +++ b/pkg/schemaserver/urlbuilder/url.go @@ -72,7 +72,10 @@ func (u *DefaultURLBuilder) Marker(marker string) string { } func (u *DefaultURLBuilder) Link(schema *types.APISchema, id string, linkName string) string { - return u.schemaURL(schema, id, linkName) + if strings.Contains(id, "/") { + return u.schemaURL(schema, id, linkName) + } + return u.schemaURL(schema, id) + "?link=" + url.QueryEscape(linkName) } func (u *DefaultURLBuilder) ResourceLink(schema *types.APISchema, id string) string { diff --git a/pkg/schemaserver/writer/encoding.go b/pkg/schemaserver/writer/encoding.go index f4bd2ce..8115f2e 100644 --- a/pkg/schemaserver/writer/encoding.go +++ b/pkg/schemaserver/writer/encoding.go @@ -102,6 +102,15 @@ func (j *EncodingResponseWriter) addLinks(schema *types.APISchema, context *type rawResource.Links["remove"] = self } } + for link := range schema.LinkHandlers { + rawResource.Links[link] = context.URLBuilder.Link(schema, rawResource.ID, link) + } + for action := range schema.ActionHandlers { + if rawResource.Actions == nil { + rawResource.Actions = map[string]string{} + } + rawResource.Actions[action] = context.URLBuilder.Action(schema, rawResource.ID, action) + } } func getLimit(req *http.Request) int { diff --git a/pkg/server/resources/clusters/clusters.go b/pkg/server/resources/clusters/clusters.go index 4e69b11..06ba0dd 100644 --- a/pkg/server/resources/clusters/clusters.go +++ b/pkg/server/resources/clusters/clusters.go @@ -3,6 +3,8 @@ package clusters import ( "net/http" + "github.com/rancher/steve/pkg/server/store/proxy" + "github.com/rancher/steve/pkg/schemaserver/store/empty" "github.com/rancher/steve/pkg/schemaserver/types" "github.com/rancher/wrangler/pkg/schemas/validation" @@ -29,11 +31,20 @@ var ( type Cluster struct { } -func Register(schemas *types.APISchemas) { +func Register(schemas *types.APISchemas, cg proxy.ClientGetter) { schemas.MustImportAndCustomize(Cluster{}, func(schema *types.APISchema) { schema.CollectionMethods = []string{http.MethodGet} schema.ResourceMethods = []string{http.MethodGet} schema.Store = &Store{} + + shell := &shell{ + cg: cg, + namespace: "dashboard-shells", + } + schema.LinkHandlers = map[string]http.Handler{ + "shell": shell, + } + schema.Formatter = func(request *types.APIRequest, resource *types.RawResource) { resource.Links["api"] = request.URLBuilder.RelativeToRoot("/k8s/clusters/" + resource.ID) } diff --git a/pkg/server/resources/clusters/shell.go b/pkg/server/resources/clusters/shell.go new file mode 100644 index 0000000..54c0b3f --- /dev/null +++ b/pkg/server/resources/clusters/shell.go @@ -0,0 +1,343 @@ +package clusters + +import ( + "context" + "fmt" + "net/http" + "net/http/httputil" + "time" + + "github.com/rancher/steve/pkg/schemaserver/types" + "github.com/rancher/steve/pkg/server/store/proxy" + "github.com/rancher/wrangler/pkg/condition" + "github.com/rancher/wrangler/pkg/schemas/validation" + "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +type shell struct { + namespace string + cg proxy.ClientGetter +} + +func (s *shell) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + ctx, user, client, err := s.contextAndClient(req) + if err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + + role, err := s.createRole(ctx, user, client) + if err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + defer client.RbacV1().ClusterRoles().Delete(ctx, role.Name, metav1.DeleteOptions{}) + + pod, err := s.createPod(ctx, user, role, client) + if err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + + s.proxyRequest(rw, req, pod, client) +} + +func (s *shell) proxyRequest(rw http.ResponseWriter, req *http.Request, pod *v1.Pod, client kubernetes.Interface) { + attachURL := client.CoreV1().RESTClient(). + Get(). + Namespace(pod.Namespace). + Resource("pods"). + Name(pod.Name). + SubResource("attach"). + VersionedParams(&v1.PodAttachOptions{ + TypeMeta: metav1.TypeMeta{}, + Stdin: false, + Stdout: false, + Stderr: false, + TTY: false, + Container: "", + }, scheme.ParameterCodec).URL() + + httpClient := client.CoreV1().RESTClient().(*rest.RESTClient).Client + p := httputil.ReverseProxy{ + Director: func(req *http.Request) { + req.URL = attachURL + req.Host = attachURL.Host + delete(req.Header, "Authorization") + delete(req.Header, "Cookie") + }, + Transport: httpClient.Transport, + FlushInterval: time.Millisecond * 100, + } + + p.ServeHTTP(rw, req) +} + +func (s *shell) contextAndClient(req *http.Request) (context.Context, user.Info, kubernetes.Interface, error) { + ctx := req.Context() + apiContext := types.GetAPIContext(req.Context()) + + client, err := s.cg.AdminK8sInterface(apiContext) + if err != nil { + return ctx, nil, nil, err + } + + user, ok := request.UserFrom(ctx) + if !ok { + return ctx, nil, nil, validation.Unauthorized + } + + return ctx, user, client, nil +} + +func (s *shell) createNamespace(ctx context.Context, client kubernetes.Interface) (*v1.Namespace, error) { + ns, err := client.CoreV1().Namespaces().Get(ctx, s.namespace, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return client.CoreV1().Namespaces().Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: s.namespace, + }, + }, metav1.CreateOptions{}) + } + return ns, err +} + +func (s *shell) createRole(ctx context.Context, user user.Info, client kubernetes.Interface) (*rbacv1.ClusterRole, error) { + _, err := s.createNamespace(ctx, client) + if err != nil { + return nil, err + } + + return client.RbacV1().ClusterRoles().Create(ctx, &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "dashboard-shell-", + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"impersonate"}, + APIGroups: []string{""}, + Resources: []string{"users"}, + ResourceNames: []string{user.GetName()}, + }, + { + Verbs: []string{"impersonate"}, + APIGroups: []string{""}, + Resources: []string{"groups"}, + ResourceNames: user.GetGroups(), + }, + }, + AggregationRule: nil, + }, metav1.CreateOptions{}) + +} + +func (s *shell) createRoleBinding(ctx context.Context, role *rbacv1.ClusterRole, serviceAccount *v1.ServiceAccount, client kubernetes.Interface) error { + _, err := client.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "dashboard-shell-", + OwnerReferences: ref(role), + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + APIGroup: "", + Name: serviceAccount.Name, + Namespace: serviceAccount.Namespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: role.Name, + }, + }, metav1.CreateOptions{}) + return err +} + +func ref(role *rbacv1.ClusterRole) []metav1.OwnerReference { + ref := metav1.OwnerReference{ + Name: role.Name, + UID: role.UID, + } + ref.APIVersion, ref.Kind = rbacv1.SchemeGroupVersion.WithKind("ClusterRole").ToAPIVersionAndKind() + return []metav1.OwnerReference{ + ref, + } +} + +func (s *shell) updateServiceAccount(ctx context.Context, pod *v1.Pod, serviceAccount *v1.ServiceAccount, client kubernetes.Interface) error { + serviceAccount, err := client.CoreV1().ServiceAccounts(s.namespace).Get(ctx, serviceAccount.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + serviceAccount.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "Pod", + Name: pod.Name, + UID: pod.UID, + }, + } + + _, err = client.CoreV1().ServiceAccounts(s.namespace).Update(ctx, serviceAccount, metav1.UpdateOptions{}) + return err +} + +func (s *shell) createServiceAccount(ctx context.Context, role *rbacv1.ClusterRole, client kubernetes.Interface) (*v1.ServiceAccount, error) { + return client.CoreV1().ServiceAccounts(s.namespace).Create(ctx, &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "dashboard-shell-", + OwnerReferences: ref(role), + }, + }, metav1.CreateOptions{}) +} + +func (s *shell) createConfigMap(ctx context.Context, user user.Info, role *rbacv1.ClusterRole, client kubernetes.Interface) (*v1.ConfigMap, error) { + cfg := clientcmdapi.Config{ + Clusters: map[string]*clientcmdapi.Cluster{ + "cluster": { + Server: "https://kubernetes.default", + CertificateAuthority: "/run/secrets/kubernetes.io/serviceaccount/ca.crt", + }, + }, + AuthInfos: map[string]*clientcmdapi.AuthInfo{ + "user": { + TokenFile: "/run/secrets/kubernetes.io/serviceaccount/token", + Impersonate: user.GetName(), + ImpersonateGroups: user.GetGroups(), + }, + }, + Contexts: map[string]*clientcmdapi.Context{ + "default": { + Cluster: "cluster", + AuthInfo: "user", + }, + }, + CurrentContext: "default", + } + + cfgData, err := clientcmd.Write(cfg) + if err != nil { + return nil, err + } + + return client.CoreV1().ConfigMaps(s.namespace).Create(ctx, &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "dashboard-shell-", + Namespace: s.namespace, + OwnerReferences: ref(role), + }, + Data: map[string]string{ + "config": string(cfgData), + }, + }, metav1.CreateOptions{}) +} + +func (s *shell) createPod(ctx context.Context, user user.Info, role *rbacv1.ClusterRole, client kubernetes.Interface) (*v1.Pod, error) { + sa, err := s.createServiceAccount(ctx, role, client) + if err != nil { + return nil, err + } + + if err := s.createRoleBinding(ctx, role, sa, client); err != nil { + return nil, err + } + + cm, err := s.createConfigMap(ctx, user, role, client) + if err != nil { + return nil, err + } + + hour := int64(15) + pod, err := client.CoreV1().Pods(s.namespace).Create(ctx, &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "dashboard-shell-", + Namespace: s.namespace, + Labels: map[string]string{ + "clusterrolename": role.Name, + "clusterroleuid": string(role.UID), + }, + OwnerReferences: ref(role), + }, + Spec: v1.PodSpec{ + ActiveDeadlineSeconds: &hour, + Volumes: []v1.Volume{ + { + Name: "config", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: cm.Name, + }, + }, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + ServiceAccountName: sa.Name, + Containers: []v1.Container{ + { + Name: "shell", + TTY: true, + Stdin: true, + StdinOnce: true, + Image: "rancher/rancher-agent:v2.4.3", + ImagePullPolicy: v1.PullIfNotPresent, + Command: []string{"bash"}, + VolumeMounts: []v1.VolumeMount{ + { + Name: "config", + ReadOnly: true, + MountPath: "/root/.kube/config", + SubPath: "config", + }, + }, + }, + }, + }, + }, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + + // ignore any error here + err = s.updateServiceAccount(ctx, pod, sa, client) + if err != nil { + logrus.Warnf("failed to update service account %s/%s to be owned by pod %s/%s", sa.Namespace, sa.Name, pod.Namespace, pod.Name) + } + + sec := int64(60) + resp, err := client.CoreV1().Pods(s.namespace).Watch(ctx, metav1.ListOptions{ + FieldSelector: "metadata.name=" + pod.Name, + ResourceVersion: pod.ResourceVersion, + TimeoutSeconds: &sec, + }) + if err != nil { + return nil, err + } + defer resp.Stop() + + for event := range resp.ResultChan() { + newPod, ok := event.Object.(*v1.Pod) + if !ok { + continue + } + if condition.Cond(v1.PodReady).IsTrue(newPod) { + return newPod, nil + } + } + + return nil, fmt.Errorf("failed to find shell") +} diff --git a/pkg/server/resources/schema.go b/pkg/server/resources/schema.go index d958774..163e5ac 100644 --- a/pkg/server/resources/schema.go +++ b/pkg/server/resources/schema.go @@ -22,7 +22,7 @@ func DefaultSchemas(baseSchema *types.APISchemas, ccache clustercache.ClusterCac subscribe.Register(baseSchema) apiroot.Register(baseSchema, []string{"v1"}, []string{"proxy:/apis"}) userpreferences.Register(baseSchema, cg) - clusters.Register(baseSchema) + clusters.Register(baseSchema, cg) return baseSchema } diff --git a/pkg/server/router/router.go b/pkg/server/router/router.go index 5155bc6..86158bf 100644 --- a/pkg/server/router/router.go +++ b/pkg/server/router/router.go @@ -27,10 +27,13 @@ func Routes(h Handlers) http.Handler { m.Path("/{name:v1}").Handler(h.APIRoot) m.Path("/v1/{type}").Handler(h.K8sResource) - m.Path("/v1/{type}/{nameorns}").Handler(h.K8sResource) - m.Path("/v1/{type}/{namespace}/{name}").Handler(h.K8sResource) + m.Path("/v1/{type}/{nameorns}").Queries("link", "{link}").Handler(h.K8sResource) m.Path("/v1/{type}/{nameorns}").Queries("action", "{action}").Handler(h.K8sResource) + m.Path("/v1/{type}/{nameorns}").Handler(h.K8sResource) m.Path("/v1/{type}/{namespace}/{name}").Queries("action", "{action}").Handler(h.K8sResource) + m.Path("/v1/{type}/{namespace}/{name}").Queries("link", "{link}").Handler(h.K8sResource) + m.Path("/v1/{type}/{namespace}/{name}").Handler(h.K8sResource) + m.Path("/v1/{type}/{namespace}/{name}/{link}").Handler(h.K8sResource) m.Path("/api").Handler(h.K8sProxy) // Can't just prefix this as UI needs /apikeys path m.PathPrefix("/api/").Handler(h.K8sProxy) m.PathPrefix("/apis").Handler(h.K8sProxy) diff --git a/pkg/server/store/proxy/proxy_store.go b/pkg/server/store/proxy/proxy_store.go index a1afe05..5019f82 100644 --- a/pkg/server/store/proxy/proxy_store.go +++ b/pkg/server/store/proxy/proxy_store.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" ) var ( @@ -31,6 +32,9 @@ var ( ) type ClientGetter interface { + IsImpersonating() bool + K8sInterface(ctx *types.APIRequest) (kubernetes.Interface, error) + AdminK8sInterface(ctx *types.APIRequest) (kubernetes.Interface, error) Client(ctx *types.APIRequest, schema *types.APISchema, namespace string) (dynamic.ResourceInterface, error) AdminClient(ctx *types.APIRequest, schema *types.APISchema, namespace string) (dynamic.ResourceInterface, error) TableClient(ctx *types.APIRequest, schema *types.APISchema, namespace string) (dynamic.ResourceInterface, error)