From 985d502ed3d47da3987dfc4c25d82facd5103b5a Mon Sep 17 00:00:00 2001 From: deads2k Date: Fri, 2 Dec 2016 11:19:06 -0500 Subject: [PATCH] add basic wiring for kubernetes-discovery to become a kube-like api server --- cmd/kubernetes-discovery/BUILD | 1 + cmd/kubernetes-discovery/main.go | 3 +- cmd/kubernetes-discovery/pkg/apiserver/BUILD | 23 +++++ .../pkg/apiserver/apiserver.go | 89 +++++++++++++++++++ cmd/kubernetes-discovery/pkg/cmd/server/BUILD | 11 +++ .../pkg/cmd/server/start.go | 80 +++++++++++++++-- cmd/kubernetes-discovery/pkg/legacy/doc.go | 2 +- hack/local-up-cluster.sh | 29 +++++- 8 files changed, 229 insertions(+), 9 deletions(-) create mode 100644 cmd/kubernetes-discovery/pkg/apiserver/BUILD create mode 100644 cmd/kubernetes-discovery/pkg/apiserver/apiserver.go diff --git a/cmd/kubernetes-discovery/BUILD b/cmd/kubernetes-discovery/BUILD index 815d3ca3741..71499fc00d6 100644 --- a/cmd/kubernetes-discovery/BUILD +++ b/cmd/kubernetes-discovery/BUILD @@ -16,6 +16,7 @@ go_binary( tags = ["automanaged"], deps = [ "//cmd/kubernetes-discovery/pkg/cmd/server:go_default_library", + "//pkg/kubectl/cmd/util:go_default_library", "//pkg/util/logs:go_default_library", ], ) diff --git a/cmd/kubernetes-discovery/main.go b/cmd/kubernetes-discovery/main.go index bbea56672d8..2bde47a0d2f 100644 --- a/cmd/kubernetes-discovery/main.go +++ b/cmd/kubernetes-discovery/main.go @@ -21,6 +21,7 @@ import ( "runtime" "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/cmd/server" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/util/logs" ) @@ -34,6 +35,6 @@ func main() { cmd := server.NewCommandStartDiscoveryServer(os.Stdout, os.Stderr) if err := cmd.Execute(); err != nil { - os.Exit(1) + cmdutil.CheckErr(err) } } diff --git a/cmd/kubernetes-discovery/pkg/apiserver/BUILD b/cmd/kubernetes-discovery/pkg/apiserver/BUILD new file mode 100644 index 00000000000..00ad8a80619 --- /dev/null +++ b/cmd/kubernetes-discovery/pkg/apiserver/BUILD @@ -0,0 +1,23 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_binary", + "go_library", + "go_test", + "cgo_library", +) + +go_library( + name = "go_default_library", + srcs = ["apiserver.go"], + tags = ["automanaged"], + deps = [ + "//pkg/genericapiserver:go_default_library", + "//pkg/registry/generic:go_default_library", + "//pkg/runtime/schema:go_default_library", + "//pkg/version:go_default_library", + ], +) diff --git a/cmd/kubernetes-discovery/pkg/apiserver/apiserver.go b/cmd/kubernetes-discovery/pkg/apiserver/apiserver.go new file mode 100644 index 00000000000..500384db5f2 --- /dev/null +++ b/cmd/kubernetes-discovery/pkg/apiserver/apiserver.go @@ -0,0 +1,89 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiserver + +import ( + "k8s.io/kubernetes/pkg/genericapiserver" + "k8s.io/kubernetes/pkg/registry/generic" + "k8s.io/kubernetes/pkg/runtime/schema" + "k8s.io/kubernetes/pkg/version" +) + +// TODO move to genericapiserver or something like that +// RESTOptionsGetter is used to construct storage for a particular resource +type RESTOptionsGetter interface { + NewFor(resource schema.GroupResource) generic.RESTOptions +} + +type Config struct { + GenericConfig *genericapiserver.Config + + // RESTOptionsGetter is used to construct storage for a particular resource + RESTOptionsGetter RESTOptionsGetter +} + +// APIDiscoveryServer contains state for a Kubernetes cluster master/api server. +type APIDiscoveryServer struct { + GenericAPIServer *genericapiserver.GenericAPIServer +} + +type completedConfig struct { + *Config +} + +// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. +func (c *Config) Complete() completedConfig { + c.GenericConfig.Complete() + + version := version.Get() + c.GenericConfig.Version = &version + + return completedConfig{c} +} + +// SkipComplete provides a way to construct a server instance without config completion. +func (c *Config) SkipComplete() completedConfig { + return completedConfig{c} +} + +// New returns a new instance of APIDiscoveryServer from the given config. +func (c completedConfig) New() (*APIDiscoveryServer, error) { + genericServer, err := c.Config.GenericConfig.SkipComplete().New() // completion is done in Complete, no need for a second time + if err != nil { + return nil, err + } + + s := &APIDiscoveryServer{ + GenericAPIServer: genericServer, + } + + // TODO switch to constants once we have an API + // TODO install RESTStorage for API + // apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo("apifederation.k8s.io") + // apiGroupInfo.GroupMeta.GroupVersion = schema.GroupVersion{Group: "apifederation.k8s.io", Version: "v1alpha1"} + + // v1alpha1storage := map[string]rest.Storage{} + // // v1alpha1storage["apiservices"] = apiserverstorage.NewREST(c.RESTOptionsGetter.NewFor(apifederation.Resource("apiservices"))) + + // apiGroupInfo.VersionedResourcesStorageMap["v1alpha1"] = v1alpha1storage + + // if err := s.GenericAPIServer.InstallAPIGroup(&apiGroupInfo); err != nil { + // return nil, err + // } + + return s, nil +} diff --git a/cmd/kubernetes-discovery/pkg/cmd/server/BUILD b/cmd/kubernetes-discovery/pkg/cmd/server/BUILD index 9da48f077f4..c9f0d6a8955 100644 --- a/cmd/kubernetes-discovery/pkg/cmd/server/BUILD +++ b/cmd/kubernetes-discovery/pkg/cmd/server/BUILD @@ -15,8 +15,19 @@ go_library( srcs = ["start.go"], tags = ["automanaged"], deps = [ + "//cmd/kubernetes-discovery/pkg/apiserver:go_default_library", "//cmd/kubernetes-discovery/pkg/legacy:go_default_library", + "//pkg/api:go_default_library", + "//pkg/apimachinery/registered:go_default_library", + "//pkg/genericapiserver:go_default_library", + "//pkg/genericapiserver/options:go_default_library", "//pkg/kubectl/cmd/util:go_default_library", + "//pkg/registry/generic:go_default_library", + "//pkg/registry/generic/registry:go_default_library", + "//pkg/runtime/schema:go_default_library", + "//pkg/storage/storagebackend:go_default_library", + "//pkg/util/wait:go_default_library", + "//vendor:github.com/pborman/uuid", "//vendor:github.com/spf13/cobra", ], ) diff --git a/cmd/kubernetes-discovery/pkg/cmd/server/start.go b/cmd/kubernetes-discovery/pkg/cmd/server/start.go index 9bc2d6bf033..009dd16d6dd 100644 --- a/cmd/kubernetes-discovery/pkg/cmd/server/start.go +++ b/cmd/kubernetes-discovery/pkg/cmd/server/start.go @@ -17,15 +17,32 @@ limitations under the License. package server import ( + "fmt" "io" + "github.com/pborman/uuid" "github.com/spf13/cobra" + "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/apiserver" "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/legacy" - kcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/apimachinery/registered" + "k8s.io/kubernetes/pkg/genericapiserver" + genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/registry/generic" + "k8s.io/kubernetes/pkg/registry/generic/registry" + "k8s.io/kubernetes/pkg/runtime/schema" + "k8s.io/kubernetes/pkg/storage/storagebackend" + "k8s.io/kubernetes/pkg/util/wait" ) +const defaultEtcdPathPrefix = "/registry/kubernetes.io/kubernetes-discovery" + type DiscoveryServerOptions struct { + Etcd *genericoptions.EtcdOptions + SecureServing *genericoptions.SecureServingOptions + StdOut io.Writer StdErr io.Writer } @@ -33,20 +50,30 @@ type DiscoveryServerOptions struct { // NewCommandStartMaster provides a CLI handler for 'start master' command func NewCommandStartDiscoveryServer(out, err io.Writer) *cobra.Command { o := &DiscoveryServerOptions{ + Etcd: genericoptions.NewEtcdOptions(), + SecureServing: genericoptions.NewSecureServingOptions(), + StdOut: out, StdErr: err, } + o.Etcd.StorageConfig.Prefix = defaultEtcdPathPrefix + o.Etcd.StorageConfig.Codec = api.Codecs.LegacyCodec(registered.EnabledVersionsForGroup(api.GroupName)...) + o.SecureServing.ServingOptions.BindPort = 9090 cmd := &cobra.Command{ Short: "Launch a discovery summarizer and proxy server", Long: "Launch a discovery summarizer and proxy server", Run: func(c *cobra.Command, args []string) { - kcmdutil.CheckErr(o.Complete()) - kcmdutil.CheckErr(o.Validate(args)) - kcmdutil.CheckErr(o.RunDiscoveryServer()) + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.Validate(args)) + cmdutil.CheckErr(o.RunDiscoveryServer()) }, } + flags := cmd.Flags() + o.Etcd.AddFlags(flags) + o.SecureServing.AddFlags(flags) + return cmd } @@ -59,11 +86,38 @@ func (o *DiscoveryServerOptions) Complete() error { } func (o DiscoveryServerOptions) RunDiscoveryServer() error { - if true { - // for now this is the only option. later, only use this if no etcd is configured + // if we don't have an etcd to back the server, we must be a legacy server + if len(o.Etcd.StorageConfig.ServerList) == 0 { return o.RunLegacyDiscoveryServer() } + // TODO have a "real" external address + if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost"); err != nil { + return fmt.Errorf("error creating self-signed certificates: %v", err) + } + + genericAPIServerConfig := genericapiserver.NewConfig() + if _, err := genericAPIServerConfig.ApplySecureServingOptions(o.SecureServing); err != nil { + return err + } + + var err error + privilegedLoopbackToken := uuid.NewRandom().String() + if genericAPIServerConfig.LoopbackClientConfig, err = genericAPIServerConfig.SecureServingInfo.NewSelfClientConfig(privilegedLoopbackToken); err != nil { + return err + } + + config := apiserver.Config{ + GenericConfig: genericAPIServerConfig, + RESTOptionsGetter: restOptionsFactory{storageConfig: &o.Etcd.StorageConfig}, + } + + server, err := config.Complete().New() + if err != nil { + return err + } + server.GenericAPIServer.PrepareRun().Run(wait.NeverStop) + return nil } @@ -77,3 +131,17 @@ func (o DiscoveryServerOptions) RunLegacyDiscoveryServer() error { } return s.Run(port) } + +type restOptionsFactory struct { + storageConfig *storagebackend.Config +} + +func (f restOptionsFactory) NewFor(resource schema.GroupResource) generic.RESTOptions { + return generic.RESTOptions{ + StorageConfig: f.storageConfig, + Decorator: registry.StorageWithCacher, + DeleteCollectionWorkers: 1, + EnableGarbageCollection: false, + ResourcePrefix: f.storageConfig.Prefix + "/" + resource.Group + "/" + resource.Resource, + } +} diff --git a/cmd/kubernetes-discovery/pkg/legacy/doc.go b/cmd/kubernetes-discovery/pkg/legacy/doc.go index 3cf7af401f0..589bfeb0851 100644 --- a/cmd/kubernetes-discovery/pkg/legacy/doc.go +++ b/cmd/kubernetes-discovery/pkg/legacy/doc.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package discoverysummarizer contains code for the discovery summarizer +// Package discoverysummarizer contains code for the legacy discovery summarizer // (program to summarize discovery information from all federated api servers) // as per https://github.com/kubernetes/kubernetes/blob/master/docs/proposals/federated-api-servers.md package legacy // import "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/legacy" diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index ccfd1a39017..3c61d129fd4 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -135,7 +135,7 @@ do done if [ "x$GO_OUT" == "x" ]; then - make -C "${KUBE_ROOT}" WHAT="cmd/kubectl cmd/hyperkube" + make -C "${KUBE_ROOT}" WHAT="cmd/kubectl cmd/hyperkube cmd/kubernetes-discovery" else echo "skipped the build." fi @@ -198,6 +198,8 @@ ENABLE_CONTROLLER_ATTACH_DETACH=${ENABLE_CONTROLLER_ATTACH_DETACH:-"true"} # cur CERT_DIR=${CERT_DIR:-"/var/run/kubernetes"} ROOT_CA_FILE=$CERT_DIR/apiserver.crt EXPERIMENTAL_CRI=${EXPERIMENTAL_CRI:-"false"} +DISCOVERY_SECURE_PORT=${DISCOVERY_SECURE_PORT:-9090} + # Ensure CERT_DIR is created for auto-generated crt/key and kubeconfig mkdir -p "${CERT_DIR}" &>/dev/null || sudo mkdir -p "${CERT_DIR}" @@ -300,6 +302,10 @@ cleanup() [[ -n "${APISERVER_PID-}" ]] && APISERVER_PIDS=$(pgrep -P ${APISERVER_PID} ; ps -o pid= -p ${APISERVER_PID}) [[ -n "${APISERVER_PIDS-}" ]] && sudo kill ${APISERVER_PIDS} + # Check if the discovery server is still running + [[ -n "${DISCOVERY_PID-}" ]] && DISCOVERY_PIDS=$(pgrep -P ${DISCOVERY_PID} ; ps -o pid= -p ${DISCOVERY_PID}) + [[ -n "${DISCOVERY_PIDS-}" ]] && sudo kill ${DISCOVERY_PIDS} + # Check if the controller-manager is still running [[ -n "${CTLRMGR_PID-}" ]] && CTLRMGR_PIDS=$(pgrep -P ${CTLRMGR_PID} ; ps -o pid= -p ${CTLRMGR_PID}) [[ -n "${CTLRMGR_PIDS-}" ]] && sudo kill ${CTLRMGR_PIDS} @@ -504,6 +510,26 @@ EOF fi } +# start_discovery relies on certificates created by start_apiserver +function start_discovery { + # TODO generate serving certificates + + DISCOVERY_SERVER_LOG=/tmp/kubernetes-discovery.log + ${CONTROLPLANE_SUDO} "${GO_OUT}/kubernetes-discovery" \ + --cert-dir="${CERT_DIR}" \ + --client-ca-file="${CERT_DIR}/client-ca-bundle.crt" \ + --bind-address="${API_BIND_ADDR}" \ + --secure-port="${DISCOVERY_SECURE_PORT}" \ + --tls-ca-file="${ROOT_CA_FILE}" \ + --etcd-servers="http://${ETCD_HOST}:${ETCD_PORT}" >"${DISCOVERY_SERVER_LOG}" 2>&1 & + DISCOVERY_PID=$! + + # Wait for kubernetes-discovery to come up before launching the rest of the components. + echo "Waiting for kubernetes-discovery to come up" + kube::util::wait_for_url "https://${API_HOST}:${DISCOVERY_SECURE_PORT}/version" "kubernetes-discovery: " 1 ${WAIT_FOR_URL_API_SERVER} || exit 1 +} + + function start_controller_manager { node_cidr_args="" if [[ "${NET_PLUGIN}" == "kubenet" ]]; then @@ -778,6 +804,7 @@ if [[ "${START_MODE}" != "kubeletonly" ]]; then start_etcd set_service_accounts start_apiserver + start_discovery start_controller_manager start_kubeproxy start_kubedns