Merge pull request #31491 from dims/fixes-issue-13598

Automatic merge from submit-queue

Allow secure access to apiserver from Admission Controllers

* Allow options.InsecurePort to be set to 0 to switch off insecure access
* In NewSelfClient, Set the TLSClientConfig to the cert and key files
  if InsecurePort is switched off
* Mint a bearer token that allows the client(s) created in NewSelfClient
  to talk to the api server
* Add a new authenticator that checks for this specific bearer token

Fixes #13598

cmd/kube-apiserver/app/server.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/golang/glog"
+	"github.com/pborman/uuid"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
@@ -41,6 +42,8 @@ import (
 	"k8s.io/kubernetes/pkg/apis/rbac"
 	"k8s.io/kubernetes/pkg/apiserver"
 	"k8s.io/kubernetes/pkg/apiserver/authenticator"
+	authorizerunion "k8s.io/kubernetes/pkg/auth/authorizer/union"
+	"k8s.io/kubernetes/pkg/auth/user"
 	"k8s.io/kubernetes/pkg/capabilities"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/controller/informers"
@@ -64,6 +67,7 @@ import (
 	rolebindingetcd "k8s.io/kubernetes/pkg/registry/rbac/rolebinding/etcd"
 	"k8s.io/kubernetes/pkg/serviceaccount"
 	"k8s.io/kubernetes/pkg/util/wait"
+	authenticatorunion "k8s.io/kubernetes/plugin/pkg/auth/authenticator/request/union"
 )
 
 // NewAPIServerCommand creates a *cobra.Command object with default parameters
@@ -196,7 +200,7 @@ func Run(s *options.APIServer) error {
 		serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(storageConfig, storageFactory.ResourcePrefix(api.Resource("serviceaccounts")), storageFactory.ResourcePrefix(api.Resource("secrets")))
 	}
 
-	authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{
+	apiAuthenticator, err := authenticator.New(authenticator.AuthenticatorConfig{
 		BasicAuthFile:               s.BasicAuthFile,
 		ClientCAFile:                s.ClientCAFile,
 		TokenAuthFile:               s.TokenAuthFile,
@@ -251,16 +255,36 @@ func Run(s *options.APIServer) error {
 		authorizationConfig.RBACClusterRoleBindingRegistry = clusterrolebinding.NewRegistry(clusterrolebindingetcd.NewREST(mustGetRESTOptions("clusterrolebindings")))
 	}
 
-	authorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, authorizationConfig)
+	apiAuthorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, authorizationConfig)
 	if err != nil {
 		glog.Fatalf("Invalid Authorization Config: %v", err)
 	}
 
 	admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")
-	client, err := s.NewSelfClient()
+	privilegedLoopbackToken := uuid.NewRandom().String()
+
+	client, err := s.NewSelfClient(privilegedLoopbackToken)
 	if err != nil {
 		glog.Errorf("Failed to create clientset: %v", err)
 	}
+
+	// TODO(dims): We probably need to add an option "EnableLoopbackToken"
+	if apiAuthenticator != nil {
+		var uid = uuid.NewRandom().String()
+		tokens := make(map[string]*user.DefaultInfo)
+		tokens[privilegedLoopbackToken] = &user.DefaultInfo{
+			Name:   "system:apiserver",
+			UID:    uid,
+			Groups: []string{"system:masters"},
+		}
+
+		tokenAuthenticator := authenticator.NewAuthenticatorFromTokens(tokens)
+		apiAuthenticator = authenticatorunion.New(apiAuthenticator, tokenAuthenticator)
+
+		tokenAuthorizer := authorizer.NewPrivilegedGroups("system:masters")
+		apiAuthorizer = authorizerunion.New(apiAuthorizer, tokenAuthorizer)
+	}
+
 	sharedInformers := informers.NewSharedInformerFactory(client, 10*time.Minute)
 	pluginInitializer := admission.NewPluginInitializer(sharedInformers)
 
@@ -272,9 +296,9 @@ func Run(s *options.APIServer) error {
 	genericConfig := genericapiserver.NewConfig(s.ServerRunOptions)
 	// TODO: Move the following to generic api server as well.
 	genericConfig.StorageFactory = storageFactory
-	genericConfig.Authenticator = authenticator
+	genericConfig.Authenticator = apiAuthenticator
 	genericConfig.SupportsBasicAuth = len(s.BasicAuthFile) > 0
-	genericConfig.Authorizer = authorizer
+	genericConfig.Authorizer = apiAuthorizer
 	genericConfig.AuthorizerRBACSuperUser = s.AuthorizationRBACSuperUser
 	genericConfig.AdmissionControl = admissionController
 	genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource

examples/apiserver/apiserver.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/kubernetes/pkg/api/unversioned"
 	"k8s.io/kubernetes/pkg/apimachinery/registered"
 	"k8s.io/kubernetes/pkg/genericapiserver"
+	"k8s.io/kubernetes/pkg/genericapiserver/authorizer"
 	genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options"
 	genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation"
 	"k8s.io/kubernetes/pkg/storage/storagebackend"
@@ -39,6 +40,7 @@ const (
 	// Ports on which to run the server.
 	// Explicitly setting these to a different value than the default values, to prevent this from clashing with a local cluster.
 	InsecurePort = 8081
+	SecurePort   = 6444
 )
 
 func newStorageFactory() genericapiserver.StorageFactory {
@@ -65,6 +67,7 @@ func Run(serverOptions *genericoptions.ServerRunOptions) error {
 	genericvalidation.ValidateRunOptions(serverOptions)
 	genericvalidation.VerifyEtcdServersList(serverOptions)
 	config := genericapiserver.NewConfig(serverOptions)
+	config.Authorizer = authorizer.NewAlwaysAllowAuthorizer()
 	config.Serializer = api.Codecs
 	s, err := config.New()
 	if err != nil {

federation/cmd/federation-apiserver/app/server.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/golang/glog"
+	"github.com/pborman/uuid"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
@@ -33,6 +34,8 @@ import (
 	"k8s.io/kubernetes/pkg/api/unversioned"
 	"k8s.io/kubernetes/pkg/apis/rbac"
 	"k8s.io/kubernetes/pkg/apiserver/authenticator"
+	authorizerunion "k8s.io/kubernetes/pkg/auth/authorizer/union"
+	"k8s.io/kubernetes/pkg/auth/user"
 	"k8s.io/kubernetes/pkg/controller/informers"
 	"k8s.io/kubernetes/pkg/generated/openapi"
 	"k8s.io/kubernetes/pkg/genericapiserver"
@@ -51,6 +54,7 @@ import (
 	rolebindingetcd "k8s.io/kubernetes/pkg/registry/rbac/rolebinding/etcd"
 	"k8s.io/kubernetes/pkg/routes"
 	"k8s.io/kubernetes/pkg/util/wait"
+	authenticatorunion "k8s.io/kubernetes/plugin/pkg/auth/authenticator/request/union"
 )
 
 // NewAPIServerCommand creates a *cobra.Command object with default parameters
@@ -109,7 +113,7 @@ func Run(s *options.ServerRunOptions) error {
 		storageFactory.SetEtcdLocation(groupResource, servers)
 	}
 
-	authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{
+	apiAuthenticator, err := authenticator.New(authenticator.AuthenticatorConfig{
 		BasicAuthFile:     s.BasicAuthFile,
 		ClientCAFile:      s.ClientCAFile,
 		TokenAuthFile:     s.TokenAuthFile,
@@ -158,16 +162,36 @@ func Run(s *options.ServerRunOptions) error {
 		authorizationConfig.RBACClusterRoleBindingRegistry = clusterrolebinding.NewRegistry(clusterrolebindingetcd.NewREST(mustGetRESTOptions("clusterrolebindings")))
 	}
 
-	authorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, authorizationConfig)
+	apiAuthorizer, err := authorizer.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, authorizationConfig)
 	if err != nil {
 		glog.Fatalf("Invalid Authorization Config: %v", err)
 	}
 
 	admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")
-	client, err := s.NewSelfClient()
+	privilegedLoopbackToken := uuid.NewRandom().String()
+
+	client, err := s.NewSelfClient(privilegedLoopbackToken)
 	if err != nil {
 		glog.Errorf("Failed to create clientset: %v", err)
 	}
+
+	// TODO(dims): We probably need to add an option "EnableLoopbackToken"
+	if apiAuthenticator != nil {
+		var uid = uuid.NewRandom().String()
+		tokens := make(map[string]*user.DefaultInfo)
+		tokens[privilegedLoopbackToken] = &user.DefaultInfo{
+			Name:   "system:apiserver",
+			UID:    uid,
+			Groups: []string{"system:masters"},
+		}
+
+		tokenAuthenticator := authenticator.NewAuthenticatorFromTokens(tokens)
+		apiAuthenticator = authenticatorunion.New(apiAuthenticator, tokenAuthenticator)
+
+		tokenAuthorizer := authorizer.NewPrivilegedGroups("system:masters")
+		apiAuthorizer = authorizerunion.New(apiAuthorizer, tokenAuthorizer)
+	}
+
 	sharedInformers := informers.NewSharedInformerFactory(client, 10*time.Minute)
 	pluginInitializer := admission.NewPluginInitializer(sharedInformers)
 
@@ -178,9 +202,9 @@ func Run(s *options.ServerRunOptions) error {
 	genericConfig := genericapiserver.NewConfig(s.ServerRunOptions)
 	// TODO: Move the following to generic api server as well.
 	genericConfig.StorageFactory = storageFactory
-	genericConfig.Authenticator = authenticator
+	genericConfig.Authenticator = apiAuthenticator
 	genericConfig.SupportsBasicAuth = len(s.BasicAuthFile) > 0
-	genericConfig.Authorizer = authorizer
+	genericConfig.Authorizer = apiAuthorizer
 	genericConfig.AuthorizerRBACSuperUser = s.AuthorizationRBACSuperUser
 	genericConfig.AdmissionControl = admissionController
 	genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource

hack/verify-flags/known-flags.txt

@@ -506,6 +506,7 @@ tcp-services
 terminated-pod-gc-threshold
 test-flags
 test-timeout
+tls-ca-file
 tls-cert-file
 tls-private-key-file
 to-version

pkg/apiserver/authenticator/authn.go

@@ -22,6 +22,7 @@ import (
 
 	"k8s.io/kubernetes/pkg/auth/authenticator"
 	"k8s.io/kubernetes/pkg/auth/authenticator/bearertoken"
+	"k8s.io/kubernetes/pkg/auth/user"
 	"k8s.io/kubernetes/pkg/serviceaccount"
 	"k8s.io/kubernetes/pkg/util/crypto"
 	"k8s.io/kubernetes/plugin/pkg/auth/authenticator/password/keystone"
@@ -154,6 +155,11 @@ func newAuthenticatorFromTokenFile(tokenAuthFile string) (authenticator.Request,
 	return bearertoken.New(tokenAuthenticator), nil
 }
 
+// newAuthenticatorFromToken returns an authenticator.Request or an error
+func NewAuthenticatorFromTokens(tokens map[string]*user.DefaultInfo) authenticator.Request {
+	return bearertoken.New(tokenfile.New(tokens))
+}
+
 // newAuthenticatorFromOIDCIssuerURL returns an authenticator.Request or an error.
 func newAuthenticatorFromOIDCIssuerURL(issuerURL, clientID, caFile, usernameClaim, groupsClaim string) (authenticator.Request, error) {
 	tokenAuthenticator, err := oidc.New(oidc.OIDCOptions{

pkg/genericapiserver/authorizer/authz.go

@@ -72,6 +72,28 @@ func NewAlwaysFailAuthorizer() authorizer.Authorizer {
 	return new(alwaysFailAuthorizer)
 }
 
+type privilegedGroupAuthorizer struct {
+	groups []string
+}
+
+func (r *privilegedGroupAuthorizer) Authorize(attr authorizer.Attributes) (bool, string, error) {
+	for attr_group := range attr.GetUser().GetGroups() {
+		for priv_group := range r.groups {
+			if priv_group == attr_group {
+				return true, "", nil
+			}
+		}
+	}
+	return false, "Not in privileged list.", nil
+}
+
+// NewPrivilegedGroups is for use in loopback scenarios
+func NewPrivilegedGroups(groups ...string) *privilegedGroupAuthorizer {
+	return &privilegedGroupAuthorizer{
+		groups: groups,
+	}
+}
+
 type AuthorizationConfig struct {
 	// Options for ModeABAC
 

pkg/genericapiserver/options/server_run_options.go

@@ -17,6 +17,7 @@ limitations under the License.
 package options
 
 import (
+	"errors"
 	"net"
 	"strconv"
 	"strings"
@@ -114,6 +115,7 @@ type ServerRunOptions struct {
 	// for testing). This is not actually exposed as a flag.
 	DefaultStorageVersions string
 	TargetRAMMB            int
+	TLSCAFile              string
 	TLSCertFile            string
 	TLSPrivateKeyFile      string
 	TokenAuthFile          string
@@ -205,15 +207,24 @@ func mergeGroupVersionIntoMap(gvList string, dest map[string]unversioned.GroupVe
 }
 
 // Returns a clientset which can be used to talk to this apiserver.
-func (s *ServerRunOptions) NewSelfClient() (clientset.Interface, error) {
+func (s *ServerRunOptions) NewSelfClient(token string) (clientset.Interface, error) {
 	clientConfig := &restclient.Config{
-		Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)),
 		// Increase QPS limits. The client is currently passed to all admission plugins,
 		// and those can be throttled in case of higher load on apiserver - see #22340 and #22422
 		// for more details. Once #22422 is fixed, we may want to remove it.
 		QPS:   50,
 		Burst: 100,
 	}
+	if s.SecurePort > 0 && len(s.TLSCAFile) > 0 {
+		clientConfig.Host = "https://" + net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.SecurePort))
+		clientConfig.CAFile = s.TLSCAFile
+		clientConfig.BearerToken = token
+	} else if s.InsecurePort > 0 {
+		clientConfig.Host = net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort))
+	} else {
+		return nil, errors.New("Unable to set url for apiserver local client")
+	}
+
 	return clientset.NewForConfig(clientConfig)
 }
 
@@ -438,6 +449,10 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"It defaults to a list of preferred versions of all registered groups, "+
 		"which is derived from the KUBE_API_VERSIONS environment variable.")
 
+	fs.StringVar(&s.TLSCAFile, "tls-ca-file", s.TLSCAFile, "If set, this "+
+		"certificate authority will used for secure access from Admission "+
+		"Controllers. This must be a valid PEM-encoded CA bundle.")
+
 	fs.StringVar(&s.TLSCertFile, "tls-cert-file", s.TLSCertFile, ""+
 		"File containing x509 Certificate for HTTPS. (CA cert, if any, concatenated "+
 		"after server cert). If HTTPS serving is enabled, and --tls-cert-file and "+

pkg/genericapiserver/validation/universal_validation.go

@@ -55,9 +55,12 @@ func verifySecureAndInsecurePort(options *options.ServerRunOptions) []error {
 		errors = append(errors, fmt.Errorf("--secure-port %v must be between 0 and 65535, inclusive. 0 for turning off secure port.", options.SecurePort))
 	}
 
-	// TODO: Allow 0 to turn off insecure port.
-	if options.InsecurePort < 1 || options.InsecurePort > 65535 {
-		errors = append(errors, fmt.Errorf("--insecure-port %v must be between 1 and 65535, inclusive.", options.InsecurePort))
+	if options.InsecurePort < 0 || options.InsecurePort > 65535 {
+		errors = append(errors, fmt.Errorf("--insecure-port %v must be between 0 and 65535, inclusive. 0 for turning off insecure port.", options.InsecurePort))
+	}
+
+	if options.SecurePort == 0 && options.InsecurePort == 0 {
+		glog.Fatalf("--secure-port and --insecure-port cannot be turned off at the same time.")
 	}
 
 	if options.SecurePort == options.InsecurePort {

plugin/pkg/auth/authenticator/token/tokenfile/tokenfile.go

@@ -30,6 +30,13 @@ type TokenAuthenticator struct {
 	tokens map[string]*user.DefaultInfo
 }
 
+// New returns a TokenAuthenticator for a single token
+func New(tokens map[string]*user.DefaultInfo) *TokenAuthenticator {
+	return &TokenAuthenticator{
+		tokens: tokens,
+	}
+}
+
 // NewCSV returns a TokenAuthenticator, populated from a CSV file.
 // The CSV file must contain records in the format "token,username,useruid"
 func NewCSV(path string) (*TokenAuthenticator, error) {

test/integration/examples/apiserver_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package apiserver
 
 import (
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -26,13 +27,12 @@ import (
 
 	"k8s.io/kubernetes/cmd/libs/go2idl/client-gen/test_apis/testgroup.k8s.io/v1"
 
+	"github.com/golang/glog"
 	"github.com/stretchr/testify/assert"
 	"k8s.io/kubernetes/examples/apiserver"
 	"k8s.io/kubernetes/pkg/api/unversioned"
 )
 
-var serverIP = fmt.Sprintf("http://localhost:%d", apiserver.InsecurePort)
-
 var groupVersion = v1.SchemeGroupVersion
 
 var groupVersionForDiscovery = unversioned.GroupVersionForDiscovery{
@@ -40,24 +40,49 @@ var groupVersionForDiscovery = unversioned.GroupVersionForDiscovery{
 	Version:      groupVersion.Version,
 }
 
-func TestRun(t *testing.T) {
+func TestRunServer(t *testing.T) {
+	serverIP := fmt.Sprintf("http://localhost:%d", apiserver.InsecurePort)
 	go func() {
 		if err := apiserver.Run(apiserver.NewServerRunOptions()); err != nil {
 			t.Fatalf("Error in bringing up the server: %v", err)
 		}
 	}()
-	if err := waitForApiserverUp(); err != nil {
+	if err := waitForApiserverUp(serverIP); err != nil {
 		t.Fatalf("%v", err)
 	}
-	testSwaggerSpec(t)
-	testAPIGroupList(t)
-	testAPIGroup(t)
-	testAPIResourceList(t)
+	testSwaggerSpec(t, serverIP)
+	testAPIGroupList(t, serverIP)
+	testAPIGroup(t, serverIP)
+	testAPIResourceList(t, serverIP)
 }
 
-func waitForApiserverUp() error {
+func TestRunSecureServer(t *testing.T) {
+	serverIP := fmt.Sprintf("https://localhost:%d", apiserver.SecurePort)
+	go func() {
+		options := apiserver.NewServerRunOptions()
+		options.InsecurePort = 0
+		options.SecurePort = apiserver.SecurePort
+		if err := apiserver.Run(options); err != nil {
+			t.Fatalf("Error in bringing up the server: %v", err)
+		}
+	}()
+	if err := waitForApiserverUp(serverIP); err != nil {
+		t.Fatalf("%v", err)
+	}
+	testSwaggerSpec(t, serverIP)
+	testAPIGroupList(t, serverIP)
+	testAPIGroup(t, serverIP)
+	testAPIResourceList(t, serverIP)
+}
+
+func waitForApiserverUp(serverIP string) error {
 	for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(5 * time.Second) {
-		_, err := http.Get(serverIP)
+		glog.Errorf("Waiting for : %#v", serverIP)
+		tr := &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		}
+		client := &http.Client{Transport: tr}
+		_, err := client.Get(serverIP)
 		if err == nil {
 			return nil
 		}
@@ -66,11 +91,17 @@ func waitForApiserverUp() error {
 }
 
 func readResponse(serverURL string) ([]byte, error) {
-	response, err := http.Get(serverURL)
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{Transport: tr}
+	response, err := client.Get(serverURL)
 	if err != nil {
+		glog.Errorf("http get err code : %#v", err)
 		return nil, fmt.Errorf("Error in fetching %s: %v", serverURL, err)
 	}
 	defer response.Body.Close()
+	glog.Errorf("http get response code : %#v", response.StatusCode)
 	if response.StatusCode != http.StatusOK {
 		return nil, fmt.Errorf("unexpected status: %d for URL: %s, expected status: %d", response.StatusCode, serverURL, http.StatusOK)
 	}
@@ -81,7 +112,7 @@ func readResponse(serverURL string) ([]byte, error) {
 	return contents, nil
 }
 
-func testSwaggerSpec(t *testing.T) {
+func testSwaggerSpec(t *testing.T, serverIP string) {
 	serverURL := serverIP + "/swaggerapi"
 	_, err := readResponse(serverURL)
 	if err != nil {
@@ -89,7 +120,7 @@ func testSwaggerSpec(t *testing.T) {
 	}
 }
 
-func testAPIGroupList(t *testing.T) {
+func testAPIGroupList(t *testing.T, serverIP string) {
 	serverURL := serverIP + "/apis"
 	contents, err := readResponse(serverURL)
 	if err != nil {
@@ -107,7 +138,7 @@ func testAPIGroupList(t *testing.T) {
 	assert.Equal(t, apiGroupList.Groups[0].PreferredVersion, groupVersionForDiscovery)
 }
 
-func testAPIGroup(t *testing.T) {
+func testAPIGroup(t *testing.T, serverIP string) {
 	serverURL := serverIP + "/apis/testgroup.k8s.io"
 	contents, err := readResponse(serverURL)
 	if err != nil {
@@ -126,7 +157,7 @@ func testAPIGroup(t *testing.T) {
 	assert.Equal(t, apiGroup.Versions[0], apiGroup.PreferredVersion)
 }
 
-func testAPIResourceList(t *testing.T) {
+func testAPIResourceList(t *testing.T, serverIP string) {
 	serverURL := serverIP + "/apis/testgroup.k8s.io/v1"
 	contents, err := readResponse(serverURL)
 	if err != nil {