Merge pull request #32203 from dgoodwin/kubediscovery

Automatic merge from submit-queue

Alpha JWS Discovery API for locating an apiserver securely

This PR contains an early alpha prototype of the JWS discovery API outlined in proposal #30707.

CA certificate, API endpoints, and the token to be used to authenticate to this discovery API are currently passed in as secrets. If the caller provides a valid token ID, a JWS signed blob of ClusterInfo containing the API endpoints and the CA cert to use will be returned to the caller. This is used by the alpha kubeadm to allow seamless, very quick cluster setup with simple commands well suited for copy paste.

Current TODO list:

- [x] Allow the use of arbitrary strings as token ID/token, we're currently treating them as raw keys.
- [x] Integrate the building of the pod container, move to cluster/images/kube-discovery.
  - [x] Build for: amd64, arm, arm64 and ppc64le. (just replace GOARCH=)
  - [x] Rename to gcr.io/google_containers/kube-discovery-ARCH:1.0
  - [x] Cleanup rogue files in discovery sub-dir.
  - [x] Move pkg/discovery/ to cmd/discovery/app.

There is additional pending work to return a kubeconfig rather than ClusterInfo, however I believe this is slated for post-alpha.

cluster/images/kube-discovery/Dockerfile

@@ -0,0 +1,18 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM BASEIMAGE
+
+COPY kube-discovery /usr/local/bin
+ENTRYPOINT "/usr/local/bin/kube-discovery"

cluster/images/kube-discovery/Makefile

@@ -0,0 +1,53 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build the kube-discovery image.
+#
+# Requires a pre-built kube-discovery binary:
+#   build/run.sh /bin/bash -c "KUBE_BUILD_PLATFORMS=linux/ARCH make WHAT=cmd/kube-discovery"
+#
+# Usage:
+#   [ARCH=amd64] [REGISTRY="gcr.io/google_containers"] make (build|push) VERSION={some_released_version_of_kubernetes}
+
+REGISTRY?=gcr.io/google_containers
+ARCH?=amd64
+TEMP_DIR:=$(shell mktemp -d)
+VERSION?=1.0
+
+ifeq ($(ARCH),amd64)
+	BASEIMAGE?=debian:jessie
+endif
+ifeq ($(ARCH),arm)
+	BASEIMAGE?=armel/debian:jessie
+endif
+ifeq ($(ARCH),arm64)
+	BASEIMAGE?=aarch64/debian:jessie
+endif
+ifeq ($(ARCH),ppc64le)
+	BASEIMAGE?=ppc64le/debian:jessie
+endif
+
+all: build
+
+build:
+	cp -r ./* ${TEMP_DIR}
+	cp ../../../_output/dockerized/bin/linux/${ARCH}/kube-discovery ${TEMP_DIR}
+	cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile
+	docker build -t ${REGISTRY}/kube-discovery-${ARCH}:${VERSION} ${TEMP_DIR}
+	rm -rf "${TEMP_DIR}"
+
+push: build
+	gcloud docker push ${REGISTRY}/kube-discovery-${ARCH}:${VERSION}
+
+.PHONY: all

cluster/images/kube-discovery/README.md

@@ -0,0 +1,45 @@
+### kube-discovery
+
+An initial implementation of a Kubernetes discovery service using JSON Web Signatures.
+
+This prototype is configured by kubeadm and run within Kubernetes itself.
+
+## Requirements
+
+This pod expects the cluster CA, endpoints list, and token map to exist in /tmp/secret. This allows us to pass them in as kubernetes secrets when deployed as a pod.
+
+```
+$ cd /tmp/secret
+$ ls
+ca.pem  endpoint-list.json  token-map.json
+$ cat endpoint-list.json
+["http://192.168.1.5:8080", "http://192.168.1.6:8080"]
+$ cat token-map.json
+{
+    "TOKENID": "ABCDEF1234123456"
+}
+```
+
+## Build And Run From Source
+
+```
+$ build/run.sh /bin/bash -c "KUBE_BUILD_PLATFORMS=linux/amd64 make WHAT=cmd/kube-discovery"
+$ _output/dockerized/bin/linux/amd64/kube-discovery
+2016/08/23 19:17:28 Listening for requests on port 9898.
+
+```
+
+## Running in Docker
+
+This image is published at: gcr.io/google_containers/kube-discovery
+
+`docker run -d -p 9898:9898 -v /tmp/secret/ca.pem:/tmp/secret/ca.pem -v /tmp/secret/endpoint-list.json:/tmp/secret/endpoint-list.json -v /tmp/secret/token-map.json:/tmp/secret/token-map.json --name kubediscovery gcr.io/google_containers/kube-discovery`
+
+## Testing the API
+
+`curl "http://localhost:9898/cluster-info/v1/?token-id=TOKENID"`
+
+You should see JSON containing a signed payload. For code to verify and decode that payload see handler_test.go.
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/images/kube-discovery/README.md?pixel)]()

cmd/kube-discovery/app/handlers.go

@@ -0,0 +1,203 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package discovery
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+
+	"github.com/square/go-jose"
+)
+
+const secretPath = "/tmp/secret"
+
+// CAPath is the expected location of our cluster's CA to be distributed to
+// clients looking to connect. Because we expect to use kubernetes secrets
+// for the time being, this file is expected to be a base64 encoded version
+// of the normal cert PEM.
+const CAPath = secretPath + "/ca.pem"
+
+// caLoader is an interface for abstracting how we load the CA certificates
+// for the cluster.
+type caLoader interface {
+	LoadPEM() (string, error)
+}
+
+// fsCALoader is a caLoader for loading the PEM encoded CA from
+// /tmp/secret/ca.pem.
+type fsCALoader struct {
+	certData string
+}
+
+func (cl *fsCALoader) LoadPEM() (string, error) {
+	if cl.certData == "" {
+		data, err := ioutil.ReadFile(CAPath)
+		if err != nil {
+			return "", err
+		}
+
+		cl.certData = string(data)
+	}
+
+	return cl.certData, nil
+}
+
+const TokenMapPath = secretPath + "/token-map.json"
+const EndpointListPath = secretPath + "/endpoint-list.json"
+
+// tokenLoader is an interface for abstracting how we validate
+// token IDs and lookup their corresponding token.
+type tokenLoader interface {
+	// Lookup returns the token for a given token ID, or an error if the token ID
+	// does not exist. Both token and it's ID are expected be strings.
+	LoadAndLookup(tokenID string) (string, error)
+}
+
+type jsonFileTokenLoader struct {
+	tokenMap map[string]string
+}
+
+func (tl *jsonFileTokenLoader) LoadAndLookup(tokenID string) (string, error) {
+	if len(tl.tokenMap) == 0 {
+		data, err := ioutil.ReadFile(TokenMapPath)
+		if err != nil {
+			return "", err
+		}
+		if err := json.Unmarshal(data, &tl.tokenMap); err != nil {
+			return "", err
+		}
+	}
+	if val, ok := tl.tokenMap[tokenID]; ok {
+		return val, nil
+	}
+	return "", errors.New(fmt.Sprintf("invalid token: %s", tokenID))
+}
+
+type endpointsLoader interface {
+	LoadList() ([]string, error)
+}
+
+type jsonFileEndpointsLoader struct {
+	endpoints []string
+}
+
+func (el *jsonFileEndpointsLoader) LoadList() ([]string, error) {
+	if len(el.endpoints) == 0 {
+		data, err := ioutil.ReadFile(EndpointListPath)
+		if err != nil {
+			return nil, err
+		}
+		if err := json.Unmarshal(data, &el.endpoints); err != nil {
+			return nil, err
+		}
+	}
+	return el.endpoints, nil
+}
+
+// ClusterInfoHandler implements the http.ServeHTTP method and allows us to
+// mock out portions of the request handler in tests.
+type ClusterInfoHandler struct {
+	tokenLoader     tokenLoader
+	caLoader        caLoader
+	endpointsLoader endpointsLoader
+}
+
+func NewClusterInfoHandler() *ClusterInfoHandler {
+	return &ClusterInfoHandler{
+		tokenLoader:     &jsonFileTokenLoader{},
+		caLoader:        &fsCALoader{},
+		endpointsLoader: &jsonFileEndpointsLoader{},
+	}
+}
+
+func (cih *ClusterInfoHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	tokenID := req.FormValue("token-id")
+	log.Printf("Got token ID: %s", tokenID)
+	token, err := cih.tokenLoader.LoadAndLookup(tokenID)
+	if err != nil {
+		log.Print(err)
+		http.Error(resp, "Forbidden", http.StatusForbidden)
+		return
+	}
+	log.Printf("Loaded token: %s", token)
+
+	// TODO probably should not leak server-side errors to the client
+	caPEM, err := cih.caLoader.LoadPEM()
+	log.Printf("Loaded CA: %s", caPEM)
+	if err != nil {
+		err = fmt.Errorf("Error loading root CA certificate data: %s", err)
+		log.Println(err)
+		http.Error(resp, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	endpoints, err := cih.endpointsLoader.LoadList()
+	if err != nil {
+		err = fmt.Errorf("Error loading list of API endpoints: %s", err)
+		log.Println(err)
+		http.Error(resp, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	clusterInfo := ClusterInfo{
+		CertificateAuthorities: []string{caPEM},
+		Endpoints:              endpoints,
+	}
+
+	// Instantiate an signer using HMAC-SHA256.
+	hmacKey := []byte(token)
+
+	log.Printf("Key is %d bytes long", len(hmacKey))
+	signer, err := jose.NewSigner(jose.HS256, hmacKey)
+	if err != nil {
+		err = fmt.Errorf("Error creating JWS signer: %s", err)
+		log.Println(err)
+		http.Error(resp, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	payload, err := json.Marshal(clusterInfo)
+	if err != nil {
+		err = fmt.Errorf("Error serializing clusterInfo to JSON: %s", err)
+		log.Println(err)
+		http.Error(resp, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	// Sign a sample payload. Calling the signer returns a protected JWS object,
+	// which can then be serialized for output afterwards. An error would
+	// indicate a problem in an underlying cryptographic primitive.
+	jws, err := signer.Sign(payload)
+	if err != nil {
+		err = fmt.Errorf("Error signing clusterInfo with JWS: %s", err)
+		log.Println(err)
+		http.Error(resp, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	// Serialize the encrypted object using the full serialization format.
+	// Alternatively you can also use the compact format here by calling
+	// object.CompactSerialize() instead.
+	serialized := jws.FullSerialize()
+
+	resp.Write([]byte(serialized))
+
+}

cmd/kube-discovery/app/handlers_test.go

@@ -0,0 +1,208 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package discovery
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/square/go-jose"
+)
+
+type mockTokenLoader struct {
+	tokenID string
+	token   string
+}
+
+func (tl *mockTokenLoader) LoadAndLookup(tokenID string) (string, error) {
+	if tokenID == tl.tokenID {
+		return tl.token, nil
+	}
+	return "", errors.New(fmt.Sprintf("invalid token: %s", tokenID))
+}
+
+const mockEndpoint1 = "https://192.168.1.5:8080"
+const mockEndpoint2 = "https://192.168.1.6:8080"
+
+type mockEndpointsLoader struct {
+}
+
+func (el *mockEndpointsLoader) LoadList() ([]string, error) {
+	return []string{mockEndpoint1, mockEndpoint2}, nil
+}
+
+const mockCA = "---BEGIN------END---DUMMYDATA"
+
+type mockCALoader struct {
+}
+
+func (cl *mockCALoader) LoadPEM() (string, error) {
+	return mockCA, nil
+}
+
+const mockTokenID = "AAAAAA"
+const mockToken = "9537434E638E4378"
+
+const mockTokenIDCustom = "SHAREDSECRET"
+const mockTokenCustom = "VERYSECRETTOKEN"
+
+func TestClusterInfoIndex(t *testing.T) {
+	longToken := strings.Repeat("a", 1000)
+	tests := map[string]struct {
+		tokenID          string // token ID the mock loader will use
+		token            string // token the mock loader will use
+		reqTokenID       string // token ID the will request with
+		reqToken         string // token the caller will validate response with
+		expStatus        int
+		expVerifyFailure bool
+	}{
+		"no token": {
+			tokenID:    mockTokenID,
+			token:      mockToken,
+			reqTokenID: "",
+			reqToken:   "",
+			expStatus:  http.StatusForbidden,
+		},
+		"valid token ID": {
+			tokenID:    mockTokenID,
+			token:      mockToken,
+			reqTokenID: mockTokenID,
+			reqToken:   mockToken,
+			expStatus:  http.StatusOK,
+		},
+		"valid arbitrary string token": {
+			tokenID:    mockTokenIDCustom,
+			token:      mockTokenCustom,
+			reqTokenID: mockTokenIDCustom,
+			reqToken:   mockTokenCustom,
+			expStatus:  http.StatusOK,
+		},
+		"valid arbitrary long string token": {
+			tokenID:    "LONGTOKENTEST",
+			token:      longToken,
+			reqTokenID: "LONGTOKENTEST",
+			reqToken:   longToken,
+			expStatus:  http.StatusOK,
+		},
+		"invalid token ID": {
+			tokenID:    mockTokenID,
+			token:      mockToken,
+			reqTokenID: "BADTOKENID",
+			reqToken:   mockToken,
+			expStatus:  http.StatusForbidden,
+		},
+		"invalid token": {
+			tokenID:          mockTokenID,
+			token:            mockToken,
+			reqTokenID:       mockTokenID,
+			reqToken:         "badtoken",
+			expStatus:        http.StatusOK,
+			expVerifyFailure: true,
+		},
+	}
+
+	for name, test := range tests {
+		t.Logf("Running test: %s", name)
+		tokenLoader := &mockTokenLoader{test.tokenID, test.token}
+		// Create a request to pass to our handler. We don't have any query parameters for now, so we'll
+		// pass 'nil' as the third parameter.
+		url := "/cluster-info/v1/"
+		if test.tokenID != "" {
+			url = fmt.Sprintf("%s?token-id=%s", url, test.reqTokenID)
+		}
+		req, err := http.NewRequest("GET", url, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		rr := httptest.NewRecorder()
+		handler := &ClusterInfoHandler{
+			tokenLoader:     tokenLoader,
+			caLoader:        &mockCALoader{},
+			endpointsLoader: &mockEndpointsLoader{},
+		}
+
+		handler.ServeHTTP(rr, req)
+
+		if status := rr.Code; status != test.expStatus {
+			t.Errorf("handler returned wrong status code: got %v want %v",
+				status, test.expStatus)
+			continue
+		}
+
+		// If we were expecting valid status validate the body:
+		if test.expStatus == http.StatusOK {
+			var ci ClusterInfo
+
+			body := string(rr.Body.Bytes())
+
+			// Parse the JSON web signature:
+			jws, err := jose.ParseSigned(body)
+			if err != nil {
+				t.Errorf("Error parsing JWS from request body: %s", err)
+				continue
+			}
+
+			// Now we can verify the signature on the payload. An error here would
+			// indicate the the message failed to verify, e.g. because the signature was
+			// broken or the message was tampered with.
+			var clusterInfoBytes []byte
+			hmacTestKey := []byte(test.reqToken)
+			clusterInfoBytes, err = jws.Verify(hmacTestKey)
+
+			if test.expVerifyFailure {
+				if err == nil {
+					t.Errorf("Signature verification did not fail as expected.")
+				}
+				// We are done the test here either way.
+				continue
+			}
+
+			if err != nil {
+				t.Errorf("Error verifing signature: %s", err)
+				continue
+			}
+
+			err = json.Unmarshal(clusterInfoBytes, &ci)
+			if err != nil {
+				t.Errorf("Unable to unmarshall payload to JSON: error=%s body=%s", err, rr.Body.String())
+				continue
+			}
+			if len(ci.Endpoints) != 2 {
+				t.Errorf("Expected 2 endpoints, got: %d", len(ci.Endpoints))
+			}
+			if mockEndpoint1 != ci.Endpoints[0] {
+				t.Errorf("Unexpected endpoint: %s", ci.Endpoints[0])
+			}
+			if mockEndpoint2 != ci.Endpoints[1] {
+				t.Errorf("Unexpected endpoint: %s", ci.Endpoints[1])
+			}
+
+			if len(ci.CertificateAuthorities) != 1 {
+				t.Errorf("Expected 1 root certificate, got: %d", len(ci.CertificateAuthorities))
+			}
+			if ci.CertificateAuthorities[0] != mockCA {
+				t.Errorf("Expected CA: %s, got: %s", mockCA, ci.CertificateAuthorities[0])
+			}
+		}
+	}
+}

cmd/kube-discovery/app/model.go

@@ -14,11 +14,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package main
+package discovery
 
-import (
-	_ "github.com/square/go-jose"
-)
-
-func main() {
+type ClusterInfo struct {
+	// TODO Kind, apiVersion
+	// TODO clusterId, fetchedTime, expiredTime
+	CertificateAuthorities []string `json:"certificateAuthorities,omitempty"`
+	Endpoints              []string `json:"endpoints,omitempty"`
 }

cmd/kube-discovery/app/routes.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package discovery
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+)
+
+type Route struct {
+	Name    string
+	Method  string
+	Pattern string
+	Handler http.Handler
+}
+
+type Routes []Route
+
+var routes = Routes{
+	Route{
+		"ClusterInfoIndex",
+		"GET",
+		"/cluster-info/v1/",
+		NewClusterInfoHandler(),
+	},
+}
+
+func NewRouter() *mux.Router {
+
+	router := mux.NewRouter().StrictSlash(true)
+	for _, route := range routes {
+		router.
+			Methods(route.Method).
+			Path(route.Pattern).
+			Name(route.Name).
+			Handler(route.Handler)
+	}
+
+	return router
+}

cmd/kube-discovery/kubediscovery.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"log"
+	"net/http"
+	"os"
+
+	kd "k8s.io/kubernetes/cmd/kube-discovery/app"
+)
+
+func main() {
+	// Make sure we can load critical files, and be nice to the user by
+	// printing descriptive error message when we fail.
+	for desc, path := range map[string]string{
+		"root CA certificate":   kd.CAPath,
+		"token map file":        kd.TokenMapPath,
+		"list of API endpoints": kd.EndpointListPath,
+	} {
+		if _, err := os.Stat(path); os.IsNotExist(err) {
+			log.Fatalf("%s does not exist: %s", desc, path)
+		}
+		// Test read permissions
+		file, err := os.Open(path)
+		if err != nil {
+			log.Fatalf("Unable to open %s (%q [%s])", desc, path, err)
+		}
+		file.Close()
+	}
+
+	router := kd.NewRouter()
+	log.Printf("Listening for requests on port 9898.")
+	log.Fatal(http.ListenAndServe(":9898", router))
+}