Make integration test the manifest url feature. Make kubelet's docker pull command testable.

cmd/integration/integration.go

@@ -22,6 +22,7 @@ import (
 	"encoding/json"
 	"io/ioutil"
 	"log"
+	"net/http"
 	"net/http/httptest"
 	"time"
 
@@ -34,6 +35,7 @@ import (
 )
 
 func main() {
+	manifestUrl := ServeCachedManifestFile()
 	// Setup
 	servers := []string{"http://localhost:4001"}
 	log.Printf("Creating etcd client pointing to %v", servers)
@@ -48,21 +50,24 @@ func main() {
 	controllerManager.Run(10 * time.Second)
 
 	// Kublet
-	fakeDocker := &kubelet.FakeDockerClient{}
+	fakeDocker1 := &kubelet.FakeDockerClient{}
 	myKubelet := kubelet.Kubelet{
 		Hostname:           machineList[0],
-		DockerClient:       fakeDocker,
+		DockerClient:       fakeDocker1,
+		DockerPuller:       &kubelet.FakeDockerPuller{},
 		FileCheckFrequency: 5 * time.Second,
 		SyncFrequency:      5 * time.Second,
 		HTTPCheckFrequency: 5 * time.Second,
 	}
-	go myKubelet.RunKubelet("", "https://raw.githubusercontent.com/GoogleCloudPlatform/container-vm-guestbook-redis-python/master/manifest.yaml", servers[0], "localhost", 0)
+	go myKubelet.RunKubelet("", manifestUrl, servers[0], "localhost", 0)
 
 	// Create a second kublet so that the guestbook example's two redis slaves both
 	// have a place they can schedule.
+	fakeDocker2 := &kubelet.FakeDockerClient{}
 	otherKubelet := kubelet.Kubelet{
 		Hostname:           machineList[1],
-		DockerClient:       &kubelet.FakeDockerClient{},
+		DockerClient:       fakeDocker2,
+		DockerPuller:       &kubelet.FakeDockerPuller{},
 		FileCheckFrequency: 5 * time.Second,
 		SyncFrequency:      5 * time.Second,
 		HTTPCheckFrequency: 5 * time.Second,
@@ -100,15 +105,57 @@ func main() {
 	// Using a set to list unique creation attempts. Our fake is
 	// really stupid, so kubelet tries to create these multiple times.
 	createdPods := map[string]struct{}{}
-	for _, p := range fakeDocker.Created {
+	for _, p := range fakeDocker1.Created {
 		// The last 8 characters are random, so slice them off.
 		if n := len(p); n > 8 {
 			createdPods[p[:n-8]] = struct{}{}
 		}
 	}
-	// We expect 3: 1 net container + 1 pod from the replication controller + 1 pod from the URL.
+	for _, p := range fakeDocker2.Created {
-	if len(createdPods) != 3 {
+		// The last 8 characters are random, so slice them off.
+		if n := len(p); n > 8 {
+			createdPods[p[:n-8]] = struct{}{}
+		}
+	}
+	// We expect 5: 2 net containers + 2 pods from the replication controller +
+	//              1 net container + 2 pods from the URL.
+	if len(createdPods) != 7 {
 		log.Fatalf("Unexpected list of created pods: %#v\n", createdPods)
 	}
 	log.Printf("OK")
 }
+
+// Serve a file for kubelet to read.
+func ServeCachedManifestFile() (servingAddress string) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/manifest" {
+			w.Write([]byte(testManifestFile))
+			return
+		}
+		log.Fatalf("Got request: %#v\n", r)
+		http.NotFound(w, r)
+	}))
+	return server.URL + "/manifest"
+}
+
+const (
+	// This is copied from, and should be kept in sync with:
+	// https://raw.githubusercontent.com/GoogleCloudPlatform/container-vm-guestbook-redis-python/master/manifest.yaml
+	testManifestFile = `version: v1beta1
+containers:
+  - name: redis
+    image: dockerfile/redis
+    volumeMounts:
+      - name: redis-data
+        path: /data
+
+  - name: guestbook
+    image: google/guestbook-python-redis
+    ports:
+      - name: www
+        hostPort: 80
+        containerPort: 80
+
+volumes:
+  - name: redis-data`
+)

hack/integration-test.sh

@@ -22,7 +22,7 @@ fi
 # Stop right away if the build fails
 set -e
 
-$(dirname $0)/build-go.sh
+$(dirname $0)/build-go.sh integration
 
 ETCD_DIR=$(mktemp -d -t kube-integration.XXXXXX)
 trap "rm -rf ${ETCD_DIR}" EXIT

pkg/kubelet/fake_docker_client.go

@@ -67,3 +67,23 @@ func (f *FakeDockerClient) StopContainer(id string, timeout uint) error {
 	f.stopped = append(f.stopped, id)
 	return nil
 }
+
+type FakeDockerPuller struct {
+	ImagesPulled []string
+
+	// Every pull will return the first error here, and then reslice
+	// to remove it. Will give nil errors if this slice is empty.
+	ErrorsToInject []error
+}
+
+// Records the image pull attempt, and optionally injects an error.
+func (f *FakeDockerPuller) Pull(image string) error {
+	f.ImagesPulled = append(f.ImagesPulled, image)
+
+	if n := len(f.ErrorsToInject); n > 0 {
+		err := f.ErrorsToInject[0]
+		f.ErrorsToInject = f.ErrorsToInject[:n-1]
+		return err
+	}
+	return nil
+}

pkg/kubelet/kubelet.go

@@ -60,6 +60,11 @@ type DockerInterface interface {
 	StopContainer(id string, timeout uint) error
 }
 
+//Interface for testability
+type DockerPuller interface {
+	Pull(image string) error
+}
+
 type CadvisorInterface interface {
 	ContainerInfo(name string) (*info.ContainerInfo, error)
 	MachineInfo() (*info.MachineInfo, error)
@@ -70,6 +75,7 @@ type Kubelet struct {
 	Hostname           string
 	EtcdClient         util.EtcdClient
 	DockerClient       DockerInterface
+	DockerPuller       DockerPuller
 	CadvisorClient     CadvisorInterface
 	FileCheckFrequency time.Duration
 	SyncFrequency      time.Duration
@@ -92,6 +98,9 @@ const (
 // Starts background goroutines. If config_path, manifest_url, or address are empty,
 // they are not watched. Never returns.
 func (kl *Kubelet) RunKubelet(config_path, manifest_url, etcd_servers, address string, port uint) {
+	if kl.DockerPuller == nil {
+		kl.DockerPuller = MakeDockerPuller()
+	}
 	updateChannel := make(chan manifestUpdate)
 	if config_path != "" {
 		go util.Forever(func() { kl.WatchFiles(config_path, updateChannel) }, kl.FileCheckFrequency)
@@ -216,9 +225,13 @@ func (kl *Kubelet) ListContainers() ([]string, error) {
 	return result, err
 }
 
-func (kl *Kubelet) pullImage(image string) error {
+type dockerPuller struct{}
-	kl.pullLock.Lock()
+
-	defer kl.pullLock.Unlock()
+func MakeDockerPuller() DockerPuller {
+	return dockerPuller{}
+}
+
+func (dockerPuller) Pull(image string) error {
 	cmd := exec.Command("docker", "pull", image)
 	err := cmd.Start()
 	if err != nil {
@@ -676,7 +689,7 @@ func (kl *Kubelet) createNetworkContainer(manifest *api.ContainerManifest) (stri
 		Command: []string{"sh", "-c", "rm -f nap && mkfifo nap && exec cat nap"},
 		Ports:   ports,
 	}
-	kl.pullImage("busybox")
+	kl.DockerPuller.Pull("busybox")
 	return kl.RunContainer(manifest, container, "")
 }
 
@@ -710,7 +723,7 @@ func (kl *Kubelet) SyncManifests(config []api.ContainerManifest) error {
 			}
 			if !exists {
 				log.Printf("%#v doesn't exist, creating", element)
-				err = kl.pullImage(element.Image)
+				kl.DockerPuller.Pull(element.Image)
 				if err != nil {
 					log.Printf("Error pulling container: %#v", err)
 					continue

pkg/kubelet/kubelet_test.go

@@ -75,9 +75,15 @@ func verifyError(t *testing.T, e error) {
 	}
 }
 
+func makeTestKubelet() *Kubelet {
+	return &Kubelet{
+		DockerPuller: &FakeDockerPuller{},
+	}
+}
+
 func TestExtractJSON(t *testing.T) {
 	obj := TestObject{}
-	kubelet := Kubelet{}
+	kubelet := makeTestKubelet()
 	data := `{ "name": "foo", "data": { "value": "bar", "number": 10 } }`
 	kubelet.ExtractYAMLData([]byte(data), &obj)
 
@@ -133,6 +139,7 @@ func TestContainerExists(t *testing.T) {
 	}
 	kubelet := Kubelet{
 		DockerClient: &fakeDocker,
+		DockerPuller: &FakeDockerPuller{},
 	}
 	manifest := api.ContainerManifest{
 		Id: "qux",
@@ -176,6 +183,7 @@ func TestGetContainerID(t *testing.T) {
 	}
 	kubelet := Kubelet{
 		DockerClient: &fakeDocker,
+		DockerPuller: &FakeDockerPuller{},
 	}
 	fakeDocker.containerList = []docker.APIContainers{
 		{
@@ -214,6 +222,7 @@ func TestGetContainerByName(t *testing.T) {
 	}
 	kubelet := Kubelet{
 		DockerClient: &fakeDocker,
+		DockerPuller: &FakeDockerPuller{},
 	}
 	fakeDocker.containerList = []docker.APIContainers{
 		{
@@ -242,6 +251,7 @@ func TestListContainers(t *testing.T) {
 	}
 	kubelet := Kubelet{
 		DockerClient: &fakeDocker,
+		DockerPuller: &FakeDockerPuller{},
 	}
 	fakeDocker.containerList = []docker.APIContainers{
 		{
@@ -272,6 +282,7 @@ func TestKillContainerWithError(t *testing.T) {
 	}
 	kubelet := Kubelet{
 		DockerClient: &fakeDocker,
+		DockerPuller: &FakeDockerPuller{},
 	}
 	err := kubelet.KillContainer("foo")
 	verifyError(t, err)
@@ -284,6 +295,7 @@ func TestKillContainer(t *testing.T) {
 	}
 	kubelet := Kubelet{
 		DockerClient: &fakeDocker,
+		DockerPuller: &FakeDockerPuller{},
 	}
 	fakeDocker.containerList = []docker.APIContainers{
 		{
@@ -303,7 +315,7 @@ func TestKillContainer(t *testing.T) {
 }
 
 func TestResponseToContainersNil(t *testing.T) {
-	kubelet := Kubelet{}
+	kubelet := makeTestKubelet()
 	list, err := kubelet.ResponseToManifests(&etcd.Response{Node: nil})
 	if len(list) != 0 {
 		t.Errorf("Unexpected non-zero list: %#v", list)
@@ -314,7 +326,7 @@ func TestResponseToContainersNil(t *testing.T) {
 }
 
 func TestResponseToManifests(t *testing.T) {
-	kubelet := Kubelet{}
+	kubelet := makeTestKubelet()
 	list, err := kubelet.ResponseToManifests(&etcd.Response{
 		Node: &etcd.Node{
 			Value: util.MakeJSONString([]api.ContainerManifest{
@@ -468,6 +480,7 @@ func TestSyncManifestsDoesNothing(t *testing.T) {
 	}
 	kubelet := Kubelet{
 		DockerClient: &fakeDocker,
+		DockerPuller: &FakeDockerPuller{},
 	}
 	err := kubelet.SyncManifests([]api.ContainerManifest{
 		{
@@ -510,6 +523,7 @@ func TestSyncManifestsDeletes(t *testing.T) {
 	}
 	kubelet := Kubelet{
 		DockerClient: &fakeDocker,
+		DockerPuller: &FakeDockerPuller{},
 	}
 	err := kubelet.SyncManifests([]api.ContainerManifest{})
 	expectNoError(t, err)
@@ -962,6 +976,7 @@ func TestGetContainerStats(t *testing.T) {
 
 	kubelet := Kubelet{
 		DockerClient:   &fakeDocker,
+		DockerPuller:   &FakeDockerPuller{},
 		CadvisorClient: mockCadvisor,
 	}
 	fakeDocker.containerList = []docker.APIContainers{
@@ -990,6 +1005,7 @@ func TestGetContainerStatsWithoutCadvisor(t *testing.T) {
 
 	kubelet := Kubelet{
 		DockerClient: &fakeDocker,
+		DockerPuller: &FakeDockerPuller{},
 	}
 	fakeDocker.containerList = []docker.APIContainers{
 		{
@@ -1027,6 +1043,7 @@ func TestGetContainerStatsWhenCadvisorFailed(t *testing.T) {
 
 	kubelet := Kubelet{
 		DockerClient:   &fakeDocker,
+		DockerPuller:   &FakeDockerPuller{},
 		CadvisorClient: mockCadvisor,
 	}
 	fakeDocker.containerList = []docker.APIContainers{
@@ -1059,6 +1076,7 @@ func TestGetContainerStatsOnNonExistContainer(t *testing.T) {
 
 	kubelet := Kubelet{
 		DockerClient:   &fakeDocker,
+		DockerPuller:   &FakeDockerPuller{},
 		CadvisorClient: mockCadvisor,
 	}
 	fakeDocker.containerList = []docker.APIContainers{}