diff --git a/pkg/api/types.go b/pkg/api/types.go index 2df6fe12cd7..b6d504a0935 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -16,6 +16,10 @@ limitations under the License. package api +import ( + "github.com/fsouza/go-dockerclient" +) + // Common string formats // --------------------- // Many fields in this API have formatting requirements. The commonly used @@ -159,7 +163,13 @@ type PodState struct { Status PodStatus `json:"status,omitempty" yaml:"status,omitempty"` Host string `json:"host,omitempty" yaml:"host,omitempty"` HostIP string `json:"hostIP,omitempty" yaml:"hostIP,omitempty"` - Info interface{} `json:"info,omitempty" yaml:"info,omitempty"` + + // The key of this map is the *name* of the container within the manifest; it has one + // entry per container in the manifest. The value of this map is currently the output + // of `docker inspect`. This output format is *not* final and should not be relied + // upon. + // TODO: Make real decisions about what our info should look like. + Info map[string]docker.Container `json:"info,omitempty" yaml:"info,omitempty"` } type PodList struct { diff --git a/pkg/client/container_info.go b/pkg/client/container_info.go index ce408206f0a..c4ec0c031a2 100644 --- a/pkg/client/container_info.go +++ b/pkg/client/container_info.go @@ -20,15 +20,20 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net" "net/http" + "strconv" + + "github.com/fsouza/go-dockerclient" ) // ContainerInfo is an interface for things that can get information about a container. // Injectable for easy testing. type ContainerInfo interface { // GetContainerInfo returns information about container 'name' on 'host' - // Returns an untyped interface, and an error, if one occurs - GetContainerInfo(host, name string) (interface{}, error) + // Returns a json-formatted []byte (which can be unmarshalled into a + // map[string]interface{}) or an error if one occurs. + GetContainerInfo(host, name string) (*docker.Container, error) } // The default implementation, accesses the kubelet over HTTP @@ -37,8 +42,14 @@ type HTTPContainerInfo struct { Port uint } -func (c *HTTPContainerInfo) GetContainerInfo(host, name string) (interface{}, error) { - request, err := http.NewRequest("GET", fmt.Sprintf("http://%s:%d/containerInfo?container=%s", host, c.Port, name), nil) +func (c *HTTPContainerInfo) GetContainerInfo(host, name string) (*docker.Container, error) { + request, err := http.NewRequest( + "GET", + fmt.Sprintf( + "http://%s/containerInfo?container=%s", + net.JoinHostPort(host, strconv.FormatUint(uint64(c.Port), 10)), + name), + nil) if err != nil { return nil, err } @@ -51,17 +62,21 @@ func (c *HTTPContainerInfo) GetContainerInfo(host, name string) (interface{}, er if err != nil { return nil, err } - var data interface{} - err = json.Unmarshal(body, &data) - return data, err + // Check that this data can be unmarshalled + var container docker.Container + err = json.Unmarshal(body, &container) + if err != nil { + return nil, err + } + return &container, nil } // Useful for testing. type FakeContainerInfo struct { - data interface{} + data *docker.Container err error } -func (c *FakeContainerInfo) GetContainerInfo(host, name string) (interface{}, error) { +func (c *FakeContainerInfo) GetContainerInfo(host, name string) (*docker.Container, error) { return c.data, c.err } diff --git a/pkg/client/container_info_test.go b/pkg/client/container_info_test.go index 1b0904f9596..5d04c361ad6 100644 --- a/pkg/client/container_info_test.go +++ b/pkg/client/container_info_test.go @@ -26,6 +26,7 @@ import ( "testing" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + "github.com/fsouza/go-dockerclient" ) // TODO: This doesn't reduce typing enough to make it worth the less readable errors. Remove. @@ -36,10 +37,12 @@ func expectNoError(t *testing.T, err error) { } func TestHTTPContainerInfo(t *testing.T) { - body := `{"items":[]}` + expectObj := &docker.Container{ID: "myID"} + body, err := json.Marshal(expectObj) + expectNoError(t, err) fakeHandler := util.FakeHandler{ StatusCode: 200, - ResponseBody: body, + ResponseBody: string(body), } testServer := httptest.NewServer(&fakeHandler) @@ -53,10 +56,11 @@ func TestHTTPContainerInfo(t *testing.T) { Client: http.DefaultClient, Port: uint(port), } - data, err := containerInfo.GetContainerInfo(parts[0], "foo") + gotObj, err := containerInfo.GetContainerInfo(parts[0], "foo") expectNoError(t, err) - dataString, _ := json.Marshal(data) - if string(dataString) != body { - t.Errorf("Unexpected response. Expected: %s, received %s", body, string(dataString)) + + // reflect.DeepEqual(expectObj, gotObj) doesn't handle blank times well + if expectObj.ID != gotObj.ID { + t.Errorf("Unexpected response. Expected: %#v, received %#v", expectObj, gotObj) } } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 7bdb48cb4fe..79a4d48a6cf 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -784,17 +784,16 @@ func (kl *Kubelet) getContainerIdFromName(name string) (DockerId, bool, error) { } // Returns docker info for a container -func (kl *Kubelet) GetContainerInfo(name string) (string, error) { +func (kl *Kubelet) GetContainerInfo(name string) (*docker.Container, error) { dockerId, found, err := kl.getContainerIdFromName(name) if err != nil || !found { - return "{}", err + return nil, err } info, err := kl.DockerClient.InspectContainer(string(dockerId)) if err != nil { - return "{}", err + return nil, err } - data, err := json.Marshal(info) - return string(data), err + return info, nil } //Returns stats (from Cadvisor) for a container diff --git a/pkg/kubelet/kubelet_server.go b/pkg/kubelet/kubelet_server.go index 4c7f45ea45c..965cda1d0e1 100644 --- a/pkg/kubelet/kubelet_server.go +++ b/pkg/kubelet/kubelet_server.go @@ -24,6 +24,7 @@ import ( "net/url" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + "github.com/fsouza/go-dockerclient" "gopkg.in/v1/yaml" ) @@ -36,7 +37,7 @@ type KubeletServer struct { // For testablitiy. type kubeletInterface interface { GetContainerStats(name string) (*api.ContainerStats, error) - GetContainerInfo(name string) (string, error) + GetContainerInfo(name string) (*docker.Container, error) } func (s *KubeletServer) error(w http.ResponseWriter, err error) { @@ -113,7 +114,13 @@ func (s *KubeletServer) ServeHTTP(w http.ResponseWriter, req *http.Request) { fmt.Fprint(w, "Missing container selector arg.") return } - data, err := s.Kubelet.GetContainerInfo(container) + info, err := s.Kubelet.GetContainerInfo(container) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(w, "Internal Error: %v", err) + return + } + data, err := json.Marshal(info) if err != nil { w.WriteHeader(http.StatusInternalServerError) fmt.Fprintf(w, "Internal Error: %v", err) @@ -121,7 +128,7 @@ func (s *KubeletServer) ServeHTTP(w http.ResponseWriter, req *http.Request) { } w.WriteHeader(http.StatusOK) w.Header().Add("Content-type", "application/json") - fmt.Fprint(w, data) + w.Write(data) default: w.WriteHeader(http.StatusNotFound) fmt.Fprint(w, "Not found.") diff --git a/pkg/kubelet/kubelet_server_test.go b/pkg/kubelet/kubelet_server_test.go index 22891e2a42f..c63df291cf5 100644 --- a/pkg/kubelet/kubelet_server_test.go +++ b/pkg/kubelet/kubelet_server_test.go @@ -28,14 +28,15 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + "github.com/fsouza/go-dockerclient" ) type fakeKubelet struct { - infoFunc func(name string) (string, error) + infoFunc func(name string) (*docker.Container, error) statsFunc func(name string) (*api.ContainerStats, error) } -func (fk *fakeKubelet) GetContainerInfo(name string) (string, error) { +func (fk *fakeKubelet) GetContainerInfo(name string) (*docker.Container, error) { return fk.infoFunc(name) } @@ -116,12 +117,12 @@ func TestContainers(t *testing.T) { func TestContainerInfo(t *testing.T) { fw := makeServerTest() - expected := "good container info string" - fw.fakeKubelet.infoFunc = func(name string) (string, error) { + expected := &docker.Container{ID: "myContainerID"} + fw.fakeKubelet.infoFunc = func(name string) (*docker.Container, error) { if name == "goodcontainer" { return expected, nil } - return "", fmt.Errorf("bad container") + return nil, fmt.Errorf("bad container") } resp, err := http.Get(fw.testHttpServer.URL + "/containerInfo?container=goodcontainer") if err != nil { @@ -131,7 +132,11 @@ func TestContainerInfo(t *testing.T) { if err != nil { t.Errorf("Error reading body: %v", err) } - if got != expected { + expectedBytes, err := json.Marshal(expected) + if err != nil { + t.Fatalf("Unexpected marshal error %v", err) + } + if got != string(expectedBytes) { t.Errorf("Expected: '%v', got: '%v'", expected, got) } } diff --git a/pkg/master/pod_cache.go b/pkg/master/pod_cache.go index 33b6da00819..d3b20601b4b 100644 --- a/pkg/master/pod_cache.go +++ b/pkg/master/pod_cache.go @@ -17,6 +17,7 @@ limitations under the License. package master import ( + "errors" "sync" "time" @@ -24,6 +25,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/labels" "github.com/GoogleCloudPlatform/kubernetes/pkg/registry" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + "github.com/fsouza/go-dockerclient" "github.com/golang/glog" ) @@ -32,7 +34,7 @@ import ( type PodCache struct { containerInfo client.ContainerInfo pods registry.PodRegistry - podInfo map[string]interface{} + podInfo map[string]docker.Container period time.Duration podLock sync.Mutex } @@ -41,21 +43,20 @@ func NewPodCache(info client.ContainerInfo, pods registry.PodRegistry, period ti return &PodCache{ containerInfo: info, pods: pods, - podInfo: map[string]interface{}{}, + podInfo: map[string]docker.Container{}, period: period, } } -// Implements the ContainerInfo interface -// The returned value should be treated as read-only -func (p *PodCache) GetContainerInfo(host, id string) (interface{}, error) { +// Implements the ContainerInfo interface. +func (p *PodCache) GetContainerInfo(host, id string) (*docker.Container, error) { p.podLock.Lock() defer p.podLock.Unlock() value, ok := p.podInfo[id] if !ok { - return nil, nil + return nil, errors.New("No cached pod info") } else { - return value, nil + return &value, nil } } @@ -66,7 +67,7 @@ func (p *PodCache) updateContainerInfo(host, id string) error { } p.podLock.Lock() defer p.podLock.Unlock() - p.podInfo[id] = info + p.podInfo[id] = *info return nil } diff --git a/pkg/master/pod_cache_test.go b/pkg/master/pod_cache_test.go index 9db9d8db4bb..7465041b5e9 100644 --- a/pkg/master/pod_cache_test.go +++ b/pkg/master/pod_cache_test.go @@ -23,16 +23,17 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/registry" + "github.com/fsouza/go-dockerclient" ) type FakeContainerInfo struct { host string id string - data interface{} + data *docker.Container err error } -func (f *FakeContainerInfo) GetContainerInfo(host, id string) (interface{}, error) { +func (f *FakeContainerInfo) GetContainerInfo(host, id string) (*docker.Container, error) { f.host = host f.id = id return f.data, f.err @@ -41,17 +42,15 @@ func (f *FakeContainerInfo) GetContainerInfo(host, id string) (interface{}, erro func TestPodCacheGet(t *testing.T) { cache := NewPodCache(nil, nil, time.Second*1) - pod := api.Pod{ - JSONBase: api.JSONBase{ID: "foo"}, - } - cache.podInfo["foo"] = pod + expected := docker.Container{ID: "foo"} + cache.podInfo["foo"] = expected info, err := cache.GetContainerInfo("host", "foo") if err != nil { t.Errorf("Unexpected error: %#v", err) } - if !reflect.DeepEqual(info, pod) { - t.Errorf("Unexpected mismatch. Expected: %#v, Got: #%v", pod, info) + if !reflect.DeepEqual(info, &expected) { + t.Errorf("Unexpected mismatch. Expected: %#v, Got: #%v", &expected, info) } } @@ -59,8 +58,8 @@ func TestPodCacheGetMissing(t *testing.T) { cache := NewPodCache(nil, nil, time.Second*1) info, err := cache.GetContainerInfo("host", "foo") - if err != nil { - t.Errorf("Unexpected error: %#v", err) + if err == nil { + t.Errorf("Unexpected non-error: %#v", err) } if info != nil { t.Errorf("Unexpected info: %#v", info) @@ -68,11 +67,9 @@ func TestPodCacheGetMissing(t *testing.T) { } func TestPodGetContainerInfo(t *testing.T) { - pod := api.Pod{ - JSONBase: api.JSONBase{ID: "foo"}, - } + expected := docker.Container{ID: "foo"} fake := FakeContainerInfo{ - data: pod, + data: &expected, } cache := NewPodCache(&fake, nil, time.Second*1) @@ -86,8 +83,8 @@ func TestPodGetContainerInfo(t *testing.T) { if err != nil { t.Errorf("Unexpected error: %#v", err) } - if !reflect.DeepEqual(info, pod) { - t.Errorf("Unexpected mismatch. Expected: %#v, Got: #%v", pod, info) + if !reflect.DeepEqual(info, &expected) { + t.Errorf("Unexpected mismatch. Expected: %#v, Got: #%v", &expected, info) } } @@ -98,10 +95,13 @@ func TestPodUpdateAllContainers(t *testing.T) { Host: "machine", }, } + pods := []api.Pod{pod} mockRegistry := registry.MakeMockPodRegistry(pods) + + expected := docker.Container{ID: "foo"} fake := FakeContainerInfo{ - data: pod, + data: &expected, } cache := NewPodCache(&fake, mockRegistry, time.Second*1) @@ -115,7 +115,7 @@ func TestPodUpdateAllContainers(t *testing.T) { if err != nil { t.Errorf("Unexpected error: %#v", err) } - if !reflect.DeepEqual(info, pod) { - t.Errorf("Unexpected mismatch. Expected: %#v, Got: #%v", pod, info) + if !reflect.DeepEqual(info, &expected) { + t.Errorf("Unexpected mismatch. Expected: %#v, Got: #%v", &expected, info) } } diff --git a/pkg/registry/pod_registry.go b/pkg/registry/pod_registry.go index 6b1321366e4..495cee38719 100644 --- a/pkg/registry/pod_registry.go +++ b/pkg/registry/pod_registry.go @@ -27,6 +27,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider" "github.com/GoogleCloudPlatform/kubernetes/pkg/labels" "github.com/GoogleCloudPlatform/kubernetes/pkg/scheduler" + "github.com/fsouza/go-dockerclient" "github.com/golang/glog" ) @@ -71,34 +72,62 @@ func (storage *PodRegistryStorage) List(selector labels.Selector) (interface{}, pods, err := storage.registry.ListPods(selector) if err == nil { result.Items = pods - // Get cached info for the list currently. - // TODO: Optionally use fresh info - if storage.podCache != nil { - for ix, pod := range pods { - info, err := storage.podCache.GetContainerInfo(pod.CurrentState.Host, pod.ID) - if err != nil { - glog.Errorf("Error getting container info: %#v", err) - continue - } - result.Items[ix].CurrentState.Info = info - } + for i := range result.Items { + storage.fillPodInfo(&result.Items[i]) } } return result, err } -func makePodStatus(info interface{}) api.PodStatus { - if state, ok := info.(map[string]interface{})["State"]; ok { - if running, ok := state.(map[string]interface{})["Running"]; ok { - if running.(bool) { - return api.PodRunning - } else { - return api.PodStopped +func (storage *PodRegistryStorage) fillPodInfo(pod *api.Pod) { + // Get cached info for the list currently. + // TODO: Optionally use fresh info + if storage.podCache != nil { + pod.CurrentState.Info = map[string]docker.Container{} + infoMap := pod.CurrentState.Info + + for _, container := range pod.DesiredState.Manifest.Containers { + // TODO: clearly need to pass both pod ID and container name here. + info, err := storage.podCache.GetContainerInfo(pod.CurrentState.Host, pod.ID) + if err != nil { + glog.Errorf("Error getting container info: %#v", err) + continue } + infoMap[container.Name] = *info } } - return api.PodPending +} + +func makePodStatus(pod *api.Pod) api.PodStatus { + if pod.CurrentState.Info == nil { + return api.PodPending + } + running := 0 + stopped := 0 + unknown := 0 + for _, container := range pod.DesiredState.Manifest.Containers { + if info, ok := pod.CurrentState.Info[container.Name]; ok { + if info.State.Running { + running++ + } else { + stopped++ + } + } else { + unknown++ + } + } + + switch { + case running > 0 && stopped == 0 && unknown == 0: + return api.PodRunning + case running == 0 && stopped > 0 && unknown == 0: + return api.PodStopped + case running == 0 && stopped == 0 && unknown > 0: + return api.PodPending + default: + return api.PodPending + } } func getInstanceIP(cloud cloudprovider.Interface, host string) string { @@ -130,12 +159,8 @@ func (storage *PodRegistryStorage) Get(id string) (interface{}, error) { return pod, nil } if storage.containerInfo != nil { - info, err := storage.containerInfo.GetContainerInfo(pod.CurrentState.Host, id) - if err != nil { - return pod, err - } - pod.CurrentState.Info = info - pod.CurrentState.Status = makePodStatus(info) + storage.fillPodInfo(pod) + pod.CurrentState.Status = makePodStatus(pod) } pod.CurrentState.HostIP = getInstanceIP(storage.cloud, pod.CurrentState.Host) diff --git a/pkg/registry/pod_registry_test.go b/pkg/registry/pod_registry_test.go index 42505ca876c..c5a42d081ba 100644 --- a/pkg/registry/pod_registry_test.go +++ b/pkg/registry/pod_registry_test.go @@ -26,6 +26,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider" "github.com/GoogleCloudPlatform/kubernetes/pkg/labels" "github.com/GoogleCloudPlatform/kubernetes/pkg/scheduler" + "github.com/fsouza/go-dockerclient" ) func expectNoError(t *testing.T, err error) { @@ -153,29 +154,88 @@ func TestGetPodCloud(t *testing.T) { } func TestMakePodStatus(t *testing.T) { - status := makePodStatus(map[string]interface{}{}) + desiredState := api.PodState{ + Manifest: api.ContainerManifest{ + Containers: []api.Container{ + {Name: "containerA"}, + {Name: "containerB"}, + }, + }, + } + pod := &api.Pod{DesiredState: desiredState} + status := makePodStatus(pod) if status != api.PodPending { t.Errorf("Expected 'Pending', got '%s'", status) } - status = makePodStatus(map[string]interface{}{ - "State": map[string]interface{}{ - "Running": false, + runningState := docker.Container{ + State: docker.State{ + Running: true, }, - }) + } + stoppedState := docker.Container{ + State: docker.State{ + Running: false, + }, + } + // All running. + pod = &api.Pod{ + DesiredState: desiredState, + CurrentState: api.PodState{ + Info: map[string]docker.Container{ + "containerA": runningState, + "containerB": runningState, + }, + }, + } + status = makePodStatus(pod) + if status != api.PodRunning { + t.Errorf("Expected 'Running', got '%s'", status) + } + + // All stopped. + pod = &api.Pod{ + DesiredState: desiredState, + CurrentState: api.PodState{ + Info: map[string]docker.Container{ + "containerA": stoppedState, + "containerB": stoppedState, + }, + }, + } + status = makePodStatus(pod) if status != api.PodStopped { t.Errorf("Expected 'Stopped', got '%s'", status) } - status = makePodStatus(map[string]interface{}{ - "State": map[string]interface{}{ - "Running": true, + // Mixed state. + pod = &api.Pod{ + DesiredState: desiredState, + CurrentState: api.PodState{ + Info: map[string]docker.Container{ + "containerA": runningState, + "containerB": stoppedState, + }, }, - }) + } + status = makePodStatus(pod) + if status != api.PodPending { + t.Errorf("Expected 'Pending', got '%s'", status) + } - if status != api.PodRunning { - t.Errorf("Expected 'Running', got '%s'", status) + // Mixed state. + pod = &api.Pod{ + DesiredState: desiredState, + CurrentState: api.PodState{ + Info: map[string]docker.Container{ + "containerA": runningState, + }, + }, + } + status = makePodStatus(pod) + if status != api.PodPending { + t.Errorf("Expected 'Pending', got '%s'", status) } }