From e2f37f81a98c7966616f12c18c74b3aca32035c4 Mon Sep 17 00:00:00 2001 From: Victor Marmol Date: Thu, 23 Apr 2015 09:23:57 -0700 Subject: [PATCH 1/2] Add SubcontainerInfo() to cAdvisor API. Returns information about a container and its subcontainers. --- pkg/kubelet/cadvisor/cadvisor_fake.go | 4 ++++ pkg/kubelet/cadvisor/cadvisor_linux.go | 13 +++++++++++++ pkg/kubelet/cadvisor/cadvisor_mock.go | 5 +++++ pkg/kubelet/cadvisor/cadvisor_unsupported.go | 4 ++++ pkg/kubelet/cadvisor/types.go | 1 + 5 files changed, 27 insertions(+) diff --git a/pkg/kubelet/cadvisor/cadvisor_fake.go b/pkg/kubelet/cadvisor/cadvisor_fake.go index d43645f2f4e..1bd57c7e41e 100644 --- a/pkg/kubelet/cadvisor/cadvisor_fake.go +++ b/pkg/kubelet/cadvisor/cadvisor_fake.go @@ -32,6 +32,10 @@ func (c *Fake) ContainerInfo(name string, req *cadvisorApi.ContainerInfoRequest) return new(cadvisorApi.ContainerInfo), nil } +func (c *Fake) SubcontainerInfo(name string, req *cadvisorApi.ContainerInfoRequest) (map[string]*cadvisorApi.ContainerInfo, error) { + return map[string]*cadvisorApi.ContainerInfo{}, nil +} + func (c *Fake) DockerContainer(name string, req *cadvisorApi.ContainerInfoRequest) (cadvisorApi.ContainerInfo, error) { return cadvisorApi.ContainerInfo{}, nil } diff --git a/pkg/kubelet/cadvisor/cadvisor_linux.go b/pkg/kubelet/cadvisor/cadvisor_linux.go index e0c4d27c625..c3f691b0312 100644 --- a/pkg/kubelet/cadvisor/cadvisor_linux.go +++ b/pkg/kubelet/cadvisor/cadvisor_linux.go @@ -114,6 +114,19 @@ func (cc *cadvisorClient) VersionInfo() (*cadvisorApi.VersionInfo, error) { return cc.GetVersionInfo() } +func (cc *cadvisorClient) SubcontainerInfo(name string, req *cadvisorApi.ContainerInfoRequest) (map[string]*cadvisorApi.ContainerInfo, error) { + infos, err := cc.SubcontainersInfo(name, req) + if err != nil { + return nil, err + } + + result := make(map[string]*cadvisorApi.ContainerInfo, len(infos)) + for _, info := range infos { + result[info.Name] = info + } + return result, nil +} + func (cc *cadvisorClient) MachineInfo() (*cadvisorApi.MachineInfo, error) { return cc.GetMachineInfo() } diff --git a/pkg/kubelet/cadvisor/cadvisor_mock.go b/pkg/kubelet/cadvisor/cadvisor_mock.go index b1aa1cfd2df..371f30526cb 100644 --- a/pkg/kubelet/cadvisor/cadvisor_mock.go +++ b/pkg/kubelet/cadvisor/cadvisor_mock.go @@ -35,6 +35,11 @@ func (c *Mock) ContainerInfo(name string, req *cadvisorApi.ContainerInfoRequest) return args.Get(0).(*cadvisorApi.ContainerInfo), args.Error(1) } +func (c *Mock) SubcontainerInfo(name string, req *cadvisorApi.ContainerInfoRequest) (map[string]*cadvisorApi.ContainerInfo, error) { + args := c.Called(name, req) + return args.Get(0).(map[string]*cadvisorApi.ContainerInfo), args.Error(1) +} + // DockerContainer is a mock implementation of Interface.DockerContainer. func (c *Mock) DockerContainer(name string, req *cadvisorApi.ContainerInfoRequest) (cadvisorApi.ContainerInfo, error) { args := c.Called(name, req) diff --git a/pkg/kubelet/cadvisor/cadvisor_unsupported.go b/pkg/kubelet/cadvisor/cadvisor_unsupported.go index 21e22a9cc60..b0300ef5bc0 100644 --- a/pkg/kubelet/cadvisor/cadvisor_unsupported.go +++ b/pkg/kubelet/cadvisor/cadvisor_unsupported.go @@ -45,6 +45,10 @@ func (cu *cadvisorUnsupported) ContainerInfo(name string, req *cadvisorApi.Conta return nil, unsupportedErr } +func (cu *cadvisorUnsupported) SubcontainerInfo(name string, req *cadvisorApi.ContainerInfoRequest) (map[string]*cadvisorApi.ContainerInfo, error) { + return nil, unsupportedErr +} + func (cu *cadvisorUnsupported) MachineInfo() (*cadvisorApi.MachineInfo, error) { return nil, unsupportedErr } diff --git a/pkg/kubelet/cadvisor/types.go b/pkg/kubelet/cadvisor/types.go index a82390c4a99..b14154645c6 100644 --- a/pkg/kubelet/cadvisor/types.go +++ b/pkg/kubelet/cadvisor/types.go @@ -26,6 +26,7 @@ import ( type Interface interface { DockerContainer(name string, req *cadvisorApi.ContainerInfoRequest) (cadvisorApi.ContainerInfo, error) ContainerInfo(name string, req *cadvisorApi.ContainerInfoRequest) (*cadvisorApi.ContainerInfo, error) + SubcontainerInfo(name string, req *cadvisorApi.ContainerInfoRequest) (map[string]*cadvisorApi.ContainerInfo, error) MachineInfo() (*cadvisorApi.MachineInfo, error) VersionInfo() (*cadvisorApi.VersionInfo, error) From c29d328c559673b8631a92a9c305a6e6f1167bfb Mon Sep 17 00:00:00 2001 From: Victor Marmol Date: Thu, 23 Apr 2015 10:14:08 -0700 Subject: [PATCH 2/2] Kubelet: Add /stats/container endpoint. This endpoint exposes container stats for all raw containers on the machine. The addition is backwards compatible. --- pkg/kubelet/kubelet.go | 16 +++++-- pkg/kubelet/kubelet_test.go | 40 +++++++++++++++-- pkg/kubelet/server.go | 66 ++++++++++++++++++++++----- pkg/kubelet/server_test.go | 89 ++++++++++++++++++++++++------------- 4 files changed, 163 insertions(+), 48 deletions(-) diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 7cc7bac8869..e28128bc3f0 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -2200,9 +2200,19 @@ func (kl *Kubelet) GetContainerInfo(podFullName string, uid types.UID, container return &ci, nil } -// GetRootInfo returns stats (from Cadvisor) of current machine (root container). -func (kl *Kubelet) GetRootInfo(req *cadvisorApi.ContainerInfoRequest) (*cadvisorApi.ContainerInfo, error) { - return kl.cadvisor.ContainerInfo("/", req) +// Returns stats (from Cadvisor) for a non-Kubernetes container. +func (kl *Kubelet) GetRawContainerInfo(containerName string, req *cadvisorApi.ContainerInfoRequest, subcontainers bool) (map[string]*cadvisorApi.ContainerInfo, error) { + if subcontainers { + return kl.cadvisor.SubcontainerInfo(containerName, req) + } else { + containerInfo, err := kl.cadvisor.ContainerInfo(containerName, req) + if err != nil { + return nil, err + } + return map[string]*cadvisorApi.ContainerInfo{ + containerInfo.Name: containerInfo, + }, nil + } } // GetCachedMachineInfo assumes that the machine info can't change without a reboot diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 23d6ee5c77e..710073f4911 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -1441,7 +1441,7 @@ func TestGetContainerInfo(t *testing.T) { mockCadvisor.AssertExpectations(t) } -func TestGetRootInfo(t *testing.T) { +func TestGetRawContainerInfoRoot(t *testing.T) { containerPath := "/" containerInfo := &cadvisorApi.ContainerInfo{ ContainerReference: cadvisorApi.ContainerReference{ @@ -1459,14 +1459,48 @@ func TestGetRootInfo(t *testing.T) { cadvisor: mockCadvisor, } - // If the container name is an empty string, then it means the root container. - _, err := kubelet.GetRootInfo(cadvisorReq) + _, err := kubelet.GetRawContainerInfo(containerPath, cadvisorReq, false) if err != nil { t.Errorf("unexpected error: %v", err) } mockCadvisor.AssertExpectations(t) } +func TestGetRawContainerInfoSubcontainers(t *testing.T) { + containerPath := "/kubelet" + containerInfo := map[string]*cadvisorApi.ContainerInfo{ + containerPath: { + ContainerReference: cadvisorApi.ContainerReference{ + Name: containerPath, + }, + }, + "/kubelet/sub": { + ContainerReference: cadvisorApi.ContainerReference{ + Name: "/kubelet/sub", + }, + }, + } + fakeDocker := dockertools.FakeDockerClient{} + + mockCadvisor := &cadvisor.Mock{} + cadvisorReq := &cadvisorApi.ContainerInfoRequest{} + mockCadvisor.On("SubcontainerInfo", containerPath, cadvisorReq).Return(containerInfo, nil) + + kubelet := Kubelet{ + dockerClient: &fakeDocker, + cadvisor: mockCadvisor, + } + + result, err := kubelet.GetRawContainerInfo(containerPath, cadvisorReq, true) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if len(result) != 2 { + t.Errorf("Expected 2 elements, received: %+v", result) + } + mockCadvisor.AssertExpectations(t) +} + func TestGetContainerInfoWhenCadvisorFailed(t *testing.T) { containerID := "ab2cdf" diff --git a/pkg/kubelet/server.go b/pkg/kubelet/server.go index e261e0ff8c3..cf76aa565a3 100644 --- a/pkg/kubelet/server.go +++ b/pkg/kubelet/server.go @@ -99,8 +99,8 @@ func ListenAndServeKubeletReadOnlyServer(host HostInterface, address net.IP, por // For testablitiy. type HostInterface interface { GetContainerInfo(podFullName string, uid types.UID, containerName string, req *cadvisorApi.ContainerInfoRequest) (*cadvisorApi.ContainerInfo, error) - GetRootInfo(req *cadvisorApi.ContainerInfoRequest) (*cadvisorApi.ContainerInfo, error) GetContainerRuntimeVersion() (kubecontainer.Version, error) + GetRawContainerInfo(containerName string, req *cadvisorApi.ContainerInfoRequest, subcontainers bool) (map[string]*cadvisorApi.ContainerInfo, error) GetCachedMachineInfo() (*cadvisorApi.MachineInfo, error) GetPods() []*api.Pod GetPodByName(namespace, name string) (*api.Pod, bool) @@ -634,26 +634,68 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { s.mux.ServeHTTP(w, req) } +type StatsRequest struct { + // The name of the container for which to request stats. + // Default: / + ContainerName string `json:"containerName,omitempty"` + + // Max number of stats to return. + // If start and end time are specified this limit is ignored. + // Default: 60 + NumStats int `json:"num_stats,omitempty"` + + // Start time for which to query information. + // If ommitted, the beginning of time is assumed. + Start time.Time `json:"start,omitempty"` + + // End time for which to query information. + // If ommitted, current time is assumed. + End time.Time `json:"end,omitempty"` + + // Whether to also include information from subcontainers. + // Default: false. + Subcontainers bool `json:"subcontainers,omitempty"` +} + // serveStats implements stats logic. func (s *Server) serveStats(w http.ResponseWriter, req *http.Request) { - // /stats// or /stats//// + // Stats requests are in the following forms: + // + // /stats/ : Root container stats + // /stats/container/ : Non-Kubernetes container stats (returns a map) + // /stats// : Stats for Kubernetes pod/container + // /stats//// : Stats for Kubernetes namespace/pod/uid/container components := strings.Split(strings.TrimPrefix(path.Clean(req.URL.Path), "/"), "/") - var stats *cadvisorApi.ContainerInfo + var stats interface{} var err error - query := cadvisorApi.DefaultContainerInfoRequest() + var query StatsRequest + query.NumStats = 60 + err = json.NewDecoder(req.Body).Decode(&query) if err != nil && err != io.EOF { s.error(w, err) return } + cadvisorRequest := cadvisorApi.ContainerInfoRequest{ + NumStats: query.NumStats, + Start: query.Start, + End: query.End, + } + switch len(components) { case 1: - // Machine stats - stats, err = s.host.GetRootInfo(&query) + // Root container stats. + var statsMap map[string]*cadvisorApi.ContainerInfo + statsMap, err = s.host.GetRawContainerInfo("/", &cadvisorRequest, false) + stats = statsMap["/"] case 2: - // pod stats - // TODO(monnand) Implement this - err = errors.New("pod level status currently unimplemented") + // Non-Kubernetes container stats. + if components[1] != "container" { + http.Error(w, fmt.Sprintf("unknown stats request type %q", components[1]), http.StatusNotFound) + return + } + containerName := path.Join("/", query.ContainerName) + stats, err = s.host.GetRawContainerInfo(containerName, &cadvisorRequest, query.Subcontainers) case 3: // Backward compatibility without uid information, does not support namespace pod, ok := s.host.GetPodByName(api.NamespaceDefault, components[1]) @@ -661,16 +703,16 @@ func (s *Server) serveStats(w http.ResponseWriter, req *http.Request) { http.Error(w, "Pod does not exist", http.StatusNotFound) return } - stats, err = s.host.GetContainerInfo(kubecontainer.GetPodFullName(pod), "", components[2], &query) + stats, err = s.host.GetContainerInfo(kubecontainer.GetPodFullName(pod), "", components[2], &cadvisorRequest) case 5: pod, ok := s.host.GetPodByName(components[1], components[2]) if !ok { http.Error(w, "Pod does not exist", http.StatusNotFound) return } - stats, err = s.host.GetContainerInfo(kubecontainer.GetPodFullName(pod), types.UID(components[3]), components[4], &query) + stats, err = s.host.GetContainerInfo(kubecontainer.GetPodFullName(pod), types.UID(components[3]), components[4], &cadvisorRequest) default: - http.Error(w, "unknown resource.", http.StatusNotFound) + http.Error(w, fmt.Sprintf("Unknown resource: %v", components), http.StatusNotFound) return } switch err { diff --git a/pkg/kubelet/server_test.go b/pkg/kubelet/server_test.go index 778cf720346..229ecfc9493 100644 --- a/pkg/kubelet/server_test.go +++ b/pkg/kubelet/server_test.go @@ -17,6 +17,7 @@ limitations under the License. package kubelet import ( + "bytes" "encoding/json" "fmt" "io" @@ -44,7 +45,7 @@ type fakeKubelet struct { podByNameFunc func(namespace, name string) (*api.Pod, bool) statusFunc func(name string) (api.PodStatus, error) containerInfoFunc func(podFullName string, uid types.UID, containerName string, req *cadvisorApi.ContainerInfoRequest) (*cadvisorApi.ContainerInfo, error) - rootInfoFunc func(query *cadvisorApi.ContainerInfoRequest) (*cadvisorApi.ContainerInfo, error) + rawInfoFunc func(query *cadvisorApi.ContainerInfoRequest) (map[string]*cadvisorApi.ContainerInfo, error) machineInfoFunc func() (*cadvisorApi.MachineInfo, error) podsFunc func() []*api.Pod logFunc func(w http.ResponseWriter, req *http.Request) @@ -69,8 +70,8 @@ func (fk *fakeKubelet) GetContainerInfo(podFullName string, uid types.UID, conta return fk.containerInfoFunc(podFullName, uid, containerName, req) } -func (fk *fakeKubelet) GetRootInfo(req *cadvisorApi.ContainerInfoRequest) (*cadvisorApi.ContainerInfo, error) { - return fk.rootInfoFunc(req) +func (fk *fakeKubelet) GetRawContainerInfo(containerName string, req *cadvisorApi.ContainerInfoRequest, subcontainers bool) (map[string]*cadvisorApi.ContainerInfo, error) { + return fk.rawInfoFunc(req) } func (fk *fakeKubelet) GetContainerRuntimeVersion() (kubecontainer.Version, error) { @@ -269,9 +270,15 @@ func TestContainerNotFound(t *testing.T) { func TestRootInfo(t *testing.T) { fw := newServerTest() - expectedInfo := &cadvisorApi.ContainerInfo{} - fw.fakeKubelet.rootInfoFunc = func(req *cadvisorApi.ContainerInfoRequest) (*cadvisorApi.ContainerInfo, error) { - return expectedInfo, nil + expectedInfo := &cadvisorApi.ContainerInfo{ + ContainerReference: cadvisorApi.ContainerReference{ + Name: "/", + }, + } + fw.fakeKubelet.rawInfoFunc = func(req *cadvisorApi.ContainerInfoRequest) (map[string]*cadvisorApi.ContainerInfo, error) { + return map[string]*cadvisorApi.ContainerInfo{ + expectedInfo.Name: expectedInfo, + }, nil } resp, err := http.Get(fw.testHTTPServer.URL + "/stats") @@ -285,7 +292,52 @@ func TestRootInfo(t *testing.T) { t.Fatalf("received invalid json data: %v", err) } if !receivedInfo.Eq(expectedInfo) { - t.Errorf("received wrong data: %#v", receivedInfo) + t.Errorf("received wrong data: %#v, expected %#v", receivedInfo, expectedInfo) + } +} + +func TestSubcontainerContainerInfo(t *testing.T) { + fw := newServerTest() + const kubeletContainer = "/kubelet" + const kubeletSubContainer = "/kubelet/sub" + expectedInfo := map[string]*cadvisorApi.ContainerInfo{ + kubeletContainer: { + ContainerReference: cadvisorApi.ContainerReference{ + Name: kubeletContainer, + }, + }, + kubeletSubContainer: { + ContainerReference: cadvisorApi.ContainerReference{ + Name: kubeletSubContainer, + }, + }, + } + fw.fakeKubelet.rawInfoFunc = func(req *cadvisorApi.ContainerInfoRequest) (map[string]*cadvisorApi.ContainerInfo, error) { + return expectedInfo, nil + } + + request := fmt.Sprintf("{\"containerName\":%q, \"subcontainers\": true}", kubeletContainer) + resp, err := http.Post(fw.testHTTPServer.URL+"/stats/container", "application/json", bytes.NewBuffer([]byte(request))) + if err != nil { + t.Fatalf("Got error GETing: %v", err) + } + defer resp.Body.Close() + var receivedInfo map[string]*cadvisorApi.ContainerInfo + err = json.NewDecoder(resp.Body).Decode(&receivedInfo) + if err != nil { + t.Fatalf("Received invalid json data: %v", err) + } + if len(receivedInfo) != len(expectedInfo) { + t.Errorf("Received wrong data: %#v, expected %#v", receivedInfo, expectedInfo) + } + + for _, containerName := range []string{kubeletContainer, kubeletSubContainer} { + if _, ok := receivedInfo[containerName]; !ok { + t.Errorf("Expected container %q to be present in result: %#v", containerName, receivedInfo) + } + if !receivedInfo[containerName].Eq(expectedInfo[containerName]) { + t.Errorf("Invalid result for %q: Expected %#v, received %#v", containerName, expectedInfo[containerName], receivedInfo[containerName]) + } } } @@ -426,29 +478,6 @@ func TestServeRunInContainerWithUID(t *testing.T) { } } -// TODO: fix me when pod level stats get implemented -func TestPodsInfo(t *testing.T) { - fw := newServerTest() - - resp, err := http.Get(fw.testHTTPServer.URL + "/stats/goodpod") - if err != nil { - t.Fatalf("Got error GETing: %v", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusInternalServerError { - t.Errorf("expected status code %d, got %d", http.StatusInternalServerError, resp.StatusCode) - } - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - // copying the response body did not work - t.Fatalf("Cannot copy resp: %#v", err) - } - result := string(body) - if !strings.Contains(result, "pod level status currently unimplemented") { - t.Errorf("expected body contains pod level status currently unimplemented, got %s", result) - } -} - func TestHealthCheck(t *testing.T) { fw := newServerTest() fw.fakeKubelet.containerVersionFunc = func() (kubecontainer.Version, error) {