From 5eae7eb1664ef03355c5a3341361dc2331cf9b4f Mon Sep 17 00:00:00 2001 From: David Porter Date: Mon, 21 Aug 2017 20:28:44 +0000 Subject: [PATCH] Implement CRI stats in dockershim for Windows Implement CRI stats for dockershim using docker stats. This enables use of the summary api to get container metrics on Windows where CRI stats are enabled. --- pkg/kubelet/dockershim/BUILD | 7 +- pkg/kubelet/dockershim/docker_image.go | 5 - pkg/kubelet/dockershim/docker_image_linux.go | 30 +++++ .../dockershim/docker_image_unsupported.go | 30 +++++ .../dockershim/docker_image_windows.go | 39 +++++++ ...{docker_stats.go => docker_stats_linux.go} | 10 +- .../dockershim/docker_stats_unsupported.go | 35 ++++++ .../dockershim/docker_stats_windows.go | 105 ++++++++++++++++++ pkg/kubelet/dockershim/libdocker/client.go | 2 + .../dockershim/libdocker/fake_client.go | 24 ++++ .../libdocker/instrumented_client.go | 18 +++ .../libdocker/kube_docker_client.go | 36 ++++++ .../dockershim/remote/docker_service.go | 19 +++- 13 files changed, 346 insertions(+), 14 deletions(-) create mode 100644 pkg/kubelet/dockershim/docker_image_linux.go create mode 100644 pkg/kubelet/dockershim/docker_image_unsupported.go create mode 100644 pkg/kubelet/dockershim/docker_image_windows.go rename pkg/kubelet/dockershim/{docker_stats.go => docker_stats_linux.go} (76%) create mode 100644 pkg/kubelet/dockershim/docker_stats_unsupported.go create mode 100644 pkg/kubelet/dockershim/docker_stats_windows.go diff --git a/pkg/kubelet/dockershim/BUILD b/pkg/kubelet/dockershim/BUILD index 9b7250a03d3..6f48b128bf8 100644 --- a/pkg/kubelet/dockershim/BUILD +++ b/pkg/kubelet/dockershim/BUILD @@ -15,9 +15,10 @@ go_library( "docker_checkpoint.go", "docker_container.go", "docker_image.go", + "docker_image_unsupported.go", "docker_sandbox.go", "docker_service.go", - "docker_stats.go", + "docker_stats_unsupported.go", "docker_streaming.go", "exec.go", "helpers.go", @@ -27,9 +28,13 @@ go_library( "selinux_util.go", ] + select({ "@io_bazel_rules_go//go/platform:linux_amd64": [ + "docker_image_linux.go", + "docker_stats_linux.go", "helpers_linux.go", ], "@io_bazel_rules_go//go/platform:windows_amd64": [ + "docker_image_windows.go", + "docker_stats_windows.go", "helpers_windows.go", ], "//conditions:default": [], diff --git a/pkg/kubelet/dockershim/docker_image.go b/pkg/kubelet/dockershim/docker_image.go index 7482dc420dd..b1171c6f054 100644 --- a/pkg/kubelet/dockershim/docker_image.go +++ b/pkg/kubelet/dockershim/docker_image.go @@ -133,11 +133,6 @@ func getImageRef(client libdocker.Interface, image string) (string, error) { return img.ID, nil } -// ImageFsInfo returns information of the filesystem that is used to store images. -func (ds *dockerService) ImageFsInfo() ([]*runtimeapi.FilesystemUsage, error) { - return nil, fmt.Errorf("not implemented") -} - func filterHTTPError(err error, image string) error { // docker/docker/pull/11314 prints detailed error info for docker pull. // When it hits 502, it returns a verbose html output including an inline svg, diff --git a/pkg/kubelet/dockershim/docker_image_linux.go b/pkg/kubelet/dockershim/docker_image_linux.go new file mode 100644 index 00000000000..1af785972cc --- /dev/null +++ b/pkg/kubelet/dockershim/docker_image_linux.go @@ -0,0 +1,30 @@ +// +build linux + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dockershim + +import ( + "fmt" + + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" +) + +// ImageFsInfo returns information of the filesystem that is used to store images. +func (ds *dockerService) ImageFsInfo() ([]*runtimeapi.FilesystemUsage, error) { + return nil, fmt.Errorf("not implemented") +} diff --git a/pkg/kubelet/dockershim/docker_image_unsupported.go b/pkg/kubelet/dockershim/docker_image_unsupported.go new file mode 100644 index 00000000000..17519e0385f --- /dev/null +++ b/pkg/kubelet/dockershim/docker_image_unsupported.go @@ -0,0 +1,30 @@ +// +build !linux,!windows + +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dockershim + +import ( + "fmt" + + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" +) + +// ImageFsInfo returns information of the filesystem that is used to store images. +func (ds *dockerService) ImageFsInfo() ([]*runtimeapi.FilesystemUsage, error) { + return nil, fmt.Errorf("not implemented") +} diff --git a/pkg/kubelet/dockershim/docker_image_windows.go b/pkg/kubelet/dockershim/docker_image_windows.go new file mode 100644 index 00000000000..b147659f2b6 --- /dev/null +++ b/pkg/kubelet/dockershim/docker_image_windows.go @@ -0,0 +1,39 @@ +// +build windows + +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dockershim + +import ( + "time" + + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" +) + +// ImageFsInfo returns information of the filesystem that is used to store images. +func (ds *dockerService) ImageFsInfo() ([]*runtimeapi.FilesystemUsage, error) { + // For Windows Stats to work correctly, a file system must be provided. For now, provide a fake filesystem. + filesystems := []*runtimeapi.FilesystemUsage{ + { + Timestamp: time.Now().UnixNano(), + UsedBytes: &runtimeapi.UInt64Value{Value: 0}, + InodesUsed: &runtimeapi.UInt64Value{Value: 0}, + }, + } + + return filesystems, nil +} diff --git a/pkg/kubelet/dockershim/docker_stats.go b/pkg/kubelet/dockershim/docker_stats_linux.go similarity index 76% rename from pkg/kubelet/dockershim/docker_stats.go rename to pkg/kubelet/dockershim/docker_stats_linux.go index 9d840a45faa..48622185f1b 100644 --- a/pkg/kubelet/dockershim/docker_stats.go +++ b/pkg/kubelet/dockershim/docker_stats_linux.go @@ -1,3 +1,5 @@ +// +build linux + /* Copyright 2017 The Kubernetes Authors. @@ -18,15 +20,15 @@ package dockershim import ( "fmt" - runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" ) -// DockerService does not implement container stats. +// ContainerStats returns stats for a container stats request based on container id. func (ds *dockerService) ContainerStats(string) (*runtimeapi.ContainerStats, error) { - return nil, fmt.Errorf("Not implemented") + return nil, fmt.Errorf("not implemented") } +// ListContainerStats returns stats for a list container stats request based on a filter. func (ds *dockerService) ListContainerStats(*runtimeapi.ContainerStatsFilter) ([]*runtimeapi.ContainerStats, error) { - return nil, fmt.Errorf("Not implemented") + return nil, fmt.Errorf("not implemented") } diff --git a/pkg/kubelet/dockershim/docker_stats_unsupported.go b/pkg/kubelet/dockershim/docker_stats_unsupported.go new file mode 100644 index 00000000000..d9f7beef5c7 --- /dev/null +++ b/pkg/kubelet/dockershim/docker_stats_unsupported.go @@ -0,0 +1,35 @@ +// +build !linux,!windows + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dockershim + +import ( + "fmt" + + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" +) + +// ContainerStats returns stats for a container stats request based on container id. +func (ds *dockerService) ContainerStats(string) (*runtimeapi.ContainerStats, error) { + return nil, fmt.Errorf("not implemented") +} + +// ListContainerStats returns stats for a list container stats request based on a filter. +func (ds *dockerService) ListContainerStats(*runtimeapi.ContainerStatsFilter) ([]*runtimeapi.ContainerStats, error) { + return nil, fmt.Errorf("not implemented") +} diff --git a/pkg/kubelet/dockershim/docker_stats_windows.go b/pkg/kubelet/dockershim/docker_stats_windows.go new file mode 100644 index 00000000000..6f51239502e --- /dev/null +++ b/pkg/kubelet/dockershim/docker_stats_windows.go @@ -0,0 +1,105 @@ +// +build windows + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dockershim + +import ( + "time" + + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" +) + +// ContainerStats returns stats for a container stats request based on container id. +func (ds *dockerService) ContainerStats(containerID string) (*runtimeapi.ContainerStats, error) { + containerStats, err := ds.getContainerStats(containerID) + if err != nil { + return nil, err + } + return containerStats, nil +} + +// ListContainerStats returns stats for a list container stats request based on a filter. +func (ds *dockerService) ListContainerStats(containerStatsFilter *runtimeapi.ContainerStatsFilter) ([]*runtimeapi.ContainerStats, error) { + filter := &runtimeapi.ContainerFilter{} + + if containerStatsFilter != nil { + filter.Id = containerStatsFilter.Id + filter.PodSandboxId = containerStatsFilter.PodSandboxId + filter.LabelSelector = containerStatsFilter.LabelSelector + } + + containers, err := ds.ListContainers(filter) + if err != nil { + return nil, err + } + + var stats []*runtimeapi.ContainerStats + for _, container := range containers { + containerStats, err := ds.getContainerStats(container.Id) + if err != nil { + return nil, err + } + + stats = append(stats, containerStats) + } + + return stats, nil +} + +func (ds *dockerService) getContainerStats(containerID string) (*runtimeapi.ContainerStats, error) { + statsJSON, err := ds.client.GetContainerStats(containerID) + if err != nil { + return nil, err + } + + containerJSON, err := ds.client.InspectContainerWithSize(containerID) + if err != nil { + return nil, err + } + + status, err := ds.ContainerStatus(containerID) + if err != nil { + return nil, err + } + + dockerStats := statsJSON.Stats + timestamp := time.Now().UnixNano() + containerStats := &runtimeapi.ContainerStats{ + Attributes: &runtimeapi.ContainerAttributes{ + Id: containerID, + Metadata: status.Metadata, + Labels: status.Labels, + Annotations: status.Annotations, + }, + Cpu: &runtimeapi.CpuUsage{ + Timestamp: timestamp, + // have to multiply cpu usage by 100 since docker stats units is in 100's of nano seconds for Windows + // see https://github.com/moby/moby/blob/v1.13.1/api/types/stats.go#L22 + UsageCoreNanoSeconds: &runtimeapi.UInt64Value{Value: dockerStats.CPUStats.CPUUsage.TotalUsage * 100}, + }, + Memory: &runtimeapi.MemoryUsage{ + Timestamp: timestamp, + WorkingSetBytes: &runtimeapi.UInt64Value{Value: dockerStats.MemoryStats.PrivateWorkingSet}, + }, + WritableLayer: &runtimeapi.FilesystemUsage{ + Timestamp: timestamp, + UsedBytes: &runtimeapi.UInt64Value{Value: uint64(*containerJSON.SizeRw)}, + }, + } + return containerStats, nil +} diff --git a/pkg/kubelet/dockershim/libdocker/client.go b/pkg/kubelet/dockershim/libdocker/client.go index 40bf612a6ce..0b5dcdd8da8 100644 --- a/pkg/kubelet/dockershim/libdocker/client.go +++ b/pkg/kubelet/dockershim/libdocker/client.go @@ -46,6 +46,7 @@ const ( type Interface interface { ListContainers(options dockertypes.ContainerListOptions) ([]dockertypes.Container, error) InspectContainer(id string) (*dockertypes.ContainerJSON, error) + InspectContainerWithSize(id string) (*dockertypes.ContainerJSON, error) CreateContainer(dockertypes.ContainerCreateConfig) (*dockercontainer.ContainerCreateCreatedBody, error) StartContainer(id string) error StopContainer(id string, timeout time.Duration) error @@ -66,6 +67,7 @@ type Interface interface { AttachToContainer(string, dockertypes.ContainerAttachOptions, StreamOptions) error ResizeContainerTTY(id string, height, width uint) error ResizeExecTTY(id string, height, width uint) error + GetContainerStats(id string) (*dockertypes.StatsJSON, error) } // Get a *dockerapi.Client, either using the endpoint passed in, or using diff --git a/pkg/kubelet/dockershim/libdocker/fake_client.go b/pkg/kubelet/dockershim/libdocker/fake_client.go index dd0d9c86e2a..fc2c603176c 100644 --- a/pkg/kubelet/dockershim/libdocker/fake_client.go +++ b/pkg/kubelet/dockershim/libdocker/fake_client.go @@ -461,6 +461,23 @@ func (f *FakeDockerClient) InspectContainer(id string) (*dockertypes.ContainerJS return nil, fmt.Errorf("container %q not found", id) } +// InspectContainerWithSize is a test-spy implementation of Interface.InspectContainerWithSize. +// It adds an entry "inspect" to the internal method call record. +func (f *FakeDockerClient) InspectContainerWithSize(id string) (*dockertypes.ContainerJSON, error) { + f.Lock() + defer f.Unlock() + f.appendCalled(calledDetail{name: "inspect_container_withsize"}) + err := f.popError("inspect_container_withsize") + if container, ok := f.ContainerMap[id]; ok { + return container, err + } + if err != nil { + // Use the custom error if it exists. + return nil, err + } + return nil, fmt.Errorf("container %q not found", id) +} + // InspectImageByRef is a test-spy implementation of Interface.InspectImageByRef. // It adds an entry "inspect" to the internal method call record. func (f *FakeDockerClient) InspectImageByRef(name string) (*dockertypes.ImageInspect, error) { @@ -852,3 +869,10 @@ func (f *FakeDockerPuller) GetImageRef(image string) (string, error) { } return image, err } + +func (f *FakeDockerClient) GetContainerStats(id string) (*dockertypes.StatsJSON, error) { + f.Lock() + defer f.Unlock() + f.appendCalled(calledDetail{name: "getContainerStats"}) + return nil, fmt.Errorf("not implemented") +} diff --git a/pkg/kubelet/dockershim/libdocker/instrumented_client.go b/pkg/kubelet/dockershim/libdocker/instrumented_client.go index 69c79c5bf39..d881a7c01c2 100644 --- a/pkg/kubelet/dockershim/libdocker/instrumented_client.go +++ b/pkg/kubelet/dockershim/libdocker/instrumented_client.go @@ -73,6 +73,15 @@ func (in instrumentedInterface) InspectContainer(id string) (*dockertypes.Contai return out, err } +func (in instrumentedInterface) InspectContainerWithSize(id string) (*dockertypes.ContainerJSON, error) { + const operation = "inspect_container_withsize" + defer recordOperation(operation, time.Now()) + + out, err := in.client.InspectContainerWithSize(id) + recordError(operation, err) + return out, err +} + func (in instrumentedInterface) CreateContainer(opts dockertypes.ContainerCreateConfig) (*dockercontainer.ContainerCreateCreatedBody, error) { const operation = "create_container" defer recordOperation(operation, time.Now()) @@ -251,3 +260,12 @@ func (in instrumentedInterface) ResizeContainerTTY(id string, height, width uint recordError(operation, err) return err } + +func (in instrumentedInterface) GetContainerStats(id string) (*dockertypes.StatsJSON, error) { + const operation = "stats" + defer recordOperation(operation, time.Now()) + + out, err := in.client.GetContainerStats(id) + recordError(operation, err) + return out, err +} diff --git a/pkg/kubelet/dockershim/libdocker/kube_docker_client.go b/pkg/kubelet/dockershim/libdocker/kube_docker_client.go index e002b9dbb37..0cc0c3ae039 100644 --- a/pkg/kubelet/dockershim/libdocker/kube_docker_client.go +++ b/pkg/kubelet/dockershim/libdocker/kube_docker_client.go @@ -123,6 +123,21 @@ func (d *kubeDockerClient) InspectContainer(id string) (*dockertypes.ContainerJS return &containerJSON, nil } +// InspectContainerWithSize is currently only used for Windows container stats +func (d *kubeDockerClient) InspectContainerWithSize(id string) (*dockertypes.ContainerJSON, error) { + ctx, cancel := d.getTimeoutContext() + defer cancel() + // Inspects the container including the fields SizeRw and SizeRootFs. + containerJSON, _, err := d.client.ContainerInspectWithRaw(ctx, id, true) + if ctxErr := contextError(ctx); ctxErr != nil { + return nil, ctxErr + } + if err != nil { + return nil, err + } + return &containerJSON, nil +} + func (d *kubeDockerClient) CreateContainer(opts dockertypes.ContainerCreateConfig) (*dockercontainer.ContainerCreateCreatedBody, error) { ctx, cancel := d.getTimeoutContext() defer cancel() @@ -522,6 +537,27 @@ func (d *kubeDockerClient) ResizeContainerTTY(id string, height, width uint) err }) } +// GetContainerStats is currently only used for Windows container stats +func (d *kubeDockerClient) GetContainerStats(id string) (*dockertypes.StatsJSON, error) { + ctx, cancel := d.getCancelableContext() + defer cancel() + + response, err := d.client.ContainerStats(ctx, id, false) + if err != nil { + return nil, err + } + + dec := json.NewDecoder(response.Body) + var stats dockertypes.StatsJSON + err = dec.Decode(&stats) + if err != nil { + return nil, err + } + + defer response.Body.Close() + return &stats, nil +} + // redirectResponseToOutputStream redirect the response stream to stdout and stderr. When tty is true, all stream will // only be redirected to stdout. func (d *kubeDockerClient) redirectResponseToOutputStream(tty bool, outputStream, errorStream io.Writer, resp io.Reader) error { diff --git a/pkg/kubelet/dockershim/remote/docker_service.go b/pkg/kubelet/dockershim/remote/docker_service.go index f51004aa058..2d85f179000 100644 --- a/pkg/kubelet/dockershim/remote/docker_service.go +++ b/pkg/kubelet/dockershim/remote/docker_service.go @@ -17,7 +17,6 @@ limitations under the License. package remote import ( - "fmt" "time" "golang.org/x/net/context" @@ -226,13 +225,25 @@ func (d *dockerService) RemoveImage(ctx context.Context, r *runtimeapi.RemoveIma // ImageFsInfo returns information of the filesystem that is used to store images. func (d *dockerService) ImageFsInfo(ctx context.Context, r *runtimeapi.ImageFsInfoRequest) (*runtimeapi.ImageFsInfoResponse, error) { - return nil, fmt.Errorf("not implemented") + filesystems, err := d.imageService.ImageFsInfo() + if err != nil { + return nil, err + } + return &runtimeapi.ImageFsInfoResponse{ImageFilesystems: filesystems}, nil } func (d *dockerService) ContainerStats(ctx context.Context, r *runtimeapi.ContainerStatsRequest) (*runtimeapi.ContainerStatsResponse, error) { - return nil, fmt.Errorf("not implemented") + stats, err := d.runtimeService.ContainerStats(r.ContainerId) + if err != nil { + return nil, err + } + return &runtimeapi.ContainerStatsResponse{Stats: stats}, nil } func (d *dockerService) ListContainerStats(ctx context.Context, r *runtimeapi.ListContainerStatsRequest) (*runtimeapi.ListContainerStatsResponse, error) { - return nil, fmt.Errorf("not implemented") + stats, err := d.runtimeService.ListContainerStats(r.GetFilter()) + if err != nil { + return nil, err + } + return &runtimeapi.ListContainerStatsResponse{Stats: stats}, nil }