Implement CRI stats in dockershim for Windows

Implement CRI stats for dockershim using docker stats. This enables use
of the summary api to get container metrics on Windows where CRI stats
are enabled.

pkg/kubelet/dockershim/BUILD

@@ -15,9 +15,10 @@ go_library(
         "docker_checkpoint.go",
         "docker_container.go",
         "docker_image.go",
+        "docker_image_unsupported.go",
         "docker_sandbox.go",
         "docker_service.go",
-        "docker_stats.go",
+        "docker_stats_unsupported.go",
         "docker_streaming.go",
         "exec.go",
         "helpers.go",
@@ -27,9 +28,13 @@ go_library(
         "selinux_util.go",
     ] + select({
         "@io_bazel_rules_go//go/platform:linux_amd64": [
+            "docker_image_linux.go",
+            "docker_stats_linux.go",
             "helpers_linux.go",
         ],
         "@io_bazel_rules_go//go/platform:windows_amd64": [
+            "docker_image_windows.go",
+            "docker_stats_windows.go",
             "helpers_windows.go",
         ],
         "//conditions:default": [],

pkg/kubelet/dockershim/docker_image.go

@@ -133,11 +133,6 @@ func getImageRef(client libdocker.Interface, image string) (string, error) {
 	return img.ID, nil
 }
 
-// ImageFsInfo returns information of the filesystem that is used to store images.
-func (ds *dockerService) ImageFsInfo() ([]*runtimeapi.FilesystemUsage, error) {
-	return nil, fmt.Errorf("not implemented")
-}
-
 func filterHTTPError(err error, image string) error {
 	// docker/docker/pull/11314 prints detailed error info for docker pull.
 	// When it hits 502, it returns a verbose html output including an inline svg,

pkg/kubelet/dockershim/docker_image_linux.go

@@ -0,0 +1,30 @@
+// +build linux
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dockershim
+
+import (
+	"fmt"
+
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+)
+
+// ImageFsInfo returns information of the filesystem that is used to store images.
+func (ds *dockerService) ImageFsInfo() ([]*runtimeapi.FilesystemUsage, error) {
+	return nil, fmt.Errorf("not implemented")
+}

pkg/kubelet/dockershim/docker_image_unsupported.go

@@ -0,0 +1,30 @@
+// +build !linux,!windows
+
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dockershim
+
+import (
+	"fmt"
+
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+)
+
+// ImageFsInfo returns information of the filesystem that is used to store images.
+func (ds *dockerService) ImageFsInfo() ([]*runtimeapi.FilesystemUsage, error) {
+	return nil, fmt.Errorf("not implemented")
+}

pkg/kubelet/dockershim/docker_image_windows.go

@@ -0,0 +1,39 @@
+// +build windows
+
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dockershim
+
+import (
+	"time"
+
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+)
+
+// ImageFsInfo returns information of the filesystem that is used to store images.
+func (ds *dockerService) ImageFsInfo() ([]*runtimeapi.FilesystemUsage, error) {
+	// For Windows Stats to work correctly, a file system must be provided. For now, provide a fake filesystem.
+	filesystems := []*runtimeapi.FilesystemUsage{
+		{
+			Timestamp:  time.Now().UnixNano(),
+			UsedBytes:  &runtimeapi.UInt64Value{Value: 0},
+			InodesUsed: &runtimeapi.UInt64Value{Value: 0},
+		},
+	}
+
+	return filesystems, nil
+}

pkg/kubelet/dockershim/docker_stats_linux.go

@@ -1,3 +1,5 @@
+// +build linux
+
 /*
 Copyright 2017 The Kubernetes Authors.
 
@@ -18,15 +20,15 @@ package dockershim
 
 import (
 	"fmt"
-
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 )
 
-// DockerService does not implement container stats.
+// ContainerStats returns stats for a container stats request based on container id.
 func (ds *dockerService) ContainerStats(string) (*runtimeapi.ContainerStats, error) {
-	return nil, fmt.Errorf("Not implemented")
+	return nil, fmt.Errorf("not implemented")
 }
 
+// ListContainerStats returns stats for a list container stats request based on a filter.
 func (ds *dockerService) ListContainerStats(*runtimeapi.ContainerStatsFilter) ([]*runtimeapi.ContainerStats, error) {
-	return nil, fmt.Errorf("Not implemented")
+	return nil, fmt.Errorf("not implemented")
 }

pkg/kubelet/dockershim/docker_stats_unsupported.go

@@ -0,0 +1,35 @@
+// +build !linux,!windows
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dockershim
+
+import (
+	"fmt"
+
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+)
+
+// ContainerStats returns stats for a container stats request based on container id.
+func (ds *dockerService) ContainerStats(string) (*runtimeapi.ContainerStats, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+// ListContainerStats returns stats for a list container stats request based on a filter.
+func (ds *dockerService) ListContainerStats(*runtimeapi.ContainerStatsFilter) ([]*runtimeapi.ContainerStats, error) {
+	return nil, fmt.Errorf("not implemented")
+}

pkg/kubelet/dockershim/docker_stats_windows.go

@@ -0,0 +1,105 @@
+// +build windows
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dockershim
+
+import (
+	"time"
+
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+)
+
+// ContainerStats returns stats for a container stats request based on container id.
+func (ds *dockerService) ContainerStats(containerID string) (*runtimeapi.ContainerStats, error) {
+	containerStats, err := ds.getContainerStats(containerID)
+	if err != nil {
+		return nil, err
+	}
+	return containerStats, nil
+}
+
+// ListContainerStats returns stats for a list container stats request based on a filter.
+func (ds *dockerService) ListContainerStats(containerStatsFilter *runtimeapi.ContainerStatsFilter) ([]*runtimeapi.ContainerStats, error) {
+	filter := &runtimeapi.ContainerFilter{}
+
+	if containerStatsFilter != nil {
+		filter.Id = containerStatsFilter.Id
+		filter.PodSandboxId = containerStatsFilter.PodSandboxId
+		filter.LabelSelector = containerStatsFilter.LabelSelector
+	}
+
+	containers, err := ds.ListContainers(filter)
+	if err != nil {
+		return nil, err
+	}
+
+	var stats []*runtimeapi.ContainerStats
+	for _, container := range containers {
+		containerStats, err := ds.getContainerStats(container.Id)
+		if err != nil {
+			return nil, err
+		}
+
+		stats = append(stats, containerStats)
+	}
+
+	return stats, nil
+}
+
+func (ds *dockerService) getContainerStats(containerID string) (*runtimeapi.ContainerStats, error) {
+	statsJSON, err := ds.client.GetContainerStats(containerID)
+	if err != nil {
+		return nil, err
+	}
+
+	containerJSON, err := ds.client.InspectContainerWithSize(containerID)
+	if err != nil {
+		return nil, err
+	}
+
+	status, err := ds.ContainerStatus(containerID)
+	if err != nil {
+		return nil, err
+	}
+
+	dockerStats := statsJSON.Stats
+	timestamp := time.Now().UnixNano()
+	containerStats := &runtimeapi.ContainerStats{
+		Attributes: &runtimeapi.ContainerAttributes{
+			Id:          containerID,
+			Metadata:    status.Metadata,
+			Labels:      status.Labels,
+			Annotations: status.Annotations,
+		},
+		Cpu: &runtimeapi.CpuUsage{
+			Timestamp: timestamp,
+			// have to multiply cpu usage by 100 since docker stats units is in 100's of nano seconds for Windows
+			// see https://github.com/moby/moby/blob/v1.13.1/api/types/stats.go#L22
+			UsageCoreNanoSeconds: &runtimeapi.UInt64Value{Value: dockerStats.CPUStats.CPUUsage.TotalUsage * 100},
+		},
+		Memory: &runtimeapi.MemoryUsage{
+			Timestamp:       timestamp,
+			WorkingSetBytes: &runtimeapi.UInt64Value{Value: dockerStats.MemoryStats.PrivateWorkingSet},
+		},
+		WritableLayer: &runtimeapi.FilesystemUsage{
+			Timestamp: timestamp,
+			UsedBytes: &runtimeapi.UInt64Value{Value: uint64(*containerJSON.SizeRw)},
+		},
+	}
+	return containerStats, nil
+}

pkg/kubelet/dockershim/libdocker/client.go

@@ -46,6 +46,7 @@ const (
 type Interface interface {
 	ListContainers(options dockertypes.ContainerListOptions) ([]dockertypes.Container, error)
 	InspectContainer(id string) (*dockertypes.ContainerJSON, error)
+	InspectContainerWithSize(id string) (*dockertypes.ContainerJSON, error)
 	CreateContainer(dockertypes.ContainerCreateConfig) (*dockercontainer.ContainerCreateCreatedBody, error)
 	StartContainer(id string) error
 	StopContainer(id string, timeout time.Duration) error
@@ -66,6 +67,7 @@ type Interface interface {
 	AttachToContainer(string, dockertypes.ContainerAttachOptions, StreamOptions) error
 	ResizeContainerTTY(id string, height, width uint) error
 	ResizeExecTTY(id string, height, width uint) error
+	GetContainerStats(id string) (*dockertypes.StatsJSON, error)
 }
 
 // Get a *dockerapi.Client, either using the endpoint passed in, or using

pkg/kubelet/dockershim/libdocker/fake_client.go

@@ -461,6 +461,23 @@ func (f *FakeDockerClient) InspectContainer(id string) (*dockertypes.ContainerJS
 	return nil, fmt.Errorf("container %q not found", id)
 }
 
+// InspectContainerWithSize is a test-spy implementation of Interface.InspectContainerWithSize.
+// It adds an entry "inspect" to the internal method call record.
+func (f *FakeDockerClient) InspectContainerWithSize(id string) (*dockertypes.ContainerJSON, error) {
+	f.Lock()
+	defer f.Unlock()
+	f.appendCalled(calledDetail{name: "inspect_container_withsize"})
+	err := f.popError("inspect_container_withsize")
+	if container, ok := f.ContainerMap[id]; ok {
+		return container, err
+	}
+	if err != nil {
+		// Use the custom error if it exists.
+		return nil, err
+	}
+	return nil, fmt.Errorf("container %q not found", id)
+}
+
 // InspectImageByRef is a test-spy implementation of Interface.InspectImageByRef.
 // It adds an entry "inspect" to the internal method call record.
 func (f *FakeDockerClient) InspectImageByRef(name string) (*dockertypes.ImageInspect, error) {
@@ -852,3 +869,10 @@ func (f *FakeDockerPuller) GetImageRef(image string) (string, error) {
 	}
 	return image, err
 }
+
+func (f *FakeDockerClient) GetContainerStats(id string) (*dockertypes.StatsJSON, error) {
+	f.Lock()
+	defer f.Unlock()
+	f.appendCalled(calledDetail{name: "getContainerStats"})
+	return nil, fmt.Errorf("not implemented")
+}

pkg/kubelet/dockershim/libdocker/instrumented_client.go

@@ -73,6 +73,15 @@ func (in instrumentedInterface) InspectContainer(id string) (*dockertypes.Contai
 	return out, err
 }
 
+func (in instrumentedInterface) InspectContainerWithSize(id string) (*dockertypes.ContainerJSON, error) {
+	const operation = "inspect_container_withsize"
+	defer recordOperation(operation, time.Now())
+
+	out, err := in.client.InspectContainerWithSize(id)
+	recordError(operation, err)
+	return out, err
+}
+
 func (in instrumentedInterface) CreateContainer(opts dockertypes.ContainerCreateConfig) (*dockercontainer.ContainerCreateCreatedBody, error) {
 	const operation = "create_container"
 	defer recordOperation(operation, time.Now())
@@ -251,3 +260,12 @@ func (in instrumentedInterface) ResizeContainerTTY(id string, height, width uint
 	recordError(operation, err)
 	return err
 }
+
+func (in instrumentedInterface) GetContainerStats(id string) (*dockertypes.StatsJSON, error) {
+	const operation = "stats"
+	defer recordOperation(operation, time.Now())
+
+	out, err := in.client.GetContainerStats(id)
+	recordError(operation, err)
+	return out, err
+}

pkg/kubelet/dockershim/libdocker/kube_docker_client.go

@@ -123,6 +123,21 @@ func (d *kubeDockerClient) InspectContainer(id string) (*dockertypes.ContainerJS
 	return &containerJSON, nil
 }
 
+// InspectContainerWithSize is currently only used for Windows container stats
+func (d *kubeDockerClient) InspectContainerWithSize(id string) (*dockertypes.ContainerJSON, error) {
+	ctx, cancel := d.getTimeoutContext()
+	defer cancel()
+	// Inspects the container including the fields SizeRw and SizeRootFs.
+	containerJSON, _, err := d.client.ContainerInspectWithRaw(ctx, id, true)
+	if ctxErr := contextError(ctx); ctxErr != nil {
+		return nil, ctxErr
+	}
+	if err != nil {
+		return nil, err
+	}
+	return &containerJSON, nil
+}
+
 func (d *kubeDockerClient) CreateContainer(opts dockertypes.ContainerCreateConfig) (*dockercontainer.ContainerCreateCreatedBody, error) {
 	ctx, cancel := d.getTimeoutContext()
 	defer cancel()
@@ -522,6 +537,27 @@ func (d *kubeDockerClient) ResizeContainerTTY(id string, height, width uint) err
 	})
 }
 
+// GetContainerStats is currently only used for Windows container stats
+func (d *kubeDockerClient) GetContainerStats(id string) (*dockertypes.StatsJSON, error) {
+	ctx, cancel := d.getCancelableContext()
+	defer cancel()
+
+	response, err := d.client.ContainerStats(ctx, id, false)
+	if err != nil {
+		return nil, err
+	}
+
+	dec := json.NewDecoder(response.Body)
+	var stats dockertypes.StatsJSON
+	err = dec.Decode(&stats)
+	if err != nil {
+		return nil, err
+	}
+
+	defer response.Body.Close()
+	return &stats, nil
+}
+
 // redirectResponseToOutputStream redirect the response stream to stdout and stderr. When tty is true, all stream will
 // only be redirected to stdout.
 func (d *kubeDockerClient) redirectResponseToOutputStream(tty bool, outputStream, errorStream io.Writer, resp io.Reader) error {

pkg/kubelet/dockershim/remote/docker_service.go

@@ -17,7 +17,6 @@ limitations under the License.
 package remote
 
 import (
-	"fmt"
 	"time"
 
 	"golang.org/x/net/context"
@@ -226,13 +225,25 @@ func (d *dockerService) RemoveImage(ctx context.Context, r *runtimeapi.RemoveIma
 
 // ImageFsInfo returns information of the filesystem that is used to store images.
 func (d *dockerService) ImageFsInfo(ctx context.Context, r *runtimeapi.ImageFsInfoRequest) (*runtimeapi.ImageFsInfoResponse, error) {
-	return nil, fmt.Errorf("not implemented")
+	filesystems, err := d.imageService.ImageFsInfo()
+	if err != nil {
+		return nil, err
+	}
+	return &runtimeapi.ImageFsInfoResponse{ImageFilesystems: filesystems}, nil
 }
 
 func (d *dockerService) ContainerStats(ctx context.Context, r *runtimeapi.ContainerStatsRequest) (*runtimeapi.ContainerStatsResponse, error) {
-	return nil, fmt.Errorf("not implemented")
+	stats, err := d.runtimeService.ContainerStats(r.ContainerId)
+	if err != nil {
+		return nil, err
+	}
+	return &runtimeapi.ContainerStatsResponse{Stats: stats}, nil
 }
 
 func (d *dockerService) ListContainerStats(ctx context.Context, r *runtimeapi.ListContainerStatsRequest) (*runtimeapi.ListContainerStatsResponse, error) {
-	return nil, fmt.Errorf("not implemented")
+	stats, err := d.runtimeService.ListContainerStats(r.GetFilter())
+	if err != nil {
+		return nil, err
+	}
+	return &runtimeapi.ListContainerStatsResponse{Stats: stats}, nil
 }