Merge pull request #51152 from bobbypage/cri

Automatic merge from submit-queue (batch tested with PRs 50555, 51152). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Implement CRI stats in Docker Shim

**What this PR does / why we need it**:
This PR implements CRI Stats in the Docker Shim. It is needed to enable CRI stats for Docker and ongoing /stats/summary API changes in moving to use CRI.

Related issues:
#46984 (CRI: instruct kubelet to (optionally) consume container stats from CRI)
#45614 (CRI: add methods for container stats) 

This PR is also a followup to my original PR (https://github.com/kubernetes/kubernetes/pull/50396) to implement Windows Container Stats. The plan is that Windows Stats will use a hybrid model: pod and container level stats will come from CRI (via dockershim) and that node level stats will come from a "winstats" package that exports cadvisor like datastructures using windows specific perf counters from the node. I will update that PR to only export node level stats. 

@yujuhong @yguo0905 @dchen1107 @jdumars @anhowe @michmike

**Special notes for your reviewer**:

**Release note**:

```release-note
```

pkg/kubelet/dockershim/BUILD

@@ -15,9 +15,10 @@ go_library(
         "docker_checkpoint.go",
         "docker_container.go",
         "docker_image.go",
+        "docker_image_unsupported.go",
         "docker_sandbox.go",
         "docker_service.go",
-        "docker_stats.go",
+        "docker_stats_unsupported.go",
         "docker_streaming.go",
         "exec.go",
         "helpers.go",
@@ -27,9 +28,13 @@ go_library(
         "selinux_util.go",
     ] + select({
         "@io_bazel_rules_go//go/platform:linux_amd64": [
+            "docker_image_linux.go",
+            "docker_stats_linux.go",
             "helpers_linux.go",
         ],
         "@io_bazel_rules_go//go/platform:windows_amd64": [
+            "docker_image_windows.go",
+            "docker_stats_windows.go",
             "helpers_windows.go",
         ],
         "//conditions:default": [],

pkg/kubelet/dockershim/docker_image.go

@@ -133,11 +133,6 @@ func getImageRef(client libdocker.Interface, image string) (string, error) {
 	return img.ID, nil
 }
 
-// ImageFsInfo returns information of the filesystem that is used to store images.
-func (ds *dockerService) ImageFsInfo() ([]*runtimeapi.FilesystemUsage, error) {
-	return nil, fmt.Errorf("not implemented")
-}
-
 func filterHTTPError(err error, image string) error {
 	// docker/docker/pull/11314 prints detailed error info for docker pull.
 	// When it hits 502, it returns a verbose html output including an inline svg,

pkg/kubelet/dockershim/docker_image_linux.go

@@ -0,0 +1,30 @@
+// +build linux
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dockershim
+
+import (
+	"fmt"
+
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+)
+
+// ImageFsInfo returns information of the filesystem that is used to store images.
+func (ds *dockerService) ImageFsInfo() ([]*runtimeapi.FilesystemUsage, error) {
+	return nil, fmt.Errorf("not implemented")
+}

pkg/kubelet/dockershim/docker_image_unsupported.go

@@ -0,0 +1,30 @@
+// +build !linux,!windows
+
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dockershim
+
+import (
+	"fmt"
+
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+)
+
+// ImageFsInfo returns information of the filesystem that is used to store images.
+func (ds *dockerService) ImageFsInfo() ([]*runtimeapi.FilesystemUsage, error) {
+	return nil, fmt.Errorf("not implemented")
+}

pkg/kubelet/dockershim/docker_image_windows.go

@@ -0,0 +1,39 @@
+// +build windows
+
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dockershim
+
+import (
+	"time"
+
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+)
+
+// ImageFsInfo returns information of the filesystem that is used to store images.
+func (ds *dockerService) ImageFsInfo() ([]*runtimeapi.FilesystemUsage, error) {
+	// For Windows Stats to work correctly, a file system must be provided. For now, provide a fake filesystem.
+	filesystems := []*runtimeapi.FilesystemUsage{
+		{
+			Timestamp:  time.Now().UnixNano(),
+			UsedBytes:  &runtimeapi.UInt64Value{Value: 0},
+			InodesUsed: &runtimeapi.UInt64Value{Value: 0},
+		},
+	}
+
+	return filesystems, nil
+}

pkg/kubelet/dockershim/docker_stats_linux.go

@@ -1,3 +1,5 @@
+// +build linux
+
 /*
 Copyright 2017 The Kubernetes Authors.
 
@@ -18,15 +20,15 @@ package dockershim
 
 import (
 	"fmt"
-
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 )
 
-// DockerService does not implement container stats.
+// ContainerStats returns stats for a container stats request based on container id.
 func (ds *dockerService) ContainerStats(string) (*runtimeapi.ContainerStats, error) {
-	return nil, fmt.Errorf("Not implemented")
+	return nil, fmt.Errorf("not implemented")
 }
 
+// ListContainerStats returns stats for a list container stats request based on a filter.
 func (ds *dockerService) ListContainerStats(*runtimeapi.ContainerStatsFilter) ([]*runtimeapi.ContainerStats, error) {
-	return nil, fmt.Errorf("Not implemented")
+	return nil, fmt.Errorf("not implemented")
 }

pkg/kubelet/dockershim/docker_stats_unsupported.go

@@ -0,0 +1,35 @@
+// +build !linux,!windows
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dockershim
+
+import (
+	"fmt"
+
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+)
+
+// ContainerStats returns stats for a container stats request based on container id.
+func (ds *dockerService) ContainerStats(string) (*runtimeapi.ContainerStats, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+// ListContainerStats returns stats for a list container stats request based on a filter.
+func (ds *dockerService) ListContainerStats(*runtimeapi.ContainerStatsFilter) ([]*runtimeapi.ContainerStats, error) {
+	return nil, fmt.Errorf("not implemented")
+}

pkg/kubelet/dockershim/docker_stats_windows.go

@@ -0,0 +1,105 @@
+// +build windows
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dockershim
+
+import (
+	"time"
+
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+)
+
+// ContainerStats returns stats for a container stats request based on container id.
+func (ds *dockerService) ContainerStats(containerID string) (*runtimeapi.ContainerStats, error) {
+	containerStats, err := ds.getContainerStats(containerID)
+	if err != nil {
+		return nil, err
+	}
+	return containerStats, nil
+}
+
+// ListContainerStats returns stats for a list container stats request based on a filter.
+func (ds *dockerService) ListContainerStats(containerStatsFilter *runtimeapi.ContainerStatsFilter) ([]*runtimeapi.ContainerStats, error) {
+	filter := &runtimeapi.ContainerFilter{}
+
+	if containerStatsFilter != nil {
+		filter.Id = containerStatsFilter.Id
+		filter.PodSandboxId = containerStatsFilter.PodSandboxId
+		filter.LabelSelector = containerStatsFilter.LabelSelector
+	}
+
+	containers, err := ds.ListContainers(filter)
+	if err != nil {
+		return nil, err
+	}
+
+	var stats []*runtimeapi.ContainerStats
+	for _, container := range containers {
+		containerStats, err := ds.getContainerStats(container.Id)
+		if err != nil {
+			return nil, err
+		}
+
+		stats = append(stats, containerStats)
+	}
+
+	return stats, nil
+}
+
+func (ds *dockerService) getContainerStats(containerID string) (*runtimeapi.ContainerStats, error) {
+	statsJSON, err := ds.client.GetContainerStats(containerID)
+	if err != nil {
+		return nil, err
+	}
+
+	containerJSON, err := ds.client.InspectContainerWithSize(containerID)
+	if err != nil {
+		return nil, err
+	}
+
+	status, err := ds.ContainerStatus(containerID)
+	if err != nil {
+		return nil, err
+	}
+
+	dockerStats := statsJSON.Stats
+	timestamp := time.Now().UnixNano()
+	containerStats := &runtimeapi.ContainerStats{
+		Attributes: &runtimeapi.ContainerAttributes{
+			Id:          containerID,
+			Metadata:    status.Metadata,
+			Labels:      status.Labels,
+			Annotations: status.Annotations,
+		},
+		Cpu: &runtimeapi.CpuUsage{
+			Timestamp: timestamp,
+			// have to multiply cpu usage by 100 since docker stats units is in 100's of nano seconds for Windows
+			// see https://github.com/moby/moby/blob/v1.13.1/api/types/stats.go#L22
+			UsageCoreNanoSeconds: &runtimeapi.UInt64Value{Value: dockerStats.CPUStats.CPUUsage.TotalUsage * 100},
+		},
+		Memory: &runtimeapi.MemoryUsage{
+			Timestamp:       timestamp,
+			WorkingSetBytes: &runtimeapi.UInt64Value{Value: dockerStats.MemoryStats.PrivateWorkingSet},
+		},
+		WritableLayer: &runtimeapi.FilesystemUsage{
+			Timestamp: timestamp,
+			UsedBytes: &runtimeapi.UInt64Value{Value: uint64(*containerJSON.SizeRw)},
+		},
+	}
+	return containerStats, nil
+}

pkg/kubelet/dockershim/libdocker/client.go

@@ -46,6 +46,7 @@ const (
 type Interface interface {
 	ListContainers(options dockertypes.ContainerListOptions) ([]dockertypes.Container, error)
 	InspectContainer(id string) (*dockertypes.ContainerJSON, error)
+	InspectContainerWithSize(id string) (*dockertypes.ContainerJSON, error)
 	CreateContainer(dockertypes.ContainerCreateConfig) (*dockercontainer.ContainerCreateCreatedBody, error)
 	StartContainer(id string) error
 	StopContainer(id string, timeout time.Duration) error
@@ -66,6 +67,7 @@ type Interface interface {
 	AttachToContainer(string, dockertypes.ContainerAttachOptions, StreamOptions) error
 	ResizeContainerTTY(id string, height, width uint) error
 	ResizeExecTTY(id string, height, width uint) error
+	GetContainerStats(id string) (*dockertypes.StatsJSON, error)
 }
 
 // Get a *dockerapi.Client, either using the endpoint passed in, or using

pkg/kubelet/dockershim/libdocker/fake_client.go

@@ -461,6 +461,23 @@ func (f *FakeDockerClient) InspectContainer(id string) (*dockertypes.ContainerJS
 	return nil, fmt.Errorf("container %q not found", id)
 }
 
+// InspectContainerWithSize is a test-spy implementation of Interface.InspectContainerWithSize.
+// It adds an entry "inspect" to the internal method call record.
+func (f *FakeDockerClient) InspectContainerWithSize(id string) (*dockertypes.ContainerJSON, error) {
+	f.Lock()
+	defer f.Unlock()
+	f.appendCalled(calledDetail{name: "inspect_container_withsize"})
+	err := f.popError("inspect_container_withsize")
+	if container, ok := f.ContainerMap[id]; ok {
+		return container, err
+	}
+	if err != nil {
+		// Use the custom error if it exists.
+		return nil, err
+	}
+	return nil, fmt.Errorf("container %q not found", id)
+}
+
 // InspectImageByRef is a test-spy implementation of Interface.InspectImageByRef.
 // It adds an entry "inspect" to the internal method call record.
 func (f *FakeDockerClient) InspectImageByRef(name string) (*dockertypes.ImageInspect, error) {
@@ -852,3 +869,10 @@ func (f *FakeDockerPuller) GetImageRef(image string) (string, error) {
 	}
 	return image, err
 }
+
+func (f *FakeDockerClient) GetContainerStats(id string) (*dockertypes.StatsJSON, error) {
+	f.Lock()
+	defer f.Unlock()
+	f.appendCalled(calledDetail{name: "getContainerStats"})
+	return nil, fmt.Errorf("not implemented")
+}

pkg/kubelet/dockershim/libdocker/instrumented_client.go

@@ -74,6 +74,15 @@ func (in instrumentedInterface) InspectContainer(id string) (*dockertypes.Contai
 	return out, err
 }
 
+func (in instrumentedInterface) InspectContainerWithSize(id string) (*dockertypes.ContainerJSON, error) {
+	const operation = "inspect_container_withsize"
+	defer recordOperation(operation, time.Now())
+
+	out, err := in.client.InspectContainerWithSize(id)
+	recordError(operation, err)
+	return out, err
+}
+
 func (in instrumentedInterface) CreateContainer(opts dockertypes.ContainerCreateConfig) (*dockercontainer.ContainerCreateCreatedBody, error) {
 	const operation = "create_container"
 	defer recordOperation(operation, time.Now())
@@ -252,3 +261,12 @@ func (in instrumentedInterface) ResizeContainerTTY(id string, height, width uint
 	recordError(operation, err)
 	return err
 }
+
+func (in instrumentedInterface) GetContainerStats(id string) (*dockertypes.StatsJSON, error) {
+	const operation = "stats"
+	defer recordOperation(operation, time.Now())
+
+	out, err := in.client.GetContainerStats(id)
+	recordError(operation, err)
+	return out, err
+}

pkg/kubelet/dockershim/libdocker/kube_docker_client.go

@@ -123,6 +123,21 @@ func (d *kubeDockerClient) InspectContainer(id string) (*dockertypes.ContainerJS
 	return &containerJSON, nil
 }
 
+// InspectContainerWithSize is currently only used for Windows container stats
+func (d *kubeDockerClient) InspectContainerWithSize(id string) (*dockertypes.ContainerJSON, error) {
+	ctx, cancel := d.getTimeoutContext()
+	defer cancel()
+	// Inspects the container including the fields SizeRw and SizeRootFs.
+	containerJSON, _, err := d.client.ContainerInspectWithRaw(ctx, id, true)
+	if ctxErr := contextError(ctx); ctxErr != nil {
+		return nil, ctxErr
+	}
+	if err != nil {
+		return nil, err
+	}
+	return &containerJSON, nil
+}
+
 func (d *kubeDockerClient) CreateContainer(opts dockertypes.ContainerCreateConfig) (*dockercontainer.ContainerCreateCreatedBody, error) {
 	ctx, cancel := d.getTimeoutContext()
 	defer cancel()
@@ -522,6 +537,27 @@ func (d *kubeDockerClient) ResizeContainerTTY(id string, height, width uint) err
 	})
 }
 
+// GetContainerStats is currently only used for Windows container stats
+func (d *kubeDockerClient) GetContainerStats(id string) (*dockertypes.StatsJSON, error) {
+	ctx, cancel := d.getCancelableContext()
+	defer cancel()
+
+	response, err := d.client.ContainerStats(ctx, id, false)
+	if err != nil {
+		return nil, err
+	}
+
+	dec := json.NewDecoder(response.Body)
+	var stats dockertypes.StatsJSON
+	err = dec.Decode(&stats)
+	if err != nil {
+		return nil, err
+	}
+
+	defer response.Body.Close()
+	return &stats, nil
+}
+
 // redirectResponseToOutputStream redirect the response stream to stdout and stderr. When tty is true, all stream will
 // only be redirected to stdout.
 func (d *kubeDockerClient) redirectResponseToOutputStream(tty bool, outputStream, errorStream io.Writer, resp io.Reader) error {

pkg/kubelet/dockershim/remote/docker_service.go

@@ -17,7 +17,6 @@ limitations under the License.
 package remote
 
 import (
-	"fmt"
 	"time"
 
 	"golang.org/x/net/context"
@@ -226,13 +225,25 @@ func (d *dockerService) RemoveImage(ctx context.Context, r *runtimeapi.RemoveIma
 
 // ImageFsInfo returns information of the filesystem that is used to store images.
 func (d *dockerService) ImageFsInfo(ctx context.Context, r *runtimeapi.ImageFsInfoRequest) (*runtimeapi.ImageFsInfoResponse, error) {
-	return nil, fmt.Errorf("not implemented")
+	filesystems, err := d.imageService.ImageFsInfo()
+	if err != nil {
+		return nil, err
+	}
+	return &runtimeapi.ImageFsInfoResponse{ImageFilesystems: filesystems}, nil
 }
 
 func (d *dockerService) ContainerStats(ctx context.Context, r *runtimeapi.ContainerStatsRequest) (*runtimeapi.ContainerStatsResponse, error) {
-	return nil, fmt.Errorf("not implemented")
+	stats, err := d.runtimeService.ContainerStats(r.ContainerId)
+	if err != nil {
+		return nil, err
+	}
+	return &runtimeapi.ContainerStatsResponse{Stats: stats}, nil
 }
 
 func (d *dockerService) ListContainerStats(ctx context.Context, r *runtimeapi.ListContainerStatsRequest) (*runtimeapi.ListContainerStatsResponse, error) {
-	return nil, fmt.Errorf("not implemented")
+	stats, err := d.runtimeService.ListContainerStats(r.GetFilter())
+	if err != nil {
+		return nil, err
+	}
+	return &runtimeapi.ListContainerStatsResponse{Stats: stats}, nil
 }