diff --git a/pkg/kubelet/stats/BUILD b/pkg/kubelet/stats/BUILD index ca3efe32f00..ac9d306d2f6 100644 --- a/pkg/kubelet/stats/BUILD +++ b/pkg/kubelet/stats/BUILD @@ -5,6 +5,8 @@ go_library( srcs = [ "cadvisor_stats_provider.go", "cri_stats_provider.go", + "cri_stats_provider_unsupported.go", + "cri_stats_provider_windows.go", "helper.go", "log_metrics_provider.go", "stats_provider.go", @@ -33,7 +35,12 @@ go_library( "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/info/v2:go_default_library", "//vendor/k8s.io/klog:go_default_library", - ], + ] + select({ + "@io_bazel_rules_go//go/platform:windows": [ + "//vendor/github.com/Microsoft/hcsshim:go_default_library", + ], + "//conditions:default": [], + }), ) filegroup( diff --git a/pkg/kubelet/stats/cri_stats_provider.go b/pkg/kubelet/stats/cri_stats_provider.go index bafd24d65c4..b6eb43fd7ec 100644 --- a/pkg/kubelet/stats/cri_stats_provider.go +++ b/pkg/kubelet/stats/cri_stats_provider.go @@ -25,11 +25,10 @@ import ( "time" cadvisorfs "github.com/google/cadvisor/fs" - "k8s.io/klog" - cadvisorapiv2 "github.com/google/cadvisor/info/v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" @@ -124,6 +123,12 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { } caInfos := getCRICadvisorStats(allInfos) + // get network stats for containers. + containerNetworkStats, err := p.listContainerNetworkStats() + if err != nil { + return nil, fmt.Errorf("failed to list container network stats: %v", err) + } + for _, stats := range resp { containerID := stats.Attributes.Id container, found := containerMap[containerID] @@ -147,7 +152,7 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { // Fill available stats for full set of required pod stats cs := p.makeContainerStats(stats, container, &rootFsInfo, fsIDtoInfo, podSandbox.GetMetadata().GetUid()) - p.addPodNetworkStats(ps, podSandboxID, caInfos, cs) + p.addPodNetworkStats(ps, podSandboxID, caInfos, cs, containerNetworkStats[podSandboxID]) p.addPodCPUMemoryStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos, cs) // If cadvisor stats is available for the container, use it to populate @@ -353,11 +358,21 @@ func (p *criStatsProvider) addPodNetworkStats( podSandboxID string, caInfos map[string]cadvisorapiv2.ContainerInfo, cs *statsapi.ContainerStats, + netStats *statsapi.NetworkStats, ) { caPodSandbox, found := caInfos[podSandboxID] // try get network stats from cadvisor first. if found { - ps.Network = cadvisorInfoToNetworkStats(ps.PodRef.Name, &caPodSandbox) + networkStats := cadvisorInfoToNetworkStats(ps.PodRef.Name, &caPodSandbox) + if networkStats != nil { + ps.Network = networkStats + return + } + } + + // Not found from cadvisor, get from netStats. + if netStats != nil { + ps.Network = netStats return } diff --git a/pkg/kubelet/stats/cri_stats_provider_unsupported.go b/pkg/kubelet/stats/cri_stats_provider_unsupported.go new file mode 100644 index 00000000000..be08a241ab9 --- /dev/null +++ b/pkg/kubelet/stats/cri_stats_provider_unsupported.go @@ -0,0 +1,29 @@ +// +build !windows + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stats + +import ( + statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" +) + +// listContainerNetworkStats returns the network stats of all the running containers. +func (p *criStatsProvider) listContainerNetworkStats() (map[string]*statsapi.NetworkStats, error) { + // Always return nil for unsupported platforms. + return nil, nil +} diff --git a/pkg/kubelet/stats/cri_stats_provider_windows.go b/pkg/kubelet/stats/cri_stats_provider_windows.go new file mode 100644 index 00000000000..bbc45647424 --- /dev/null +++ b/pkg/kubelet/stats/cri_stats_provider_windows.go @@ -0,0 +1,91 @@ +// +build windows + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stats + +import ( + "time" + + "github.com/Microsoft/hcsshim" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" + statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" +) + +// listContainerNetworkStats returns the network stats of all the running containers. +func (p *criStatsProvider) listContainerNetworkStats() (map[string]*statsapi.NetworkStats, error) { + containers, err := hcsshim.GetContainers(hcsshim.ComputeSystemQuery{ + Types: []string{"Container"}, + }) + if err != nil { + return nil, err + } + + stats := make(map[string]*statsapi.NetworkStats) + for _, c := range containers { + container, err := hcsshim.OpenContainer(c.ID) + if err != nil { + klog.Warningf("Failed to open container %q with error '%v', continue to get stats for other containers", c.ID, err) + continue + } + + cstats, err := container.Statistics() + if err != nil { + klog.Warningf("Failed to get statistics for container %q with error '%v', continue to get stats for other containers", c.ID, err) + continue + } + + if len(cstats.Network) > 0 { + stats[c.ID] = hcsStatsToNetworkStats(cstats.Timestamp, cstats.Network) + } + } + + return stats, nil +} + +// hcsStatsToNetworkStats converts hcsshim.Statistics.Network to statsapi.NetworkStats +func hcsStatsToNetworkStats(timestamp time.Time, hcsStats []hcsshim.NetworkStats) *statsapi.NetworkStats { + result := &statsapi.NetworkStats{ + Time: metav1.NewTime(timestamp), + Interfaces: make([]statsapi.InterfaceStats, 0), + } + + for _, stat := range hcsStats { + iStat := hcsStatsToInterfaceStats(stat) + if iStat != nil { + result.Interfaces = append(result.Interfaces, *iStat) + } + } + + // TODO(feiskyer): add support of multiple interfaces for getting default interface. + if len(result.Interfaces) > 0 { + result.InterfaceStats = result.Interfaces[0] + } + + return result +} + +// hcsStatsToInterfaceStats converts hcsshim.NetworkStats to statsapi.InterfaceStats. +func hcsStatsToInterfaceStats(stat hcsshim.NetworkStats) *statsapi.InterfaceStats { + return &statsapi.InterfaceStats{ + Name: stat.EndpointId, + RxBytes: &stat.BytesReceived, + TxBytes: &stat.BytesSent, + } +} diff --git a/pkg/kubelet/stats/helper.go b/pkg/kubelet/stats/helper.go index 54f3093e553..024f8981d8d 100644 --- a/pkg/kubelet/stats/helper.go +++ b/pkg/kubelet/stats/helper.go @@ -20,11 +20,10 @@ import ( "fmt" "time" - "k8s.io/klog" - cadvisorapiv1 "github.com/google/cadvisor/info/v1" cadvisorapiv2 "github.com/google/cadvisor/info/v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cadvisor" ) @@ -158,6 +157,10 @@ func cadvisorInfoToNetworkStats(name string, info *cadvisorapiv2.ContainerInfo) return nil } + if cstat.Network == nil { + return nil + } + iStats := statsapi.NetworkStats{ Time: metav1.NewTime(cstat.Timestamp), } diff --git a/pkg/kubelet/winstats/BUILD b/pkg/kubelet/winstats/BUILD index 25afdd4cfc0..503e8d7297c 100644 --- a/pkg/kubelet/winstats/BUILD +++ b/pkg/kubelet/winstats/BUILD @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "doc.go", + "network_stats.go", "perfcounter_nodestats.go", "perfcounters.go", "version.go", diff --git a/pkg/kubelet/winstats/network_stats.go b/pkg/kubelet/winstats/network_stats.go new file mode 100644 index 00000000000..bfd80bc50d9 --- /dev/null +++ b/pkg/kubelet/winstats/network_stats.go @@ -0,0 +1,67 @@ +// +build windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package winstats + +import ( + "encoding/json" + "os/exec" + + cadvisorapi "github.com/google/cadvisor/info/v1" +) + +// netAdapterStat represents network statistics for an adapter. +type netAdapterStat struct { + Name string `json:"Name,omitempty"` + ReceivedBytes uint64 `json:"ReceivedBytes,omitempty"` + ReceivedErrors uint64 `json:"ReceivedPacketErrors,omitempty"` + SentBytes uint64 `json:"SentBytes,omitempty"` + SentErrors uint64 `json:"OutboundPacketErrors,omitempty"` +} + +// toCadvisorInterfaceStats converts netAdapterStat to cadvisorapi.InterfaceStats. +func (s *netAdapterStat) toCadvisorInterfaceStats() cadvisorapi.InterfaceStats { + return cadvisorapi.InterfaceStats{ + Name: s.Name, + RxBytes: s.ReceivedBytes, + RxErrors: s.ReceivedErrors, + TxBytes: s.SentBytes, + TxErrors: s.SentErrors, + } +} + +// getNetAdapterStats gets a list of network adapter statistics. +func getNetAdapterStats() ([]cadvisorapi.InterfaceStats, error) { + rawOutput, err := exec.Command("powershell", "/c", " Get-NetAdapterStatistics | ConvertTo-Json").CombinedOutput() + if err != nil { + return nil, err + } + + var stats []*netAdapterStat + err = json.Unmarshal(rawOutput, &stats) + if err != nil { + return nil, err + } + + result := make([]cadvisorapi.InterfaceStats, len(stats)) + for i := range stats { + result[i] = stats[i].toCadvisorInterfaceStats() + } + + return result, nil +} diff --git a/pkg/kubelet/winstats/winstats.go b/pkg/kubelet/winstats/winstats.go index 2c483f8967e..5a194429c5d 100644 --- a/pkg/kubelet/winstats/winstats.go +++ b/pkg/kubelet/winstats/winstats.go @@ -109,12 +109,16 @@ func (c *StatsClient) WinVersionInfo() (*cadvisorapi.VersionInfo, error) { func (c *StatsClient) createRootContainerInfo() (*cadvisorapiv2.ContainerInfo, error) { nodeMetrics, err := c.client.getNodeMetrics() - if err != nil { return nil, err } - var stats []*cadvisorapiv2.ContainerStats + netAdapterStats, err := getNetAdapterStats() + if err != nil { + return nil, err + } + + var stats []*cadvisorapiv2.ContainerStats stats = append(stats, &cadvisorapiv2.ContainerStats{ Timestamp: nodeMetrics.timeStamp, Cpu: &cadvisorapi.CpuStats{ @@ -126,6 +130,9 @@ func (c *StatsClient) createRootContainerInfo() (*cadvisorapiv2.ContainerInfo, e WorkingSet: nodeMetrics.memoryPrivWorkingSetBytes, Usage: nodeMetrics.memoryCommittedBytes, }, + Network: &cadvisorapiv2.NetworkStats{ + Interfaces: netAdapterStats, + }, }) nodeInfo := c.client.getNodeInfo() @@ -134,6 +141,7 @@ func (c *StatsClient) createRootContainerInfo() (*cadvisorapiv2.ContainerInfo, e CreationTime: nodeInfo.startTime, HasCpu: true, HasMemory: true, + HasNetwork: true, Memory: cadvisorapiv2.MemorySpec{ Limit: nodeInfo.memoryPhysicalCapacityBytes, }, diff --git a/pkg/kubelet/winstats/winstats_test.go b/pkg/kubelet/winstats/winstats_test.go index 830f76af039..51f7a290d19 100644 --- a/pkg/kubelet/winstats/winstats_test.go +++ b/pkg/kubelet/winstats/winstats_test.go @@ -86,8 +86,9 @@ func TestWinContainerInfos(t *testing.T) { infos := make(map[string]cadvisorapiv2.ContainerInfo) infos["/"] = cadvisorapiv2.ContainerInfo{ Spec: cadvisorapiv2.ContainerSpec{ - HasCpu: true, - HasMemory: true, + HasCpu: true, + HasMemory: true, + HasNetwork: true, Memory: cadvisorapiv2.MemorySpec{ Limit: 1.6e+10, }, @@ -95,7 +96,11 @@ func TestWinContainerInfos(t *testing.T) { Stats: stats, } - assert.Equal(t, actualRootInfos, infos) + assert.Equal(t, len(actualRootInfos), len(infos)) + assert.Equal(t, actualRootInfos["/"].Spec, infos["/"].Spec) + assert.Equal(t, len(actualRootInfos["/"].Stats), len(infos["/"].Stats)) + assert.Equal(t, actualRootInfos["/"].Stats[0].Cpu, infos["/"].Stats[0].Cpu) + assert.Equal(t, actualRootInfos["/"].Stats[0].Memory, infos["/"].Stats[0].Memory) } func TestWinMachineInfo(t *testing.T) {