diff --git a/src/runtime/virtcontainers/acrn.go b/src/runtime/virtcontainers/acrn.go index 93359e16ad..f5f025a346 100644 --- a/src/runtime/virtcontainers/acrn.go +++ b/src/runtime/virtcontainers/acrn.go @@ -679,6 +679,10 @@ func (a *Acrn) getPids() []int { return []int{a.state.PID} } +func (a *Acrn) getVirtioFsPid() *int { + return nil +} + func (a *Acrn) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, j []byte) error { return errors.New("acrn is not supported by VM cache") } diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 6295031cdf..4b87863dd9 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -712,6 +712,10 @@ func (clh *cloudHypervisor) getPids() []int { return pids } +func (clh *cloudHypervisor) getVirtioFsPid() *int { + return &clh.state.VirtiofsdPID +} + func (clh *cloudHypervisor) addDevice(ctx context.Context, devInfo interface{}, devType deviceType) error { span, _ := clh.trace(ctx, "addDevice") defer span.End() diff --git a/src/runtime/virtcontainers/fc.go b/src/runtime/virtcontainers/fc.go index 7dac0e78a0..6d86cd61af 100644 --- a/src/runtime/virtcontainers/fc.go +++ b/src/runtime/virtcontainers/fc.go @@ -1181,6 +1181,10 @@ func (fc *firecracker) getPids() []int { return []int{fc.info.PID} } +func (fc *firecracker) getVirtioFsPid() *int { + return nil +} + func (fc *firecracker) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, j []byte) error { return errors.New("firecracker is not supported by VM cache") } diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 0b8cac7761..73a17cdd42 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -818,6 +818,7 @@ type hypervisor interface { // getPids returns a slice of hypervisor related process ids. // The hypervisor pid must be put at index 0. getPids() []int + getVirtioFsPid() *int fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, j []byte) error toGrpc(ctx context.Context) ([]byte, error) check() error diff --git a/src/runtime/virtcontainers/mock_hypervisor.go b/src/runtime/virtcontainers/mock_hypervisor.go index af19130f40..18487aabd8 100644 --- a/src/runtime/virtcontainers/mock_hypervisor.go +++ b/src/runtime/virtcontainers/mock_hypervisor.go @@ -109,6 +109,10 @@ func (m *mockHypervisor) getPids() []int { return []int{m.mockPid} } +func (m *mockHypervisor) getVirtioFsPid() *int { + return nil +} + func (m *mockHypervisor) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, j []byte) error { return errors.New("mockHypervisor is not supported by VM cache") } diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index cf1c4377ee..02a1f4bc2e 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -2329,6 +2329,10 @@ func (q *qemu) getPids() []int { return pids } +func (q *qemu) getVirtioFsPid() *int { + return &q.state.VirtiofsdPid +} + type qemuGrpc struct { ID string QmpChannelpath string diff --git a/src/runtime/virtcontainers/sandbox_metrics.go b/src/runtime/virtcontainers/sandbox_metrics.go index 49e115cfcb..ade8b20b98 100644 --- a/src/runtime/virtcontainers/sandbox_metrics.go +++ b/src/runtime/virtcontainers/sandbox_metrics.go @@ -16,8 +16,10 @@ import ( const namespaceHypervisor = "kata_hypervisor" const namespaceKatashim = "kata_shim" +const namespaceVirtiofsd = "kata_virtiofsd" var ( + // hypervisor hypervisorThreads = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespaceHypervisor, Name: "threads", @@ -62,6 +64,7 @@ var ( Help: "Open FDs for hypervisor.", }) + // agent agentRPCDurationsHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespaceKatashim, Name: "agent_rpc_durations_histogram_milliseconds", @@ -70,16 +73,61 @@ var ( }, []string{"action"}, ) + + // virtiofsd + virtiofsdThreads = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespaceVirtiofsd, + Name: "threads", + Help: "Virtiofsd process threads.", + }) + + virtiofsdProcStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespaceVirtiofsd, + Name: "proc_status", + Help: "Virtiofsd process status.", + }, + []string{"item"}, + ) + + virtiofsdProcStat = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespaceVirtiofsd, + Name: "proc_stat", + Help: "Virtiofsd process statistics.", + }, + []string{"item"}, + ) + + virtiofsdIOStat = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespaceVirtiofsd, + Name: "io_stat", + Help: "Process IO statistics.", + }, + []string{"item"}, + ) + + virtiofsdOpenFDs = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespaceVirtiofsd, + Name: "fds", + Help: "Open FDs for virtiofsd.", + }) ) func RegisterMetrics() { + // hypervisor prometheus.MustRegister(hypervisorThreads) prometheus.MustRegister(hypervisorProcStatus) prometheus.MustRegister(hypervisorProcStat) prometheus.MustRegister(hypervisorNetdev) prometheus.MustRegister(hypervisorIOStat) prometheus.MustRegister(hypervisorOpenFDs) + // agent prometheus.MustRegister(agentRPCDurationsHistogram) + // virtiofsd + prometheus.MustRegister(virtiofsdThreads) + prometheus.MustRegister(virtiofsdProcStatus) + prometheus.MustRegister(virtiofsdProcStat) + prometheus.MustRegister(virtiofsdIOStat) + prometheus.MustRegister(virtiofsdOpenFDs) } // UpdateRuntimeMetrics update shim/hypervisor's metrics @@ -125,6 +173,48 @@ func (s *Sandbox) UpdateRuntimeMetrics() error { mutils.SetGaugeVecProcIO(hypervisorIOStat, ioStat) } + // virtiofs metrics + err = s.UpdateVirtiofsdMetrics() + if err != nil { + return err + } + + return nil +} + +func (s *Sandbox) UpdateVirtiofsdMetrics() error { + vfsPid := s.hypervisor.getVirtioFsPid() + if vfsPid == nil { + // virtiofsd is not mandatory for a VMM. + return nil + } + + proc, err := procfs.NewProc(*vfsPid) + if err != nil { + return err + } + + // process FDs + if fds, err := proc.FileDescriptorsLen(); err == nil { + virtiofsdOpenFDs.Set(float64(fds)) + } + + // process statistics + if procStat, err := proc.Stat(); err == nil { + virtiofsdThreads.Set(float64(procStat.NumThreads)) + mutils.SetGaugeVecProcStat(virtiofsdProcStat, procStat) + } + + // process status + if procStatus, err := proc.NewStatus(); err == nil { + mutils.SetGaugeVecProcStatus(virtiofsdProcStatus, procStatus) + } + + // process IO statistics + if ioStat, err := proc.IO(); err == nil { + mutils.SetGaugeVecProcIO(virtiofsdIOStat, ioStat) + } + return nil }