github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-06-29 00:37:24 +00:00
Code Issues Projects Releases Wiki Activity

tracing: Consolidate tracing into a new katatrace package

Browse Source 
Removes custom trace functions defined across the repo and creates
a single trace function in a new katatrace package. Also moves
span tag management into this package and provides a function to
dynamically add a tag at runtime, such as a container id, etc.

Fixes #1162

Signed-off-by: Benjamin Porter <bporter816@gmail.com>
This commit is contained in:
Benjamin Porter 2021-07-03 17:37:22 -05:00
parent c8f32936d3
commit b10e3e22b5
No known key found for this signature in database
GPG Key ID: 750EA3181FCE490F
18 changed files with 348 additions and 339 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/runtime/containerd-shim-v2/create.go
View File

@ -19,7 +19,6 @@ import (
"github.com/containerd/typeurl"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/pkg/errors"
otelTrace "go.opentelemetry.io/otel/trace"
// only register the proto type
crioption "github.com/containerd/containerd/pkg/runtimeoptions/v1"
@ -27,6 +26,7 @@ import (
_ "github.com/containerd/containerd/runtime/v2/runc/options"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
@ -69,19 +69,24 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
// create tracer
// This is the earliest location we can create the tracer because we must wait
// until the runtime config is loaded
_, err = katautils.CreateTracer("kata", s.config)
jaegerConfig := &katatrace.JaegerConfig{
JaegerEndpoint: s.config.JaegerEndpoint,
JaegerUser: s.config.JaegerUser,
JaegerPassword: s.config.JaegerPassword,
}
_, err = katatrace.CreateTracer("kata", jaegerConfig)
if err != nil {
return nil, err
}
// create root span
var rootSpan otelTrace.Span
rootSpan, s.rootCtx = trace(s.ctx, "root span")
rootSpan, newCtx := katatrace.Trace(s.ctx, shimLog, "root span", shimTracingTags)
s.rootCtx = newCtx
defer rootSpan.End()
// create span
var span otelTrace.Span
span, s.ctx = trace(s.rootCtx, "create")
span, newCtx := katatrace.Trace(s.rootCtx, shimLog, "create", shimTracingTags)
s.ctx = newCtx
defer span.End()
if rootFs.Mounted, err = checkAndMount(s, r); err != nil {
@ -116,8 +121,7 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
go s.startManagementServer(ctx, ociSpec)
case vc.PodContainer:
var span otelTrace.Span
span, ctx = trace(s.ctx, "create")
span, ctx := katatrace.Trace(s.ctx, shimLog, "create", shimTracingTags)
defer span.End()
if s.sandbox == nil {

57
src/runtime/containerd-shim-v2/service.go
View File

@ -27,18 +27,22 @@ import (
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/label"
otelTrace "go.opentelemetry.io/otel/trace"
"golang.org/x/sys/unix"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
)
// shimTracingTags defines tags for the trace span
var shimTracingTags = map[string]string{
"source": "runtime",
"package": "containerdshim",
}
const (
// Define the service's channel size, which is used for
// reaping the exited processes exit state and forwarding
@ -301,19 +305,8 @@ func getTopic(e interface{}) string {
return cdruntime.TaskUnknownTopic
}
func trace(ctx context.Context, name string) (otelTrace.Span, context.Context) {
if ctx == nil {
logrus.WithFields(logrus.Fields{"type": "bug", "name": name}).Error("called before context set")
ctx = context.Background()
}
tracer := otel.Tracer("kata")
ctx, span := tracer.Start(ctx, name, otelTrace.WithAttributes(label.String("source", "runtime"), label.String("package", "containerdshim")))
return span, ctx
}
func (s *service) Cleanup(ctx context.Context) (_ *taskAPI.DeleteResponse, err error) {
span, spanCtx := trace(s.rootCtx, "Cleanup")
span, spanCtx := katatrace.Trace(s.rootCtx, shimLog, "Cleanup", shimTracingTags)
defer span.End()
//Since the binary cleanup will return the DeleteResponse from stdout to
@ -428,7 +421,7 @@ func (s *service) Create(ctx context.Context, r *taskAPI.CreateTaskRequest) (_ *
// Start a process
func (s *service) Start(ctx context.Context, r *taskAPI.StartRequest) (_ *taskAPI.StartResponse, err error) {
span, spanCtx := trace(s.rootCtx, "Start")
span, spanCtx := katatrace.Trace(s.rootCtx, shimLog, "Start", shimTracingTags)
defer span.End()
start := time.Now()
@ -479,7 +472,7 @@ func (s *service) Start(ctx context.Context, r *taskAPI.StartRequest) (_ *taskAP
// Delete the initial process and container
func (s *service) Delete(ctx context.Context, r *taskAPI.DeleteRequest) (_ *taskAPI.DeleteResponse, err error) {
span, spanCtx := trace(s.rootCtx, "Delete")
span, spanCtx := katatrace.Trace(s.rootCtx, shimLog, "Delete", shimTracingTags)
defer span.End()
start := time.Now()
@ -531,7 +524,7 @@ func (s *service) Delete(ctx context.Context, r *taskAPI.DeleteRequest) (_ *task
// Exec an additional process inside the container
func (s *service) Exec(ctx context.Context, r *taskAPI.ExecProcessRequest) (_ *ptypes.Empty, err error) {
span, _ := trace(s.rootCtx, "Exec")
span, _ := katatrace.Trace(s.rootCtx, shimLog, "Exec", shimTracingTags)
defer span.End()
start := time.Now()
@ -569,7 +562,7 @@ func (s *service) Exec(ctx context.Context, r *taskAPI.ExecProcessRequest) (_ *p
// ResizePty of a process
func (s *service) ResizePty(ctx context.Context, r *taskAPI.ResizePtyRequest) (_ *ptypes.Empty, err error) {
span, spanCtx := trace(s.rootCtx, "ResizePty")
span, spanCtx := katatrace.Trace(s.rootCtx, shimLog, "ResizePty", shimTracingTags)
defer span.End()
start := time.Now()
@ -608,7 +601,7 @@ func (s *service) ResizePty(ctx context.Context, r *taskAPI.ResizePtyRequest) (_
// State returns runtime state information for a process
func (s *service) State(ctx context.Context, r *taskAPI.StateRequest) (_ *taskAPI.StateResponse, err error) {
span, _ := trace(s.rootCtx, "State")
span, _ := katatrace.Trace(s.rootCtx, shimLog, "State", shimTracingTags)
defer span.End()
start := time.Now()
@ -662,7 +655,7 @@ func (s *service) State(ctx context.Context, r *taskAPI.StateRequest) (_ *taskAP
// Pause the container
func (s *service) Pause(ctx context.Context, r *taskAPI.PauseRequest) (_ *ptypes.Empty, err error) {
span, spanCtx := trace(s.rootCtx, "Pause")
span, spanCtx := katatrace.Trace(s.rootCtx, shimLog, "Pause", shimTracingTags)
defer span.End()
start := time.Now()
@ -701,7 +694,7 @@ func (s *service) Pause(ctx context.Context, r *taskAPI.PauseRequest) (_ *ptypes
// Resume the container
func (s *service) Resume(ctx context.Context, r *taskAPI.ResumeRequest) (_ *ptypes.Empty, err error) {
span, spanCtx := trace(s.rootCtx, "Resume")
span, spanCtx := katatrace.Trace(s.rootCtx, shimLog, "Resume", shimTracingTags)
defer span.End()
start := time.Now()
@ -738,7 +731,7 @@ func (s *service) Resume(ctx context.Context, r *taskAPI.ResumeRequest) (_ *ptyp
// Kill a process with the provided signal
func (s *service) Kill(ctx context.Context, r *taskAPI.KillRequest) (_ *ptypes.Empty, err error) {
span, spanCtx := trace(s.rootCtx, "Kill")
span, spanCtx := katatrace.Trace(s.rootCtx, shimLog, "Kill", shimTracingTags)
defer span.End()
start := time.Now()
@ -799,7 +792,7 @@ func (s *service) Kill(ctx context.Context, r *taskAPI.KillRequest) (_ *ptypes.E
// Since for kata, it cannot get the process's pid from VM,
// thus only return the Shim's pid directly.
func (s *service) Pids(ctx context.Context, r *taskAPI.PidsRequest) (_ *taskAPI.PidsResponse, err error) {
span, _ := trace(s.rootCtx, "Pids")
span, _ := katatrace.Trace(s.rootCtx, shimLog, "Pids", shimTracingTags)
defer span.End()
var processes []*task.ProcessInfo
@ -822,7 +815,7 @@ func (s *service) Pids(ctx context.Context, r *taskAPI.PidsRequest) (_ *taskAPI.
// CloseIO of a process
func (s *service) CloseIO(ctx context.Context, r *taskAPI.CloseIORequest) (_ *ptypes.Empty, err error) {
span, _ := trace(s.rootCtx, "CloseIO")
span, _ := katatrace.Trace(s.rootCtx, shimLog, "CloseIO", shimTracingTags)
defer span.End()
start := time.Now()
@ -863,7 +856,7 @@ func (s *service) CloseIO(ctx context.Context, r *taskAPI.CloseIORequest) (_ *pt
// Checkpoint the container
func (s *service) Checkpoint(ctx context.Context, r *taskAPI.CheckpointTaskRequest) (_ *ptypes.Empty, err error) {
span, _ := trace(s.rootCtx, "Checkpoint")
span, _ := katatrace.Trace(s.rootCtx, shimLog, "Checkpoint", shimTracingTags)
defer span.End()
start := time.Now()
@ -877,7 +870,7 @@ func (s *service) Checkpoint(ctx context.Context, r *taskAPI.CheckpointTaskReque
// Connect returns shim information such as the shim's pid
func (s *service) Connect(ctx context.Context, r *taskAPI.ConnectRequest) (_ *taskAPI.ConnectResponse, err error) {
span, _ := trace(s.rootCtx, "Connect")
span, _ := katatrace.Trace(s.rootCtx, shimLog, "Connect", shimTracingTags)
defer span.End()
start := time.Now()
@ -897,7 +890,7 @@ func (s *service) Connect(ctx context.Context, r *taskAPI.ConnectRequest) (_ *ta
}
func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (_ *ptypes.Empty, err error) {
span, _ := trace(s.rootCtx, "Shutdown")
span, _ := katatrace.Trace(s.rootCtx, shimLog, "Shutdown", shimTracingTags)
start := time.Now()
defer func() {
@ -913,7 +906,7 @@ func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (_ *
s.mu.Unlock()
span.End()
katautils.StopTracing(s.ctx)
katatrace.StopTracing(s.ctx)
s.cancel()
@ -930,7 +923,7 @@ func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (_ *
}
func (s *service) Stats(ctx context.Context, r *taskAPI.StatsRequest) (_ *taskAPI.StatsResponse, err error) {
span, spanCtx := trace(s.rootCtx, "Stats")
span, spanCtx := katatrace.Trace(s.rootCtx, shimLog, "Stats", shimTracingTags)
defer span.End()
start := time.Now()
@ -959,7 +952,7 @@ func (s *service) Stats(ctx context.Context, r *taskAPI.StatsRequest) (_ *taskAP
// Update a running container
func (s *service) Update(ctx context.Context, r *taskAPI.UpdateTaskRequest) (_ *ptypes.Empty, err error) {
span, spanCtx := trace(s.rootCtx, "Update")
span, spanCtx := katatrace.Trace(s.rootCtx, shimLog, "Update", shimTracingTags)
defer span.End()
start := time.Now()
@ -991,7 +984,7 @@ func (s *service) Update(ctx context.Context, r *taskAPI.UpdateTaskRequest) (_ *
// Wait for a process to exit
func (s *service) Wait(ctx context.Context, r *taskAPI.WaitRequest) (_ *taskAPI.WaitResponse, err error) {
span, _ := trace(s.rootCtx, "Wait")
span, _ := katatrace.Trace(s.rootCtx, shimLog, "Wait", shimTracingTags)
defer span.End()
var ret uint32

8
src/runtime/pkg/katautils/config.go
View File

@ -16,6 +16,7 @@ import (
"github.com/BurntSushi/toml"
govmmQemu "github.com/kata-containers/govmm/qemu"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
@ -28,11 +29,6 @@ const (
defaultHypervisor = vc.QemuHypervisor
)
var (
// if true, enable opentracing support.
tracing = false
)
// The TOML configuration file contains a number of sections (or
// tables). The names of these tables are in dotted ("nested table")
// form:
@ -1116,7 +1112,7 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
}
config.Trace = tomlConf.Runtime.Tracing
tracing = config.Trace
katatrace.SetTracing(config.Trace)
if tomlConf.Runtime.InterNetworkModel != "" {
err = config.InterNetworkModel.SetModel(tomlConf.Runtime.InterNetworkModel)

19
src/runtime/pkg/katautils/create.go
View File

@ -13,13 +13,20 @@ import (
"strconv"
"strings"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
vf "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/factory"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
specs "github.com/opencontainers/runtime-spec/specs-go"
"go.opentelemetry.io/otel/label"
)
// createTracingTags defines tags for the trace span
var createTracingTags = map[string]string{
"source": "runtime",
"package": "katautils",
"subsystem": "sandbox",
}
// GetKernelParamsFunc use a variable to allow tests to modify its value
var GetKernelParamsFunc = getKernelParams
@ -104,7 +111,8 @@ func SetEphemeralStorageType(ociSpec specs.Spec) specs.Spec {
// CreateSandbox create a sandbox container
func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeConfig oci.RuntimeConfig, rootFs vc.RootFs,
containerID, bundlePath, console string, disableOutput, systemdCgroup bool) (_ vc.VCSandbox, _ vc.Process, err error) {
span, ctx := Trace(ctx, "CreateSandbox", []label.KeyValue{label.Key("source").String("runtime"), label.Key("package").String("katautils"), label.Key("subsystem").String("sandbox"), label.Key("container_id").String(containerID)}...)
span, ctx := katatrace.Trace(ctx, nil, "CreateSandbox", createTracingTags)
katatrace.AddTag(span, "container_id", containerID)
defer span.End()
sandboxConfig, err := oci.SandboxConfig(ociSpec, runtimeConfig, bundlePath, containerID, console, disableOutput, systemdCgroup)
@ -159,7 +167,7 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeCo
sid := sandbox.ID()
kataUtilsLogger = kataUtilsLogger.WithField("sandbox", sid)
span.SetAttributes(label.Key("sandbox_id").String(sid))
katatrace.AddTag(span, "sandbox_id", sid)
containers := sandbox.GetAllContainers()
if len(containers) != 1 {
@ -202,7 +210,8 @@ func checkForFIPS(sandboxConfig *vc.SandboxConfig) error {
func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Spec, rootFs vc.RootFs, containerID, bundlePath, console string, disableOutput bool) (vc.Process, error) {
var c vc.VCContainer
span, ctx := Trace(ctx, "CreateContainer", []label.KeyValue{label.Key("source").String("runtime"), label.Key("package").String("katautils"), label.Key("subsystem").String("sandbox"), label.Key("container_id").String(containerID)}...)
span, ctx := katatrace.Trace(ctx, nil, "CreateContainer", createTracingTags)
katatrace.AddTag(span, "container_id", containerID)
defer span.End()
ociSpec = SetEphemeralStorageType(ociSpec)
@ -228,7 +237,7 @@ func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Sp
return vc.Process{}, err
}
span.SetAttributes(label.Key("sandbox_id").String(sandboxID))
katatrace.AddTag(span, "sandbox_id", sandboxID)
c, err = sandbox.CreateContainer(ctx, contConfig)
if err != nil {

14
src/runtime/pkg/katautils/hook.go
View File

@ -15,18 +15,25 @@ import (
"syscall"
"time"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
"go.opentelemetry.io/otel/label"
)
// hookTracingTags defines tags for the trace span
var hookTracingTags = map[string]string{
"source": "runtime",
"package": "katautils",
"subsystem": "hook",
}
// Logger returns a logrus logger appropriate for logging hook messages
func hookLogger() *logrus.Entry {
return kataUtilsLogger.WithField("subsystem", "hook")
}
func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error {
span, _ := Trace(ctx, "runHook", []label.KeyValue{label.Key("source").String("runtime"), label.Key("package").String("katautils"), label.Key("subsystem").String("hook")}...)
span, _ := katatrace.Trace(ctx, hookLogger(), "runHook", hookTracingTags)
defer span.End()
// FIXME
@ -88,7 +95,8 @@ func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error
}
func runHooks(ctx context.Context, hooks []specs.Hook, cid, bundlePath, hookType string) error {
span, ctx := Trace(ctx, "runHooks", []label.KeyValue{label.Key("source").String("runtime"), label.Key("package").String("katautils"), label.Key("subsystem").String("hook"), label.Key("type").String(hookType)}...)
span, _ := katatrace.Trace(ctx, hookLogger(), "runHooks", hookTracingTags)
katatrace.AddTag(span, "type", hookType)
defer span.End()
for _, hook := range hooks {

54
src/runtime/pkg/katautils/tracing.go → src/runtime/pkg/katautils/katatrace/tracing.go
View File

@ -3,12 +3,12 @@
// SPDX-License-Identifier: Apache-2.0
//
package katautils
package katatrace
import (
"context"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
"github.com/sirupsen/logrus"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/trace/jaeger"
"go.opentelemetry.io/otel/label"
@ -30,7 +30,7 @@ var _ export.SpanExporter = (*kataSpanExporter)(nil)
// ExportSpans exports SpanData to Jaeger.
func (e *kataSpanExporter) ExportSpans(ctx context.Context, spans []*export.SpanData) error {
for _, span := range spans {
kataUtilsLogger.Tracef("Reporting span %+v", span)
kataTraceLogger.Tracef("Reporting span %+v", span)
}
return nil
}
@ -43,8 +43,25 @@ func (e *kataSpanExporter) Shutdown(ctx context.Context) error {
// is used by stopTracing().
var tracerCloser func()
var kataTraceLogger = logrus.NewEntry(logrus.New())
// tracing determines whether tracing is enabled.
var tracing bool
// SetTracing turns tracing on or off. Called by the configuration.
func SetTracing(isTracing bool) {
tracing = isTracing
}
// JaegerConfig defines necessary Jaeger config for exporting traces.
type JaegerConfig struct {
JaegerEndpoint string
JaegerUser string
JaegerPassword string
}
// CreateTracer create a tracer
func CreateTracer(name string, config *oci.RuntimeConfig) (func(), error) {
func CreateTracer(name string, config *JaegerConfig) (func(), error) {
if !tracing {
otel.SetTracerProvider(trace.NewNoopTracerProvider())
return func() {}, nil
@ -109,12 +126,25 @@ func StopTracing(ctx context.Context) {
}
}
// Trace creates a new tracing span based on the specified name and parent
// context and an opentelemetry label.KeyValue slice for span attributes.
func Trace(parent context.Context, name string, tags ...label.KeyValue) (otelTrace.Span, context.Context) {
// Trace creates a new tracing span based on the specified name and parent context.
// It also accepts a logger to record nil context errors and a map of tracing tags.
// Tracing tag keys and values are strings.
func Trace(parent context.Context, logger *logrus.Entry, name string, tags map[string]string) (otelTrace.Span, context.Context) {
if parent == nil {
if logger == nil {
logger = kataTraceLogger
}
logger.WithField("type", "bug").Error("trace called before context set")
parent = context.Background()
}
var otelTags []label.KeyValue
for k, v := range tags {
otelTags = append(otelTags, label.Key(k).String(v))
}
tracer := otel.Tracer("kata")
ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(tags...))
ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelTags...))
// This is slightly confusing: when tracing is disabled, trace spans
// are still created - but the tracer used is a NOP. Therefore, only
@ -122,8 +152,14 @@ func Trace(parent context.Context, name string, tags ...label.KeyValue) (otelTra
if tracing {
// This log message is *essential*: it is used by:
// https: //github.com/kata-containers/tests/blob/master/tracing/tracing-test.sh
kataUtilsLogger.Debugf("created span %v", span)
kataTraceLogger.Debugf("created span %v", span)
}
return span, ctx
}
// AddTag adds an additional key-value pair to a tracing span. This can be used to
// provide dynamic tags that are determined at runtime.
func AddTag(span otelTrace.Span, key string, value interface{}) {
span.SetAttributes(label.Any(key, value))
}

60
src/runtime/virtcontainers/acrn.go
View File

@ -19,10 +19,8 @@ import (
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/label"
otelTrace "go.opentelemetry.io/otel/trace"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
@ -31,6 +29,17 @@ import (
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
)
// tracingTags defines tags for the trace span
func (a *Acrn) tracingTags() map[string]string {
return map[string]string{
"source": "runtime",
"package": "virtcontainers",
"subsystem": "hypervisor",
"type": "acrn",
"sandbox_id": a.id,
}
}
// Since ACRN is using the store in a quite abnormal way, let's first draw it back from store to here
// UUIDPathSuffix is the suffix used for uuid storage
@ -150,7 +159,7 @@ func (a *Acrn) kernelParameters() string {
// Adds all capabilities supported by Acrn implementation of hypervisor interface
func (a *Acrn) capabilities(ctx context.Context) types.Capabilities {
span, _ := a.trace(ctx, "capabilities")
span, _ := katatrace.Trace(ctx, a.Logger(), "capabilities", a.tracingTags())
defer span.End()
return a.arch.capabilities()
@ -207,18 +216,6 @@ func (a *Acrn) Logger() *logrus.Entry {
return virtLog.WithField("subsystem", "acrn")
}
func (a *Acrn) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
if parent == nil {
a.Logger().WithField("type", "bug").Error("trace called before context set")
parent = context.Background()
}
tracer := otel.Tracer("kata")
ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(label.String("source", "runtime"), label.String("package", "virtcontainers"), label.String("subsystem", "hypervisor"), label.String("type", "acrn"), label.String("sandbox_id", a.id)))
return span, ctx
}
func (a *Acrn) memoryTopology() (Memory, error) {
memMb := uint64(a.config.MemorySize)
@ -287,7 +284,7 @@ func (a *Acrn) buildDevices(ctx context.Context, imagePath string) ([]Device, er
// setup sets the Acrn structure up.
func (a *Acrn) setup(ctx context.Context, id string, hypervisorConfig *HypervisorConfig) error {
span, _ := a.trace(ctx, "setup")
span, _ := katatrace.Trace(ctx, a.Logger(), "setup", a.tracingTags())
defer span.End()
err := hypervisorConfig.valid()
@ -330,7 +327,7 @@ func (a *Acrn) setup(ctx context.Context, id string, hypervisorConfig *Hyperviso
}
func (a *Acrn) createDummyVirtioBlkDev(ctx context.Context, devices []Device) ([]Device, error) {
span, _ := a.trace(ctx, "createDummyVirtioBlkDev")
span, _ := katatrace.Trace(ctx, a.Logger(), "createDummyVirtioBlkDev", a.tracingTags())
defer span.End()
// Since acrn doesn't support hot-plug, dummy virtio-blk
@ -353,8 +350,7 @@ func (a *Acrn) createSandbox(ctx context.Context, id string, networkNS NetworkNa
// Save the tracing context
a.ctx = ctx
var span otelTrace.Span
span, ctx = a.trace(ctx, "createSandbox")
span, ctx := katatrace.Trace(ctx, a.Logger(), "createSandbox", a.tracingTags())
defer span.End()
if err := a.setup(ctx, id, hypervisorConfig); err != nil {
@ -419,7 +415,7 @@ func (a *Acrn) createSandbox(ctx context.Context, id string, networkNS NetworkNa
// startSandbox will start the Sandbox's VM.
func (a *Acrn) startSandbox(ctx context.Context, timeoutSecs int) error {
span, ctx := a.trace(ctx, "startSandbox")
span, ctx := katatrace.Trace(ctx, a.Logger(), "startSandbox", a.tracingTags())
defer span.End()
if a.config.Debug {
@ -465,7 +461,7 @@ func (a *Acrn) startSandbox(ctx context.Context, timeoutSecs int) error {
// waitSandbox will wait for the Sandbox's VM to be up and running.
func (a *Acrn) waitSandbox(ctx context.Context, timeoutSecs int) error {
span, _ := a.trace(ctx, "waitSandbox")
span, _ := katatrace.Trace(ctx, a.Logger(), "waitSandbox", a.tracingTags())
defer span.End()
if timeoutSecs < 0 {
@ -479,7 +475,7 @@ func (a *Acrn) waitSandbox(ctx context.Context, timeoutSecs int) error {
// stopSandbox will stop the Sandbox's VM.
func (a *Acrn) stopSandbox(ctx context.Context, waitOnly bool) (err error) {
span, _ := a.trace(ctx, "stopSandbox")
span, _ := katatrace.Trace(ctx, a.Logger(), "stopSandbox", a.tracingTags())
defer span.End()
a.Logger().Info("Stopping acrn VM")
@ -547,7 +543,7 @@ func (a *Acrn) updateBlockDevice(drive *config.BlockDrive) error {
}
func (a *Acrn) hotplugAddDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
span, _ := a.trace(ctx, "hotplugAddDevice")
span, _ := katatrace.Trace(ctx, a.Logger(), "hotplugAddDevice", a.tracingTags())
defer span.End()
switch devType {
@ -561,7 +557,7 @@ func (a *Acrn) hotplugAddDevice(ctx context.Context, devInfo interface{}, devTyp
}
func (a *Acrn) hotplugRemoveDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
span, _ := a.trace(ctx, "hotplugRemoveDevice")
span, _ := katatrace.Trace(ctx, a.Logger(), "hotplugRemoveDevice", a.tracingTags())
defer span.End()
// Not supported. return success
@ -570,7 +566,7 @@ func (a *Acrn) hotplugRemoveDevice(ctx context.Context, devInfo interface{}, dev
}
func (a *Acrn) pauseSandbox(ctx context.Context) error {
span, _ := a.trace(ctx, "pauseSandbox")
span, _ := katatrace.Trace(ctx, a.Logger(), "pauseSandbox", a.tracingTags())
defer span.End()
// Not supported. return success
@ -579,7 +575,7 @@ func (a *Acrn) pauseSandbox(ctx context.Context) error {
}
func (a *Acrn) resumeSandbox(ctx context.Context) error {
span, _ := a.trace(ctx, "resumeSandbox")
span, _ := katatrace.Trace(ctx, a.Logger(), "resumeSandbox", a.tracingTags())
defer span.End()
// Not supported. return success
@ -590,7 +586,7 @@ func (a *Acrn) resumeSandbox(ctx context.Context) error {
// addDevice will add extra devices to acrn command line.
func (a *Acrn) addDevice(ctx context.Context, devInfo interface{}, devType deviceType) error {
var err error
span, _ := a.trace(ctx, "addDevice")
span, _ := katatrace.Trace(ctx, a.Logger(), "addDevice", a.tracingTags())
defer span.End()
switch v := devInfo.(type) {
@ -623,7 +619,7 @@ func (a *Acrn) addDevice(ctx context.Context, devInfo interface{}, devType devic
// getSandboxConsole builds the path of the console where we can read
// logs coming from the sandbox.
func (a *Acrn) getSandboxConsole(ctx context.Context, id string) (string, string, error) {
span, _ := a.trace(ctx, "getSandboxConsole")
span, _ := katatrace.Trace(ctx, a.Logger(), "getSandboxConsole", a.tracingTags())
defer span.End()
consoleURL, err := utils.BuildSocketPath(a.store.RunVMStoragePath(), id, acrnConsoleSocket)
@ -643,14 +639,14 @@ func (a *Acrn) saveSandbox() error {
}
func (a *Acrn) disconnect(ctx context.Context) {
span, _ := a.trace(ctx, "disconnect")
span, _ := katatrace.Trace(ctx, a.Logger(), "disconnect", a.tracingTags())
defer span.End()
// Not supported.
}
func (a *Acrn) getThreadIDs(ctx context.Context) (vcpuThreadIDs, error) {
span, _ := a.trace(ctx, "getThreadIDs")
span, _ := katatrace.Trace(ctx, a.Logger(), "getThreadIDs", a.tracingTags())
defer span.End()
// Not supported. return success
@ -668,7 +664,7 @@ func (a *Acrn) resizeVCPUs(ctx context.Context, reqVCPUs uint32) (currentVCPUs u
}
func (a *Acrn) cleanup(ctx context.Context) error {
span, _ := a.trace(ctx, "cleanup")
span, _ := katatrace.Trace(ctx, a.Logger(), "cleanup", a.tracingTags())
defer span.End()
return nil

26
src/runtime/virtcontainers/api.go
View File

@ -9,32 +9,28 @@ import (
"context"
"runtime"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
deviceApi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
deviceConfig "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/cgroups"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
"github.com/sirupsen/logrus"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/label"
otelTrace "go.opentelemetry.io/otel/trace"
)
// apiTracingTags defines tags for the trace span
var apiTracingTags = map[string]string{
"source": "runtime",
"packages": "virtcontainers",
"subsystem": "api",
}
func init() {
runtime.LockOSThread()
}
var virtLog = logrus.WithField("source", "virtcontainers")
// trace creates a new tracing span based on the specified name and parent
// context.
func trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
tracer := otel.Tracer("kata")
ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(label.String("source", "runtime"), label.String("packages", "virtcontainers"), label.String("subsystem", "api")))
return span, ctx
}
// SetLogger sets the logger for virtcontainers package.
func SetLogger(ctx context.Context, logger *logrus.Entry) {
fields := virtLog.Data
@ -49,7 +45,7 @@ func SetLogger(ctx context.Context, logger *logrus.Entry) {
// CreateSandbox is the virtcontainers sandbox creation entry point.
// CreateSandbox creates a sandbox and its containers. It does not start them.
func CreateSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factory) (VCSandbox, error) {
span, ctx := trace(ctx, "CreateSandbox")
span, ctx := katatrace.Trace(ctx, virtLog, "CreateSandbox", apiTracingTags)
defer span.End()
s, err := createSandboxFromConfig(ctx, sandboxConfig, factory)
@ -58,7 +54,7 @@ func CreateSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac
}
func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, factory Factory) (_ *Sandbox, err error) {
span, ctx := trace(ctx, "createSandboxFromConfig")
span, ctx := katatrace.Trace(ctx, virtLog, "createSandboxFromConfig", apiTracingTags)
defer span.End()
// Create the sandbox.
@ -127,7 +123,7 @@ func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, f
// in the sandbox left, do stop the sandbox and delete it. Those serial operations will be done exclusively by
// locking the sandbox.
func CleanupContainer(ctx context.Context, sandboxID, containerID string, force bool) error {
span, ctx := trace(ctx, "CleanupContainer")
span, ctx := katatrace.Trace(ctx, virtLog, "CleanupContainer", apiTracingTags)
defer span.End()
if sandboxID == "" {

45
src/runtime/virtcontainers/clh.go
View File

@ -25,16 +25,25 @@ import (
"github.com/opencontainers/selinux/go-selinux/label"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"go.opentelemetry.io/otel"
otelLabel "go.opentelemetry.io/otel/label"
otelTrace "go.opentelemetry.io/otel/trace"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
)
// tracingTags defines tags for the trace span
func (clh *cloudHypervisor) tracingTags() map[string]string {
return map[string]string{
"source": "runtime",
"package": "virtcontainers",
"subsystem": "hypervisor",
"type": "clh",
"sandbox_id": clh.id,
}
}
//
// Constants and type definitions related to cloud hypervisor
//
@ -146,8 +155,8 @@ var clhDebugKernelParams = []Param{
func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error {
clh.ctx = ctx
var span otelTrace.Span
span, clh.ctx = clh.trace(clh.ctx, "createSandbox")
span, newCtx := katatrace.Trace(clh.ctx, clh.Logger(), "createSandbox", clh.tracingTags())
clh.ctx = newCtx
defer span.End()
err := hypervisorConfig.valid()
@ -295,7 +304,7 @@ func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networ
// startSandbox will start the VMM and boot the virtual machine for the given sandbox.
func (clh *cloudHypervisor) startSandbox(ctx context.Context, timeout int) error {
span, _ := clh.trace(ctx, "startSandbox")
span, _ := katatrace.Trace(ctx, clh.Logger(), "startSandbox", clh.tracingTags())
defer span.End()
ctx, cancel := context.WithTimeout(context.Background(), clhAPITimeout*time.Second)
@ -454,7 +463,7 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device config.VFIODev) error {
}
func (clh *cloudHypervisor) hotplugAddDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
span, _ := clh.trace(ctx, "hotplugAddDevice")
span, _ := katatrace.Trace(ctx, clh.Logger(), "hotplugAddDevice", clh.tracingTags())
defer span.End()
switch devType {
@ -471,7 +480,7 @@ func (clh *cloudHypervisor) hotplugAddDevice(ctx context.Context, devInfo interf
}
func (clh *cloudHypervisor) hotplugRemoveDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
span, _ := clh.trace(ctx, "hotplugRemoveDevice")
span, _ := katatrace.Trace(ctx, clh.Logger(), "hotplugRemoveDevice", clh.tracingTags())
defer span.End()
var deviceID string
@ -631,7 +640,7 @@ func (clh *cloudHypervisor) resumeSandbox(ctx context.Context) error {
// stopSandbox will stop the Sandbox's VM.
func (clh *cloudHypervisor) stopSandbox(ctx context.Context, waitOnly bool) (err error) {
span, _ := clh.trace(ctx, "stopSandbox")
span, _ := katatrace.Trace(ctx, clh.Logger(), "stopSandbox", clh.tracingTags())
defer span.End()
clh.Logger().WithField("function", "stopSandbox").Info("Stop Sandbox")
return clh.terminate(ctx, waitOnly)
@ -681,7 +690,7 @@ func (clh *cloudHypervisor) getVirtioFsPid() *int {
}
func (clh *cloudHypervisor) addDevice(ctx context.Context, devInfo interface{}, devType deviceType) error {
span, _ := clh.trace(ctx, "addDevice")
span, _ := katatrace.Trace(ctx, clh.Logger(), "addDevice", clh.tracingTags())
defer span.End()
var err error
@ -715,7 +724,7 @@ func (clh *cloudHypervisor) Logger() *log.Entry {
// Adds all capabilities supported by cloudHypervisor implementation of hypervisor interface
func (clh *cloudHypervisor) capabilities(ctx context.Context) types.Capabilities {
span, _ := clh.trace(ctx, "capabilities")
span, _ := katatrace.Trace(ctx, clh.Logger(), "capabilities", clh.tracingTags())
defer span.End()
clh.Logger().WithField("function", "capabilities").Info("get Capabilities")
@ -725,20 +734,8 @@ func (clh *cloudHypervisor) capabilities(ctx context.Context) types.Capabilities
return caps
}
func (clh *cloudHypervisor) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
if parent == nil {
clh.Logger().WithField("type", "bug").Error("trace called before context set")
parent = context.Background()
}
tracer := otel.Tracer("kata")
ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "hypervisor"), otelLabel.String("type", "clh"), otelLabel.String("sandbox_id", clh.id)))
return span, ctx
}
func (clh *cloudHypervisor) terminate(ctx context.Context, waitOnly bool) (err error) {
span, _ := clh.trace(ctx, "terminate")
span, _ := katatrace.Trace(ctx, clh.Logger(), "terminate", clh.tracingTags())
defer span.End()
pid := clh.state.PID

38
src/runtime/virtcontainers/container.go
View File

@ -16,6 +16,7 @@ import (
"syscall"
"time"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"
@ -24,9 +25,6 @@ import (
vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
"go.opentelemetry.io/otel"
otelLabel "go.opentelemetry.io/otel/label"
otelTrace "go.opentelemetry.io/otel/trace"
"github.com/containerd/cgroups"
specs "github.com/opencontainers/runtime-spec/specs-go"
@ -35,6 +33,16 @@ import (
"golang.org/x/sys/unix"
)
// tracingTags defines tags for the trace span
func (c *Container) tracingTags() map[string]string {
return map[string]string{
"source": "runtime",
"package": "virtcontainers",
"subsystem": "container",
"container_id": c.id,
}
}
// https://github.com/torvalds/linux/blob/master/include/uapi/linux/major.h
// This file has definitions for major device numbers.
var cdromMajors = map[int64]string{
@ -354,18 +362,6 @@ func (c *Container) Logger() *logrus.Entry {
})
}
func (c *Container) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
if parent == nil {
c.Logger().WithField("type", "bug").Error("trace called before context set")
parent = context.Background()
}
tracer := otel.Tracer("kata")
ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "container"), otelLabel.String("container_id", c.id)))
return span, ctx
}
// Sandbox returns the sandbox handler related to this container.
func (c *Container) Sandbox() VCSandbox {
return c.sandbox
@ -622,14 +618,13 @@ func (c *Container) mountSharedDirMounts(ctx context.Context, sharedDirMounts, i
}
func (c *Container) unmountHostMounts(ctx context.Context) error {
var span otelTrace.Span
span, ctx = c.trace(ctx, "unmountHostMounts")
span, ctx := katatrace.Trace(ctx, c.Logger(), "unmountHostMounts", c.tracingTags())
defer span.End()
for _, m := range c.mounts {
if m.HostPath != "" {
span, _ := c.trace(ctx, "unmount")
span.SetAttributes(otelLabel.Key("host-path").String(m.HostPath))
span, _ := katatrace.Trace(ctx, c.Logger(), "unmount", c.tracingTags())
katatrace.AddTag(span, "host-path", m.HostPath)
if err := syscall.Unmount(m.HostPath, syscall.MNT_DETACH|UmountNoFollow); err != nil {
c.Logger().WithFields(logrus.Fields{
@ -751,7 +746,7 @@ func (c *Container) createBlockDevices(ctx context.Context) error {
// newContainer creates a Container structure from a sandbox and a container configuration.
func newContainer(ctx context.Context, sandbox *Sandbox, contConfig *ContainerConfig) (*Container, error) {
span, ctx := sandbox.trace(ctx, "newContainer")
span, ctx := katatrace.Trace(ctx, sandbox.Logger(), "newContainer", sandbox.tracingTags())
defer span.End()
if !contConfig.valid() {
@ -1021,8 +1016,7 @@ func (c *Container) start(ctx context.Context) error {
}
func (c *Container) stop(ctx context.Context, force bool) error {
var span otelTrace.Span
span, ctx = c.trace(ctx, "stop")
span, ctx := katatrace.Trace(ctx, c.Logger(), "stop", c.tracingTags())
defer span.End()
// In case the container status has been updated implicitly because

22
src/runtime/virtcontainers/factory/factory.go
View File

@ -9,6 +9,7 @@ import (
"context"
"fmt"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
pb "github.com/kata-containers/kata-containers/src/runtime/protocols/cache"
vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/factory/base"
@ -18,11 +19,15 @@ import (
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/factory/template"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
"github.com/sirupsen/logrus"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/label"
otelTrace "go.opentelemetry.io/otel/trace"
)
// factoryTracingTags defines tags for the trace span
var factoryTracingTags = map[string]string{
"source": "runtime",
"package": "factory",
"subsystem": "factory",
}
var factoryLogger = logrus.FieldLogger(logrus.New())
// Config is a collection of VM factory configurations.
@ -40,16 +45,9 @@ type factory struct {
base base.FactoryBase
}
func trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
tracer := otel.Tracer("kata")
ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(label.String("source", "runtime"), label.String("package", "factory"), label.String("subsystem", "factory")))
return span, ctx
}
// NewFactory returns a working factory.
func NewFactory(ctx context.Context, config Config, fetchOnly bool) (vc.Factory, error) {
span, _ := trace(ctx, "NewFactory")
span, _ := katatrace.Trace(ctx, nil, "NewFactory", factoryTracingTags)
defer span.End()
err := config.VMConfig.Valid()
@ -140,7 +138,7 @@ func (f *factory) checkConfig(config vc.VMConfig) error {
// GetVM returns a working blank VM created by the factory.
func (f *factory) GetVM(ctx context.Context, config vc.VMConfig) (*vc.VM, error) {
span, ctx := trace(ctx, "GetVM")
span, ctx := katatrace.Trace(ctx, f.log(), "GetVM", factoryTracingTags)
defer span.End()
hypervisorConfig := config.HypervisorConfig

74
src/runtime/virtcontainers/fc.go
View File

@ -22,6 +22,7 @@ import (
"syscall"
"time"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client"
@ -38,11 +39,19 @@ import (
"github.com/opencontainers/selinux/go-selinux/label"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"go.opentelemetry.io/otel"
otelLabel "go.opentelemetry.io/otel/label"
otelTrace "go.opentelemetry.io/otel/trace"
)
// tracingTags defines tags for the trace span
func (fc *firecracker) tracingTags() map[string]string {
return map[string]string{
"source": "runtime",
"package": "virtcontainers",
"subsystem": "hypervisor",
"type": "firecracker",
"sandbox_id": fc.id,
}
}
type vmmState uint8
const (
@ -168,18 +177,6 @@ func (fc *firecracker) Logger() *logrus.Entry {
return virtLog.WithField("subsystem", "firecracker")
}
func (fc *firecracker) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
if parent == nil {
fc.Logger().WithField("type", "bug").Error("trace called before context set")
parent = context.Background()
}
tracer := otel.Tracer("kata")
ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "hypervisor"), otelLabel.String("type", "firecracker"), otelLabel.String("sandbox_id", fc.id)))
return span, ctx
}
//At some cases, when sandbox id is too long, it will incur error of overlong
//firecracker API unix socket(fc.socketPath).
//In Linux, sun_path could maximumly contains 108 bytes in size.
@ -198,8 +195,7 @@ func (fc *firecracker) truncateID(id string) string {
func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error {
fc.ctx = ctx
var span otelTrace.Span
span, _ = fc.trace(ctx, "createSandbox")
span, _ := katatrace.Trace(ctx, fc.Logger(), "createSandbox", fc.tracingTags())
defer span.End()
//TODO: check validity of the hypervisor config provided
@ -242,7 +238,7 @@ func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS N
}
func (fc *firecracker) newFireClient(ctx context.Context) *client.Firecracker {
span, _ := fc.trace(ctx, "newFireClient")
span, _ := katatrace.Trace(ctx, fc.Logger(), "newFireClient", fc.tracingTags())
defer span.End()
httpClient := client.NewHTTPClient(strfmt.NewFormats())
@ -312,7 +308,7 @@ func (fc *firecracker) checkVersion(version string) error {
// waitVMMRunning will wait for timeout seconds for the VMM to be up and running.
func (fc *firecracker) waitVMMRunning(ctx context.Context, timeout int) error {
span, _ := fc.trace(ctx, "wait VMM to be running")
span, _ := katatrace.Trace(ctx, fc.Logger(), "wait VMM to be running", fc.tracingTags())
defer span.End()
if timeout < 0 {
@ -334,7 +330,7 @@ func (fc *firecracker) waitVMMRunning(ctx context.Context, timeout int) error {
}
func (fc *firecracker) fcInit(ctx context.Context, timeout int) error {
span, _ := fc.trace(ctx, "fcInit")
span, _ := katatrace.Trace(ctx, fc.Logger(), "fcInit", fc.tracingTags())
defer span.End()
var err error
@ -409,7 +405,7 @@ func (fc *firecracker) fcInit(ctx context.Context, timeout int) error {
}
func (fc *firecracker) fcEnd(ctx context.Context, waitOnly bool) (err error) {
span, _ := fc.trace(ctx, "fcEnd")
span, _ := katatrace.Trace(ctx, fc.Logger(), "fcEnd", fc.tracingTags())
defer span.End()
fc.Logger().Info("Stopping firecracker VM")
@ -436,7 +432,7 @@ func (fc *firecracker) fcEnd(ctx context.Context, waitOnly bool) (err error) {
}
func (fc *firecracker) client(ctx context.Context) *client.Firecracker {
span, _ := fc.trace(ctx, "client")
span, _ := katatrace.Trace(ctx, fc.Logger(), "client", fc.tracingTags())
defer span.End()
if fc.connection == nil {
@ -503,7 +499,7 @@ func (fc *firecracker) fcJailResource(src, dst string) (string, error) {
}
func (fc *firecracker) fcSetBootSource(ctx context.Context, path, params string) error {
span, _ := fc.trace(ctx, "fcSetBootSource")
span, _ := katatrace.Trace(ctx, fc.Logger(), "fcSetBootSource", fc.tracingTags())
defer span.End()
fc.Logger().WithFields(logrus.Fields{"kernel-path": path,
"kernel-params": params}).Debug("fcSetBootSource")
@ -524,7 +520,7 @@ func (fc *firecracker) fcSetBootSource(ctx context.Context, path, params string)
}
func (fc *firecracker) fcSetVMRootfs(ctx context.Context, path string) error {
span, _ := fc.trace(ctx, "fcSetVMRootfs")
span, _ := katatrace.Trace(ctx, fc.Logger(), "fcSetVMRootfs", fc.tracingTags())
defer span.End()
jailedRootfs, err := fc.fcJailResource(path, fcRootfs)
@ -551,7 +547,7 @@ func (fc *firecracker) fcSetVMRootfs(ctx context.Context, path string) error {
}
func (fc *firecracker) fcSetVMBaseConfig(ctx context.Context, mem int64, vcpus int64, htEnabled bool) {
span, _ := fc.trace(ctx, "fcSetVMBaseConfig")
span, _ := katatrace.Trace(ctx, fc.Logger(), "fcSetVMBaseConfig", fc.tracingTags())
defer span.End()
fc.Logger().WithFields(logrus.Fields{"mem": mem,
"vcpus": vcpus,
@ -567,7 +563,7 @@ func (fc *firecracker) fcSetVMBaseConfig(ctx context.Context, mem int64, vcpus i
}
func (fc *firecracker) fcSetLogger(ctx context.Context) error {
span, _ := fc.trace(ctx, "fcSetLogger")
span, _ := katatrace.Trace(ctx, fc.Logger(), "fcSetLogger", fc.tracingTags())
defer span.End()
fcLogLevel := "Error"
@ -590,7 +586,7 @@ func (fc *firecracker) fcSetLogger(ctx context.Context) error {
}
func (fc *firecracker) fcSetMetrics(ctx context.Context) error {
span, _ := fc.trace(ctx, "fcSetMetrics")
span, _ := katatrace.Trace(ctx, fc.Logger(), "fcSetMetrics", fc.tracingTags())
defer span.End()
// listen to metrics file and transfer error info
@ -745,7 +741,7 @@ func (fc *firecracker) fcInitConfiguration(ctx context.Context) error {
// In the context of firecracker, this will start the hypervisor,
// for configuration, but not yet start the actual virtual machine
func (fc *firecracker) startSandbox(ctx context.Context, timeout int) error {
span, _ := fc.trace(ctx, "startSandbox")
span, _ := katatrace.Trace(ctx, fc.Logger(), "startSandbox", fc.tracingTags())
defer span.End()
if err := fc.fcInitConfiguration(ctx); err != nil {
@ -797,7 +793,7 @@ func fcDriveIndexToID(i int) string {
}
func (fc *firecracker) createDiskPool(ctx context.Context) error {
span, _ := fc.trace(ctx, "createDiskPool")
span, _ := katatrace.Trace(ctx, fc.Logger(), "createDiskPool", fc.tracingTags())
defer span.End()
for i := 0; i < fcDiskPoolSize; i++ {
@ -835,7 +831,7 @@ func (fc *firecracker) umountResource(jailedPath string) {
// cleanup all jail artifacts
func (fc *firecracker) cleanupJail(ctx context.Context) {
span, _ := fc.trace(ctx, "cleanupJail")
span, _ := katatrace.Trace(ctx, fc.Logger(), "cleanupJail", fc.tracingTags())
defer span.End()
fc.umountResource(fcKernel)
@ -858,7 +854,7 @@ func (fc *firecracker) cleanupJail(ctx context.Context) {
// stopSandbox will stop the Sandbox's VM.
func (fc *firecracker) stopSandbox(ctx context.Context, waitOnly bool) (err error) {
span, _ := fc.trace(ctx, "stopSandbox")
span, _ := katatrace.Trace(ctx, fc.Logger(), "stopSandbox", fc.tracingTags())
defer span.End()
return fc.fcEnd(ctx, waitOnly)
@ -877,7 +873,7 @@ func (fc *firecracker) resumeSandbox(ctx context.Context) error {
}
func (fc *firecracker) fcAddVsock(ctx context.Context, hvs types.HybridVSock) {
span, _ := fc.trace(ctx, "fcAddVsock")
span, _ := katatrace.Trace(ctx, fc.Logger(), "fcAddVsock", fc.tracingTags())
defer span.End()
udsPath := hvs.UdsPath
@ -897,7 +893,7 @@ func (fc *firecracker) fcAddVsock(ctx context.Context, hvs types.HybridVSock) {
}
func (fc *firecracker) fcAddNetDevice(ctx context.Context, endpoint Endpoint) {
span, _ := fc.trace(ctx, "fcAddNetDevice")
span, _ := katatrace.Trace(ctx, fc.Logger(), "fcAddNetDevice", fc.tracingTags())
defer span.End()
ifaceID := endpoint.Name()
@ -953,7 +949,7 @@ func (fc *firecracker) fcAddNetDevice(ctx context.Context, endpoint Endpoint) {
}
func (fc *firecracker) fcAddBlockDrive(ctx context.Context, drive config.BlockDrive) error {
span, _ := fc.trace(ctx, "fcAddBlockDrive")
span, _ := katatrace.Trace(ctx, fc.Logger(), "fcAddBlockDrive", fc.tracingTags())
defer span.End()
driveID := drive.ID
@ -979,7 +975,7 @@ func (fc *firecracker) fcAddBlockDrive(ctx context.Context, drive config.BlockDr
// Firecracker supports replacing the host drive used once the VM has booted up
func (fc *firecracker) fcUpdateBlockDrive(ctx context.Context, path, id string) error {
span, _ := fc.trace(ctx, "fcUpdateBlockDrive")
span, _ := katatrace.Trace(ctx, fc.Logger(), "fcUpdateBlockDrive", fc.tracingTags())
defer span.End()
// Use the global block index as an index into the pool of the devices
@ -1003,7 +999,7 @@ func (fc *firecracker) fcUpdateBlockDrive(ctx context.Context, path, id string)
// addDevice will add extra devices to firecracker. Limited to configure before the
// virtual machine starts. Devices include drivers and network interfaces only.
func (fc *firecracker) addDevice(ctx context.Context, devInfo interface{}, devType deviceType) error {
span, _ := fc.trace(ctx, "addDevice")
span, _ := katatrace.Trace(ctx, fc.Logger(), "addDevice", fc.tracingTags())
defer span.End()
fc.state.RLock()
@ -1064,7 +1060,7 @@ func (fc *firecracker) hotplugBlockDevice(ctx context.Context, drive config.Bloc
// hotplugAddDevice supported in Firecracker VMM
func (fc *firecracker) hotplugAddDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
span, _ := fc.trace(ctx, "hotplugAddDevice")
span, _ := katatrace.Trace(ctx, fc.Logger(), "hotplugAddDevice", fc.tracingTags())
defer span.End()
switch devType {
@ -1080,7 +1076,7 @@ func (fc *firecracker) hotplugAddDevice(ctx context.Context, devInfo interface{}
// hotplugRemoveDevice supported in Firecracker VMM
func (fc *firecracker) hotplugRemoveDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
span, _ := fc.trace(ctx, "hotplugRemoveDevice")
span, _ := katatrace.Trace(ctx, fc.Logger(), "hotplugRemoveDevice", fc.tracingTags())
defer span.End()
switch devType {
@ -1113,7 +1109,7 @@ func (fc *firecracker) disconnect(ctx context.Context) {
// Adds all capabilities supported by firecracker implementation of hypervisor interface
func (fc *firecracker) capabilities(ctx context.Context) types.Capabilities {
span, _ := fc.trace(ctx, "capabilities")
span, _ := katatrace.Trace(ctx, fc.Logger(), "capabilities", fc.tracingTags())
defer span.End()
var caps types.Capabilities
caps.SetBlockDeviceHotplugSupport()

56
src/runtime/virtcontainers/kata_agent.go
View File

@ -18,6 +18,7 @@ import (
"syscall"
"time"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
@ -30,10 +31,6 @@ import (
vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/label"
otelLabel "go.opentelemetry.io/otel/label"
otelTrace "go.opentelemetry.io/otel/trace"
"github.com/gogo/protobuf/proto"
"github.com/opencontainers/runtime-spec/specs-go"
@ -44,6 +41,13 @@ import (
grpcStatus "google.golang.org/grpc/status"
)
// kataAgentTracingTags defines tags for the trace span
var kataAgentTracingTags = map[string]string{
"source": "runtime",
"package": "virtcontainers",
"subsystem": "agent",
}
const (
// KataEphemeralDevType creates a tmpfs backed volume for sharing files between containers.
KataEphemeralDevType = "ephemeral"
@ -246,18 +250,6 @@ type kataAgent struct {
ctx context.Context
}
func (k *kataAgent) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
if parent == nil {
k.Logger().WithField("type", "bug").Error("trace called before context set")
parent = context.Background()
}
tracer := otel.Tracer("kata")
ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(label.String("source", "runtime"), label.String("package", "virtcontainers"), label.String("subsystem", "agent")))
return span, ctx
}
func (k *kataAgent) Logger() *logrus.Entry {
return virtLog.WithField("subsystem", "kata_agent")
}
@ -341,7 +333,7 @@ func (k *kataAgent) init(ctx context.Context, sandbox *Sandbox, config KataAgent
// save
k.ctx = sandbox.ctx
span, _ := k.trace(ctx, "init")
span, _ := katatrace.Trace(ctx, k.Logger(), "init", kataAgentTracingTags)
defer span.End()
disableVMShutdown = k.handleTraceSettings(config)
@ -375,7 +367,7 @@ func (k *kataAgent) capabilities() types.Capabilities {
}
func (k *kataAgent) internalConfigure(ctx context.Context, h hypervisor, id string, config KataAgentConfig) error {
span, _ := k.trace(ctx, "configure")
span, _ := katatrace.Trace(ctx, k.Logger(), "configure", kataAgentTracingTags)
defer span.End()
var err error
@ -384,13 +376,13 @@ func (k *kataAgent) internalConfigure(ctx context.Context, h hypervisor, id stri
}
k.keepConn = config.LongLiveConn
span.SetAttributes(otelLabel.Any("socket", k.vmSocket))
katatrace.AddTag(span, "socket", k.vmSocket)
return nil
}
func (k *kataAgent) setupSandboxBindMounts(ctx context.Context, sandbox *Sandbox) (err error) {
span, ctx := k.trace(ctx, "setupSandboxBindMounts")
span, ctx := katatrace.Trace(ctx, k.Logger(), "setupSandboxBindMounts", kataAgentTracingTags)
defer span.End()
if len(sandbox.config.SandboxBindMounts) == 0 {
@ -463,7 +455,7 @@ func (k *kataAgent) cleanupSandboxBindMounts(sandbox *Sandbox) error {
}
func (k *kataAgent) configure(ctx context.Context, h hypervisor, id, sharePath string, config KataAgentConfig) error {
span, ctx := k.trace(ctx, "configure")
span, ctx := katatrace.Trace(ctx, k.Logger(), "configure", kataAgentTracingTags)
defer span.End()
err := k.internalConfigure(ctx, h, id, config)
@ -512,7 +504,7 @@ func (k *kataAgent) configureFromGrpc(ctx context.Context, h hypervisor, id stri
}
func (k *kataAgent) setupSharedPath(ctx context.Context, sandbox *Sandbox) (err error) {
span, ctx := k.trace(ctx, "setupSharedPath")
span, ctx := katatrace.Trace(ctx, k.Logger(), "setupSharedPath", kataAgentTracingTags)
defer span.End()
// create shared path structure
@ -546,7 +538,7 @@ func (k *kataAgent) setupSharedPath(ctx context.Context, sandbox *Sandbox) (err
}
func (k *kataAgent) createSandbox(ctx context.Context, sandbox *Sandbox) error {
span, ctx := k.trace(ctx, "createSandbox")
span, ctx := katatrace.Trace(ctx, k.Logger(), "createSandbox", kataAgentTracingTags)
defer span.End()
if err := k.setupSharedPath(ctx, sandbox); err != nil {
@ -633,7 +625,7 @@ func cmdEnvsToStringSlice(ev []types.EnvVar) []string {
}
func (k *kataAgent) exec(ctx context.Context, sandbox *Sandbox, c Container, cmd types.Cmd) (*Process, error) {
span, ctx := k.trace(ctx, "exec")
span, ctx := katatrace.Trace(ctx, k.Logger(), "exec", kataAgentTracingTags)
defer span.End()
var kataProcess *grpc.Process
@ -805,7 +797,7 @@ func (k *kataAgent) getDNS(sandbox *Sandbox) ([]string, error) {
}
func (k *kataAgent) startSandbox(ctx context.Context, sandbox *Sandbox) error {
span, ctx := k.trace(ctx, "startSandbox")
span, ctx := katatrace.Trace(ctx, k.Logger(), "startSandbox", kataAgentTracingTags)
defer span.End()
if err := k.setAgentURL(); err != nil {
@ -958,7 +950,7 @@ func setupStorages(ctx context.Context, sandbox *Sandbox) []*grpc.Storage {
}
func (k *kataAgent) stopSandbox(ctx context.Context, sandbox *Sandbox) error {
span, ctx := k.trace(ctx, "stopSandbox")
span, ctx := katatrace.Trace(ctx, k.Logger(), "stopSandbox", kataAgentTracingTags)
defer span.End()
req := &grpc.DestroySandboxRequest{}
@ -1315,7 +1307,7 @@ func (k *kataAgent) buildContainerRootfs(ctx context.Context, sandbox *Sandbox,
}
func (k *kataAgent) createContainer(ctx context.Context, sandbox *Sandbox, c *Container) (p *Process, err error) {
span, ctx := k.trace(ctx, "createContainer")
span, ctx := katatrace.Trace(ctx, k.Logger(), "createContainer", kataAgentTracingTags)
defer span.End()
var ctrStorages []*grpc.Storage
@ -1680,7 +1672,7 @@ func (k *kataAgent) handlePidNamespace(grpcSpec *grpc.Spec, sandbox *Sandbox) bo
}
func (k *kataAgent) startContainer(ctx context.Context, sandbox *Sandbox, c *Container) error {
span, ctx := k.trace(ctx, "startContainer")
span, ctx := katatrace.Trace(ctx, k.Logger(), "startContainer", kataAgentTracingTags)
defer span.End()
req := &grpc.StartContainerRequest{
@ -1692,7 +1684,7 @@ func (k *kataAgent) startContainer(ctx context.Context, sandbox *Sandbox, c *Con
}
func (k *kataAgent) stopContainer(ctx context.Context, sandbox *Sandbox, c Container) error {
span, ctx := k.trace(ctx, "stopContainer")
span, ctx := katatrace.Trace(ctx, k.Logger(), "stopContainer", kataAgentTracingTags)
defer span.End()
_, err := k.sendReq(ctx, &grpc.RemoveContainerRequest{ContainerId: c.id})
@ -1836,7 +1828,7 @@ func (k *kataAgent) connect(ctx context.Context) error {
return nil
}
span, _ := k.trace(ctx, "connect")
span, _ := katatrace.Trace(ctx, k.Logger(), "connect", kataAgentTracingTags)
defer span.End()
// This is for the first connection only, to prevent race
@ -1860,7 +1852,7 @@ func (k *kataAgent) connect(ctx context.Context) error {
}
func (k *kataAgent) disconnect(ctx context.Context) error {
span, _ := k.trace(ctx, "disconnect")
span, _ := katatrace.Trace(ctx, k.Logger(), "disconnect", kataAgentTracingTags)
defer span.End()
k.Lock()
@ -1890,7 +1882,7 @@ func (k *kataAgent) check(ctx context.Context) error {
}
func (k *kataAgent) waitProcess(ctx context.Context, c *Container, processID string) (int32, error) {
span, ctx := k.trace(ctx, "waitProcess")
span, ctx := katatrace.Trace(ctx, k.Logger(), "waitProcess", kataAgentTracingTags)
defer span.End()
resp, err := k.sendReq(ctx, &grpc.WaitProcessRequest{

13
src/runtime/virtcontainers/mount.go
View File

@ -16,6 +16,7 @@ import (
"syscall"
merr "github.com/hashicorp/go-multierror"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
"github.com/sirupsen/logrus"
otelLabel "go.opentelemetry.io/otel/label"
@ -239,7 +240,7 @@ func evalMountPath(source, destination string) (string, string, error) {
// * ensure the source exists
// * recursively create the destination
func moveMount(ctx context.Context, source, destination string) error {
span, _ := trace(ctx, "moveMount")
span, _ := katatrace.Trace(ctx, nil, "moveMount", apiTracingTags)
defer span.End()
source, destination, err := evalMountPath(source, destination)
@ -257,7 +258,7 @@ func moveMount(ctx context.Context, source, destination string) error {
// * recursively create the destination
// pgtypes stands for propagation types, which are shared, private, slave, and ubind.
func bindMount(ctx context.Context, source, destination string, readonly bool, pgtypes string) error {
span, _ := trace(ctx, "bindMount")
span, _ := katatrace.Trace(ctx, nil, "bindMount", apiTracingTags)
defer span.End()
span.SetAttributes(otelLabel.String("source", source), otelLabel.String("destination", destination))
@ -294,7 +295,7 @@ func bindMount(ctx context.Context, source, destination string, readonly bool, p
// The mountflags should match the values used in the original mount() call,
// except for those parameters that you are trying to change.
func remount(ctx context.Context, mountflags uintptr, src string) error {
span, _ := trace(ctx, "remount")
span, _ := katatrace.Trace(ctx, nil, "remount", apiTracingTags)
defer span.End()
span.SetAttributes(otelLabel.String("source", src))
@ -319,7 +320,7 @@ func remountRo(ctx context.Context, src string) error {
// bindMountContainerRootfs bind mounts a container rootfs into a 9pfs shared
// directory between the guest and the host.
func bindMountContainerRootfs(ctx context.Context, shareDir, cid, cRootFs string, readonly bool) error {
span, _ := trace(ctx, "bindMountContainerRootfs")
span, _ := katatrace.Trace(ctx, nil, "bindMountContainerRootfs", apiTracingTags)
defer span.End()
rootfsDest := filepath.Join(shareDir, cid, rootfsDir)
@ -359,7 +360,7 @@ func isSymlink(path string) bool {
}
func bindUnmountContainerRootfs(ctx context.Context, sharedDir, cID string) error {
span, _ := trace(ctx, "bindUnmountContainerRootfs")
span, _ := katatrace.Trace(ctx, nil, "bindUnmountContainerRootfs", apiTracingTags)
defer span.End()
span.SetAttributes(otelLabel.String("shared_dir", sharedDir), otelLabel.String("container_id", cID))
@ -382,7 +383,7 @@ func bindUnmountContainerRootfs(ctx context.Context, sharedDir, cID string) erro
}
func bindUnmountAllRootfs(ctx context.Context, sharedDir string, sandbox *Sandbox) error {
span, ctx := trace(ctx, "bindUnmountAllRootfs")
span, ctx := katatrace.Trace(ctx, nil, "bindUnmountAllRootfs", apiTracingTags)
defer span.End()
span.SetAttributes(otelLabel.String("shared_dir", sharedDir), otelLabel.String("sandbox_id", sandbox.id))

25
src/runtime/virtcontainers/network.go
View File

@ -22,17 +22,23 @@ import (
"github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
"github.com/vishvananda/netns"
"go.opentelemetry.io/otel"
otelLabel "go.opentelemetry.io/otel/label"
otelTrace "go.opentelemetry.io/otel/trace"
"golang.org/x/sys/unix"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
)
// networkTracingTags defines tags for the trace span
var networkTracingTags = map[string]string{
"source": "runtime",
"package": "virtcontainers",
"subsystem": "network",
}
// NetInterworkingModel defines the network model connecting
// the network interface to the virtual machine.
type NetInterworkingModel int
@ -1295,13 +1301,12 @@ func (n *Network) trace(ctx context.Context, name string) (otelTrace.Span, conte
func getNetworkTrace(networkType EndpointType) func(ctx context.Context, name string, endpoint interface{}) (otelTrace.Span, context.Context) {
return func(ctx context.Context, name string, endpoint interface{}) (otelTrace.Span, context.Context) {
tracer := otel.Tracer("kata")
ctx, span := tracer.Start(ctx, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "network")))
span, ctx := katatrace.Trace(ctx, networkLogger(), name, networkTracingTags)
if networkType != "" {
span.SetAttributes(otelLabel.Any("type", string(networkType)))
katatrace.AddTag(span, "type", string(networkType))
}
if endpoint != nil {
span.SetAttributes(otelLabel.Any("endpoint", endpoint))
katatrace.AddTag(span, "endpoint", endpoint)
}
return span, ctx
}
@ -1309,7 +1314,7 @@ func getNetworkTrace(networkType EndpointType) func(ctx context.Context, name st
func closeSpan(span otelTrace.Span, err error) {
if err != nil {
span.SetAttributes(otelLabel.Any("error", err))
katatrace.AddTag(span, "error", err)
}
span.End()
}
@ -1327,15 +1332,15 @@ func (n *Network) Run(ctx context.Context, networkNSPath string, cb func() error
// Add adds all needed interfaces inside the network namespace.
func (n *Network) Add(ctx context.Context, config *NetworkConfig, s *Sandbox, hotplug bool) ([]Endpoint, error) {
span, ctx := n.trace(ctx, "Add")
span.SetAttributes(otelLabel.String("type", config.InterworkingModel.GetModel()))
katatrace.AddTag(span, "type", config.InterworkingModel.GetModel())
defer span.End()
endpoints, err := createEndpointsFromScan(config.NetNSPath, config)
if err != nil {
return endpoints, err
}
span.SetAttributes(otelLabel.Any("endpoints", endpoints))
span.SetAttributes(otelLabel.Bool("hotplug", hotplug))
katatrace.AddTag(span, "endpoints", endpoints)
katatrace.AddTag(span, "hotplug", hotplug)
err = doNetNS(config.NetNSPath, func(_ ns.NetNS) error {
for _, endpoint := range endpoints {

65
src/runtime/virtcontainers/qemu.go
View File

@ -26,11 +26,9 @@ import (
"github.com/opencontainers/selinux/go-selinux/label"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"go.opentelemetry.io/otel"
otelLabel "go.opentelemetry.io/otel/label"
otelTrace "go.opentelemetry.io/otel/trace"
"golang.org/x/sys/unix"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
pkgUtils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
@ -40,6 +38,17 @@ import (
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
)
// tracingTags defines tags for the trace span
func (q *qemu) tracingTags() map[string]string {
return map[string]string{
"source": "runtime",
"package": "virtcontainers",
"subsystem": "hypervisor",
"type": "qemu",
"sandbox_id": q.id,
}
}
// romFile is the file name of the ROM that can be used for virtio-pci devices.
// If this file name is empty, this means we expect the firmware used by Qemu,
// such as SeaBIOS or OVMF for instance, to handle this directly.
@ -186,7 +195,7 @@ func (q *qemu) kernelParameters() string {
// Adds all capabilities supported by qemu implementation of hypervisor interface
func (q *qemu) capabilities(ctx context.Context) types.Capabilities {
span, _ := q.trace(ctx, "capabilities")
span, _ := katatrace.Trace(ctx, q.Logger(), "capabilities", q.tracingTags())
defer span.End()
return q.arch.capabilities()
@ -214,21 +223,9 @@ func (q *qemu) qemuPath() (string, error) {
return p, nil
}
func (q *qemu) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
if parent == nil {
q.Logger().WithField("type", "bug").Error("trace called before context set")
parent = context.Background()
}
tracer := otel.Tracer("kata")
ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "hypervisor"), otelLabel.String("type", "qemu"), otelLabel.String("sandbox_id", q.id)))
return span, ctx
}
// setup sets the Qemu structure up.
func (q *qemu) setup(ctx context.Context, id string, hypervisorConfig *HypervisorConfig) error {
span, _ := q.trace(ctx, "setup")
span, _ := katatrace.Trace(ctx, q.Logger(), "setup", q.tracingTags())
defer span.End()
err := hypervisorConfig.valid()
@ -471,7 +468,7 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa
// Save the tracing context
q.ctx = ctx
span, ctx := q.trace(ctx, "createSandbox")
span, ctx := katatrace.Trace(ctx, q.Logger(), "createSandbox", q.tracingTags())
defer span.End()
if err := q.setup(ctx, id, hypervisorConfig); err != nil {
@ -755,7 +752,7 @@ func (q *qemu) setupVirtioMem(ctx context.Context) error {
// startSandbox will start the Sandbox's VM.
func (q *qemu) startSandbox(ctx context.Context, timeout int) error {
span, ctx := q.trace(ctx, "startSandbox")
span, ctx := katatrace.Trace(ctx, q.Logger(), "startSandbox", q.tracingTags())
defer span.End()
if q.config.Debug {
@ -873,7 +870,7 @@ func (q *qemu) bootFromTemplate() error {
// waitSandbox will wait for the Sandbox's VM to be up and running.
func (q *qemu) waitSandbox(ctx context.Context, timeout int) error {
span, _ := q.trace(ctx, "waitSandbox")
span, _ := katatrace.Trace(ctx, q.Logger(), "waitSandbox", q.tracingTags())
defer span.End()
if timeout < 0 {
@ -924,7 +921,7 @@ func (q *qemu) waitSandbox(ctx context.Context, timeout int) error {
// stopSandbox will stop the Sandbox's VM.
func (q *qemu) stopSandbox(ctx context.Context, waitOnly bool) error {
span, _ := q.trace(ctx, "stopSandbox")
span, _ := katatrace.Trace(ctx, q.Logger(), "stopSandbox", q.tracingTags())
defer span.End()
q.Logger().Info("Stopping Sandbox")
@ -1012,7 +1009,7 @@ func (q *qemu) cleanupVM() error {
}
func (q *qemu) togglePauseSandbox(ctx context.Context, pause bool) error {
span, _ := q.trace(ctx, "togglePauseSandbox")
span, _ := katatrace.Trace(ctx, q.Logger(), "togglePauseSandbox", q.tracingTags())
defer span.End()
if err := q.qmpSetup(); err != nil {
@ -1613,9 +1610,9 @@ func (q *qemu) hotplugDevice(ctx context.Context, devInfo interface{}, devType d
}
func (q *qemu) hotplugAddDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
span, ctx := q.trace(ctx, "hotplugAddDevice")
span, ctx := katatrace.Trace(ctx, q.Logger(), "hotplugAddDevice", q.tracingTags())
defer span.End()
span.SetAttributes(otelLabel.Any("device", devInfo))
katatrace.AddTag(span, "device", devInfo)
data, err := q.hotplugDevice(ctx, devInfo, devType, addDevice)
if err != nil {
@ -1626,9 +1623,9 @@ func (q *qemu) hotplugAddDevice(ctx context.Context, devInfo interface{}, devTyp
}
func (q *qemu) hotplugRemoveDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
span, ctx := q.trace(ctx, "hotplugRemoveDevice")
span, ctx := katatrace.Trace(ctx, q.Logger(), "hotplugRemoveDevice", q.tracingTags())
defer span.End()
span.SetAttributes(otelLabel.Any("device", devInfo))
katatrace.AddTag(span, "device", devInfo)
data, err := q.hotplugDevice(ctx, devInfo, devType, removeDevice)
if err != nil {
@ -1839,14 +1836,14 @@ func (q *qemu) hotplugAddMemory(memDev *memoryDevice) (int, error) {
}
func (q *qemu) pauseSandbox(ctx context.Context) error {
span, ctx := q.trace(ctx, "pauseSandbox")
span, ctx := katatrace.Trace(ctx, q.Logger(), "pauseSandbox", q.tracingTags())
defer span.End()
return q.togglePauseSandbox(ctx, true)
}
func (q *qemu) resumeSandbox(ctx context.Context) error {
span, ctx := q.trace(ctx, "resumeSandbox")
span, ctx := katatrace.Trace(ctx, q.Logger(), "resumeSandbox", q.tracingTags())
defer span.End()
return q.togglePauseSandbox(ctx, false)
@ -1855,9 +1852,9 @@ func (q *qemu) resumeSandbox(ctx context.Context) error {
// addDevice will add extra devices to Qemu command line.
func (q *qemu) addDevice(ctx context.Context, devInfo interface{}, devType deviceType) error {
var err error
span, _ := q.trace(ctx, "addDevice")
span, _ := katatrace.Trace(ctx, q.Logger(), "addDevice", q.tracingTags())
defer span.End()
span.SetAttributes(otelLabel.Any("device", devInfo))
katatrace.AddTag(span, "device", devInfo)
switch v := devInfo.(type) {
case types.Volume:
@ -1914,7 +1911,7 @@ func (q *qemu) addDevice(ctx context.Context, devInfo interface{}, devType devic
// getSandboxConsole builds the path of the console where we can read
// logs coming from the sandbox.
func (q *qemu) getSandboxConsole(ctx context.Context, id string) (string, string, error) {
span, _ := q.trace(ctx, "getSandboxConsole")
span, _ := katatrace.Trace(ctx, q.Logger(), "getSandboxConsole", q.tracingTags())
defer span.End()
consoleURL, err := utils.BuildSocketPath(q.store.RunVMStoragePath(), id, consoleSocket)
@ -1979,7 +1976,7 @@ func (q *qemu) waitMigration() error {
}
func (q *qemu) disconnect(ctx context.Context) {
span, _ := q.trace(ctx, "disconnect")
span, _ := katatrace.Trace(ctx, q.Logger(), "disconnect", q.tracingTags())
defer span.End()
q.qmpShutdown()
@ -2183,7 +2180,7 @@ func genericAppendPCIeRootPort(devices []govmmQemu.Device, number uint32, machin
}
func (q *qemu) getThreadIDs(ctx context.Context) (vcpuThreadIDs, error) {
span, _ := q.trace(ctx, "getThreadIDs")
span, _ := katatrace.Trace(ctx, q.Logger(), "getThreadIDs", q.tracingTags())
defer span.End()
tid := vcpuThreadIDs{}
@ -2248,7 +2245,7 @@ func (q *qemu) resizeVCPUs(ctx context.Context, reqVCPUs uint32) (currentVCPUs u
}
func (q *qemu) cleanup(ctx context.Context) error {
span, _ := q.trace(ctx, "cleanup")
span, _ := katatrace.Trace(ctx, q.Logger(), "cleanup", q.tracingTags())
defer span.End()
for _, fd := range q.fds {

65
src/runtime/virtcontainers/sandbox.go
View File

@ -26,6 +26,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers"
@ -43,11 +44,18 @@ import (
vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
"go.opentelemetry.io/otel"
otelLabel "go.opentelemetry.io/otel/label"
otelTrace "go.opentelemetry.io/otel/trace"
)
// tracingTags defines tags for the trace span
func (s *Sandbox) tracingTags() map[string]string {
return map[string]string{
"source": "runtime",
"package": "virtcontainers",
"subsystem": "sandbox",
"sandbox_id": s.id,
}
}
const (
// vmStartTimeout represents the time in seconds a sandbox can wait before
// to consider the VM starting operation failed.
@ -127,18 +135,6 @@ type SandboxConfig struct {
Cgroups *configs.Cgroup
}
func (s *Sandbox) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
if parent == nil {
s.Logger().WithField("type", "bug").Error("trace called before context set")
parent = context.Background()
}
tracer := otel.Tracer("kata")
ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "sandbox"), otelLabel.String("sandbox_id", s.id)))
return span, ctx
}
// valid checks that the sandbox configuration is valid.
func (sandboxConfig *SandboxConfig) valid() bool {
if sandboxConfig.ID == "" {
@ -393,8 +389,9 @@ func (s *Sandbox) IOStream(containerID, processID string) (io.WriteCloser, io.Re
}
func createAssets(ctx context.Context, sandboxConfig *SandboxConfig) error {
span, _ := trace(ctx, "createAssets")
span.SetAttributes(otelLabel.String("sandbox_id", sandboxConfig.ID), otelLabel.String("subsystem", "sandbox"))
span, _ := katatrace.Trace(ctx, nil, "createAssets", nil)
katatrace.AddTag(span, "sandbox_id", sandboxConfig.ID)
katatrace.AddTag(span, "subsystem", "sandbox")
defer span.End()
for _, name := range types.AssetTypes() {
@ -444,8 +441,9 @@ func (s *Sandbox) getAndStoreGuestDetails(ctx context.Context) error {
// to physically create that sandbox i.e. starts a VM for that sandbox to eventually
// be started.
func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factory) (*Sandbox, error) {
span, ctx := trace(ctx, "createSandbox")
span.SetAttributes(otelLabel.String("sandbox_id", sandboxConfig.ID), otelLabel.String("subsystem", "sandbox"))
span, ctx := katatrace.Trace(ctx, nil, "createSandbox", nil)
katatrace.AddTag(span, "sandbox_id", sandboxConfig.ID)
katatrace.AddTag(span, "subsystem", "sandbox")
defer span.End()
if err := createAssets(ctx, &sandboxConfig); err != nil {
@ -483,8 +481,9 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac
}
func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factory) (sb *Sandbox, retErr error) {
span, ctx := trace(ctx, "newSandbox")
span.SetAttributes(otelLabel.String("sandbox_id", sandboxConfig.ID), otelLabel.String("subsystem", "sandbox"))
span, ctx := katatrace.Trace(ctx, nil, "newSandbox", nil)
katatrace.AddTag(span, "sandbox_id", sandboxConfig.ID)
katatrace.AddTag(span, "subsystem", "sandbox")
defer span.End()
if !sandboxConfig.valid() {
@ -618,7 +617,7 @@ func (s *Sandbox) createCgroupManager() error {
// storeSandbox stores a sandbox config.
func (s *Sandbox) storeSandbox(ctx context.Context) error {
span, _ := s.trace(ctx, "storeSandbox")
span, _ := katatrace.Trace(ctx, s.Logger(), "storeSandbox", s.tracingTags())
defer span.End()
// flush data to storage
@ -712,8 +711,7 @@ func (s *Sandbox) Delete(ctx context.Context) error {
}
func (s *Sandbox) startNetworkMonitor(ctx context.Context) error {
var span otelTrace.Span
span, ctx = s.trace(ctx, "startNetworkMonitor")
span, ctx := katatrace.Trace(ctx, s.Logger(), "startNetworkMonitor", s.tracingTags())
defer span.End()
binPath, err := os.Executable()
@ -752,7 +750,7 @@ func (s *Sandbox) createNetwork(ctx context.Context) error {
return nil
}
span, ctx := s.trace(ctx, "createNetwork")
span, ctx := katatrace.Trace(ctx, s.Logger(), "createNetwork", s.tracingTags())
defer span.End()
s.networkNS = NetworkNamespace{
@ -760,8 +758,8 @@ func (s *Sandbox) createNetwork(ctx context.Context) error {
NetNsCreated: s.config.NetworkConfig.NetNsCreated,
}
span.SetAttributes(otelLabel.Any("networkNS", s.networkNS))
span.SetAttributes(otelLabel.Any("NetworkConfig", s.config.NetworkConfig))
katatrace.AddTag(span, "networkNS", s.networkNS)
katatrace.AddTag(span, "NetworkConfig", s.config.NetworkConfig)
// In case there is a factory, network interfaces are hotplugged
// after vm is started.
@ -789,8 +787,7 @@ func (s *Sandbox) postCreatedNetwork(ctx context.Context) error {
}
func (s *Sandbox) removeNetwork(ctx context.Context) error {
var span otelTrace.Span
span, ctx = s.trace(ctx, "removeNetwork")
span, ctx := katatrace.Trace(ctx, s.Logger(), "removeNetwork", s.tracingTags())
defer span.End()
if s.config.NetworkConfig.NetmonConfig.Enable {
@ -997,7 +994,7 @@ func (cw *consoleWatcher) stop() {
// startVM starts the VM.
func (s *Sandbox) startVM(ctx context.Context) (err error) {
span, ctx := s.trace(ctx, "startVM")
span, ctx := katatrace.Trace(ctx, s.Logger(), "startVM", s.tracingTags())
defer span.End()
s.Logger().Info("Starting VM")
@ -1078,7 +1075,7 @@ func (s *Sandbox) startVM(ctx context.Context) (err error) {
// stopVM: stop the sandbox's VM
func (s *Sandbox) stopVM(ctx context.Context) error {
span, ctx := s.trace(ctx, "stopVM")
span, ctx := katatrace.Trace(ctx, s.Logger(), "stopVM", s.tracingTags())
defer span.End()
s.Logger().Info("Stopping sandbox in the VM")
@ -1437,7 +1434,7 @@ func (s *Sandbox) ResumeContainer(ctx context.Context, containerID string) error
// createContainers registers all containers, create the
// containers in the guest and starts one shim per container.
func (s *Sandbox) createContainers(ctx context.Context) error {
span, ctx := s.trace(ctx, "createContainers")
span, ctx := katatrace.Trace(ctx, s.Logger(), "createContainers", s.tracingTags())
defer span.End()
for i := range s.config.Containers {
@ -1509,7 +1506,7 @@ func (s *Sandbox) Start(ctx context.Context) error {
// will be destroyed.
// When force is true, ignore guest related stop failures.
func (s *Sandbox) Stop(ctx context.Context, force bool) error {
span, ctx := s.trace(ctx, "Stop")
span, ctx := katatrace.Trace(ctx, s.Logger(), "Stop", s.tracingTags())
defer span.End()
if s.state.State == types.StateStopped {
@ -1620,7 +1617,7 @@ func (s *Sandbox) unsetSandboxBlockIndex(index int) error {
// HotplugAddDevice is used for add a device to sandbox
// Sandbox implement DeviceReceiver interface from device/api/interface.go
func (s *Sandbox) HotplugAddDevice(ctx context.Context, device api.Device, devType config.DeviceType) error {
span, ctx := s.trace(ctx, "HotplugAddDevice")
span, ctx := katatrace.Trace(ctx, s.Logger(), "HotplugAddDevice", s.tracingTags())
defer span.End()
if s.config.SandboxCgroupOnly {

26
src/runtime/virtcontainers/virtiofsd.go
View File

@ -16,14 +16,19 @@ import (
"strings"
"syscall"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/label"
otelTrace "go.opentelemetry.io/otel/trace"
)
// virtiofsdTracingTags defines tags for the trace span
var virtiofsdTracingTags = map[string]string{
"source": "runtime",
"package": "virtcontainers",
"subsystem": "virtiofsd",
}
var (
errVirtiofsdDaemonPathEmpty = errors.New("virtiofsd daemon path is empty")
errVirtiofsdSocketPathEmpty = errors.New("virtiofsd socket path is empty")
@ -84,7 +89,7 @@ func (v *virtiofsd) getSocketFD() (*os.File, error) {
// Start the virtiofsd daemon
func (v *virtiofsd) Start(ctx context.Context, onQuit onQuitFunc) (int, error) {
span, _ := v.trace(ctx, "Start")
span, _ := katatrace.Trace(ctx, v.Logger(), "Start", virtiofsdTracingTags)
defer span.End()
pid := 0
@ -206,19 +211,8 @@ func (v *virtiofsd) Logger() *log.Entry {
return virtLog.WithField("subsystem", "virtiofsd")
}
func (v *virtiofsd) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
if parent == nil {
parent = context.Background()
}
tracer := otel.Tracer("kata")
ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(label.String("source", "runtime"), label.String("package", "virtcontainers"), label.String("subsystem", "virtiofsd")))
return span, ctx
}
func (v *virtiofsd) kill(ctx context.Context) (err error) {
span, _ := v.trace(ctx, "kill")
span, _ := katatrace.Trace(ctx, v.Logger(), "kill", virtiofsdTracingTags)
defer span.End()
if v.PID == 0 {