tracing: Consolidate tracing into a new katatrace package

Removes custom trace functions defined across the repo and creates
a single trace function in a new katatrace package. Also moves
span tag management into this package and provides a function to
dynamically add a tag at runtime, such as a container id, etc.

Fixes #1162

Signed-off-by: Benjamin Porter <bporter816@gmail.com>

src/runtime/containerd-shim-v2/create.go

@@ -19,7 +19,6 @@ import (
 	"github.com/containerd/typeurl"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
-	otelTrace "go.opentelemetry.io/otel/trace"
 
 	// only register the proto type
 	crioption "github.com/containerd/containerd/pkg/runtimeoptions/v1"
@@ -27,6 +26,7 @@ import (
 	_ "github.com/containerd/containerd/runtime/v2/runc/options"
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
@@ -69,19 +69,24 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
 		// create tracer
 		// This is the earliest location we can create the tracer because we must wait
 		// until the runtime config is loaded
-		_, err = katautils.CreateTracer("kata", s.config)
+		jaegerConfig := &katatrace.JaegerConfig{
+			JaegerEndpoint: s.config.JaegerEndpoint,
+			JaegerUser:     s.config.JaegerUser,
+			JaegerPassword: s.config.JaegerPassword,
+		}
+		_, err = katatrace.CreateTracer("kata", jaegerConfig)
 		if err != nil {
 			return nil, err
 		}
 
 		// create root span
-		var rootSpan otelTrace.Span
+		rootSpan, newCtx := katatrace.Trace(s.ctx, shimLog, "root span", shimTracingTags)
-		rootSpan, s.rootCtx = trace(s.ctx, "root span")
+		s.rootCtx = newCtx
 		defer rootSpan.End()
 
 		// create span
-		var span otelTrace.Span
+		span, newCtx := katatrace.Trace(s.rootCtx, shimLog, "create", shimTracingTags)
-		span, s.ctx = trace(s.rootCtx, "create")
+		s.ctx = newCtx
 		defer span.End()
 
 		if rootFs.Mounted, err = checkAndMount(s, r); err != nil {
@@ -116,8 +121,7 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
 		go s.startManagementServer(ctx, ociSpec)
 
 	case vc.PodContainer:
-		var span otelTrace.Span
+		span, ctx := katatrace.Trace(s.ctx, shimLog, "create", shimTracingTags)
-		span, ctx = trace(s.ctx, "create")
 		defer span.End()
 
 		if s.sandbox == nil {

src/runtime/containerd-shim-v2/service.go

@@ -27,18 +27,22 @@ import (
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/label"
-	otelTrace "go.opentelemetry.io/otel/trace"
 	"golang.org/x/sys/unix"
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 )
 
+// shimTracingTags defines tags for the trace span
+var shimTracingTags = map[string]string{
+	"source":  "runtime",
+	"package": "containerdshim",
+}
+
 const (
 	// Define the service's channel size, which is used for
 	// reaping the exited processes exit state and forwarding
@@ -301,19 +305,8 @@ func getTopic(e interface{}) string {
 	return cdruntime.TaskUnknownTopic
 }
 
-func trace(ctx context.Context, name string) (otelTrace.Span, context.Context) {
-	if ctx == nil {
-		logrus.WithFields(logrus.Fields{"type": "bug", "name": name}).Error("called before context set")
-		ctx = context.Background()
-	}
-	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(ctx, name, otelTrace.WithAttributes(label.String("source", "runtime"), label.String("package", "containerdshim")))
-
-	return span, ctx
-}
-
 func (s *service) Cleanup(ctx context.Context) (_ *taskAPI.DeleteResponse, err error) {
-	span, spanCtx := trace(s.rootCtx, "Cleanup")
+	span, spanCtx := katatrace.Trace(s.rootCtx, shimLog, "Cleanup", shimTracingTags)
 	defer span.End()
 
 	//Since the binary cleanup will return the DeleteResponse from stdout to
@@ -428,7 +421,7 @@ func (s *service) Create(ctx context.Context, r *taskAPI.CreateTaskRequest) (_ *
 
 // Start a process
 func (s *service) Start(ctx context.Context, r *taskAPI.StartRequest) (_ *taskAPI.StartResponse, err error) {
-	span, spanCtx := trace(s.rootCtx, "Start")
+	span, spanCtx := katatrace.Trace(s.rootCtx, shimLog, "Start", shimTracingTags)
 	defer span.End()
 
 	start := time.Now()
@@ -479,7 +472,7 @@ func (s *service) Start(ctx context.Context, r *taskAPI.StartRequest) (_ *taskAP
 
 // Delete the initial process and container
 func (s *service) Delete(ctx context.Context, r *taskAPI.DeleteRequest) (_ *taskAPI.DeleteResponse, err error) {
-	span, spanCtx := trace(s.rootCtx, "Delete")
+	span, spanCtx := katatrace.Trace(s.rootCtx, shimLog, "Delete", shimTracingTags)
 	defer span.End()
 
 	start := time.Now()
@@ -531,7 +524,7 @@ func (s *service) Delete(ctx context.Context, r *taskAPI.DeleteRequest) (_ *task
 
 // Exec an additional process inside the container
 func (s *service) Exec(ctx context.Context, r *taskAPI.ExecProcessRequest) (_ *ptypes.Empty, err error) {
-	span, _ := trace(s.rootCtx, "Exec")
+	span, _ := katatrace.Trace(s.rootCtx, shimLog, "Exec", shimTracingTags)
 	defer span.End()
 
 	start := time.Now()
@@ -569,7 +562,7 @@ func (s *service) Exec(ctx context.Context, r *taskAPI.ExecProcessRequest) (_ *p
 
 // ResizePty of a process
 func (s *service) ResizePty(ctx context.Context, r *taskAPI.ResizePtyRequest) (_ *ptypes.Empty, err error) {
-	span, spanCtx := trace(s.rootCtx, "ResizePty")
+	span, spanCtx := katatrace.Trace(s.rootCtx, shimLog, "ResizePty", shimTracingTags)
 	defer span.End()
 
 	start := time.Now()
@@ -608,7 +601,7 @@ func (s *service) ResizePty(ctx context.Context, r *taskAPI.ResizePtyRequest) (_
 
 // State returns runtime state information for a process
 func (s *service) State(ctx context.Context, r *taskAPI.StateRequest) (_ *taskAPI.StateResponse, err error) {
-	span, _ := trace(s.rootCtx, "State")
+	span, _ := katatrace.Trace(s.rootCtx, shimLog, "State", shimTracingTags)
 	defer span.End()
 
 	start := time.Now()
@@ -662,7 +655,7 @@ func (s *service) State(ctx context.Context, r *taskAPI.StateRequest) (_ *taskAP
 
 // Pause the container
 func (s *service) Pause(ctx context.Context, r *taskAPI.PauseRequest) (_ *ptypes.Empty, err error) {
-	span, spanCtx := trace(s.rootCtx, "Pause")
+	span, spanCtx := katatrace.Trace(s.rootCtx, shimLog, "Pause", shimTracingTags)
 	defer span.End()
 
 	start := time.Now()
@@ -701,7 +694,7 @@ func (s *service) Pause(ctx context.Context, r *taskAPI.PauseRequest) (_ *ptypes
 
 // Resume the container
 func (s *service) Resume(ctx context.Context, r *taskAPI.ResumeRequest) (_ *ptypes.Empty, err error) {
-	span, spanCtx := trace(s.rootCtx, "Resume")
+	span, spanCtx := katatrace.Trace(s.rootCtx, shimLog, "Resume", shimTracingTags)
 	defer span.End()
 
 	start := time.Now()
@@ -738,7 +731,7 @@ func (s *service) Resume(ctx context.Context, r *taskAPI.ResumeRequest) (_ *ptyp
 
 // Kill a process with the provided signal
 func (s *service) Kill(ctx context.Context, r *taskAPI.KillRequest) (_ *ptypes.Empty, err error) {
-	span, spanCtx := trace(s.rootCtx, "Kill")
+	span, spanCtx := katatrace.Trace(s.rootCtx, shimLog, "Kill", shimTracingTags)
 	defer span.End()
 
 	start := time.Now()
@@ -799,7 +792,7 @@ func (s *service) Kill(ctx context.Context, r *taskAPI.KillRequest) (_ *ptypes.E
 // Since for kata, it cannot get the process's pid from VM,
 // thus only return the Shim's pid directly.
 func (s *service) Pids(ctx context.Context, r *taskAPI.PidsRequest) (_ *taskAPI.PidsResponse, err error) {
-	span, _ := trace(s.rootCtx, "Pids")
+	span, _ := katatrace.Trace(s.rootCtx, shimLog, "Pids", shimTracingTags)
 	defer span.End()
 
 	var processes []*task.ProcessInfo
@@ -822,7 +815,7 @@ func (s *service) Pids(ctx context.Context, r *taskAPI.PidsRequest) (_ *taskAPI.
 
 // CloseIO of a process
 func (s *service) CloseIO(ctx context.Context, r *taskAPI.CloseIORequest) (_ *ptypes.Empty, err error) {
-	span, _ := trace(s.rootCtx, "CloseIO")
+	span, _ := katatrace.Trace(s.rootCtx, shimLog, "CloseIO", shimTracingTags)
 	defer span.End()
 
 	start := time.Now()
@@ -863,7 +856,7 @@ func (s *service) CloseIO(ctx context.Context, r *taskAPI.CloseIORequest) (_ *pt
 
 // Checkpoint the container
 func (s *service) Checkpoint(ctx context.Context, r *taskAPI.CheckpointTaskRequest) (_ *ptypes.Empty, err error) {
-	span, _ := trace(s.rootCtx, "Checkpoint")
+	span, _ := katatrace.Trace(s.rootCtx, shimLog, "Checkpoint", shimTracingTags)
 	defer span.End()
 
 	start := time.Now()
@@ -877,7 +870,7 @@ func (s *service) Checkpoint(ctx context.Context, r *taskAPI.CheckpointTaskReque
 
 // Connect returns shim information such as the shim's pid
 func (s *service) Connect(ctx context.Context, r *taskAPI.ConnectRequest) (_ *taskAPI.ConnectResponse, err error) {
-	span, _ := trace(s.rootCtx, "Connect")
+	span, _ := katatrace.Trace(s.rootCtx, shimLog, "Connect", shimTracingTags)
 	defer span.End()
 
 	start := time.Now()
@@ -897,7 +890,7 @@ func (s *service) Connect(ctx context.Context, r *taskAPI.ConnectRequest) (_ *ta
 }
 
 func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (_ *ptypes.Empty, err error) {
-	span, _ := trace(s.rootCtx, "Shutdown")
+	span, _ := katatrace.Trace(s.rootCtx, shimLog, "Shutdown", shimTracingTags)
 
 	start := time.Now()
 	defer func() {
@@ -913,7 +906,7 @@ func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (_ *
 	s.mu.Unlock()
 
 	span.End()
-	katautils.StopTracing(s.ctx)
+	katatrace.StopTracing(s.ctx)
 
 	s.cancel()
 
@@ -930,7 +923,7 @@ func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (_ *
 }
 
 func (s *service) Stats(ctx context.Context, r *taskAPI.StatsRequest) (_ *taskAPI.StatsResponse, err error) {
-	span, spanCtx := trace(s.rootCtx, "Stats")
+	span, spanCtx := katatrace.Trace(s.rootCtx, shimLog, "Stats", shimTracingTags)
 	defer span.End()
 
 	start := time.Now()
@@ -959,7 +952,7 @@ func (s *service) Stats(ctx context.Context, r *taskAPI.StatsRequest) (_ *taskAP
 
 // Update a running container
 func (s *service) Update(ctx context.Context, r *taskAPI.UpdateTaskRequest) (_ *ptypes.Empty, err error) {
-	span, spanCtx := trace(s.rootCtx, "Update")
+	span, spanCtx := katatrace.Trace(s.rootCtx, shimLog, "Update", shimTracingTags)
 	defer span.End()
 
 	start := time.Now()
@@ -991,7 +984,7 @@ func (s *service) Update(ctx context.Context, r *taskAPI.UpdateTaskRequest) (_ *
 
 // Wait for a process to exit
 func (s *service) Wait(ctx context.Context, r *taskAPI.WaitRequest) (_ *taskAPI.WaitResponse, err error) {
-	span, _ := trace(s.rootCtx, "Wait")
+	span, _ := katatrace.Trace(s.rootCtx, shimLog, "Wait", shimTracingTags)
 	defer span.End()
 
 	var ret uint32

src/runtime/pkg/katautils/config.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/BurntSushi/toml"
 	govmmQemu "github.com/kata-containers/govmm/qemu"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
@@ -28,11 +29,6 @@ const (
 	defaultHypervisor = vc.QemuHypervisor
 )
 
-var (
-	// if true, enable opentracing support.
-	tracing = false
-)
-
 // The TOML configuration file contains a number of sections (or
 // tables). The names of these tables are in dotted ("nested table")
 // form:
@@ -1116,7 +1112,7 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
 	}
 
 	config.Trace = tomlConf.Runtime.Tracing
-	tracing = config.Trace
+	katatrace.SetTracing(config.Trace)
 
 	if tomlConf.Runtime.InterNetworkModel != "" {
 		err = config.InterNetworkModel.SetModel(tomlConf.Runtime.InterNetworkModel)

src/runtime/pkg/katautils/create.go

@@ -13,13 +13,20 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	vf "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/factory"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
-	"go.opentelemetry.io/otel/label"
 )
 
+// createTracingTags defines tags for the trace span
+var createTracingTags = map[string]string{
+	"source":    "runtime",
+	"package":   "katautils",
+	"subsystem": "sandbox",
+}
+
 // GetKernelParamsFunc use a variable to allow tests to modify its value
 var GetKernelParamsFunc = getKernelParams
 
@@ -104,7 +111,8 @@ func SetEphemeralStorageType(ociSpec specs.Spec) specs.Spec {
 // CreateSandbox create a sandbox container
 func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeConfig oci.RuntimeConfig, rootFs vc.RootFs,
 	containerID, bundlePath, console string, disableOutput, systemdCgroup bool) (_ vc.VCSandbox, _ vc.Process, err error) {
-	span, ctx := Trace(ctx, "CreateSandbox", []label.KeyValue{label.Key("source").String("runtime"), label.Key("package").String("katautils"), label.Key("subsystem").String("sandbox"), label.Key("container_id").String(containerID)}...)
+	span, ctx := katatrace.Trace(ctx, nil, "CreateSandbox", createTracingTags)
+	katatrace.AddTag(span, "container_id", containerID)
 	defer span.End()
 
 	sandboxConfig, err := oci.SandboxConfig(ociSpec, runtimeConfig, bundlePath, containerID, console, disableOutput, systemdCgroup)
@@ -159,7 +167,7 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeCo
 
 	sid := sandbox.ID()
 	kataUtilsLogger = kataUtilsLogger.WithField("sandbox", sid)
-	span.SetAttributes(label.Key("sandbox_id").String(sid))
+	katatrace.AddTag(span, "sandbox_id", sid)
 
 	containers := sandbox.GetAllContainers()
 	if len(containers) != 1 {
@@ -202,7 +210,8 @@ func checkForFIPS(sandboxConfig *vc.SandboxConfig) error {
 func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Spec, rootFs vc.RootFs, containerID, bundlePath, console string, disableOutput bool) (vc.Process, error) {
 	var c vc.VCContainer
 
-	span, ctx := Trace(ctx, "CreateContainer", []label.KeyValue{label.Key("source").String("runtime"), label.Key("package").String("katautils"), label.Key("subsystem").String("sandbox"), label.Key("container_id").String(containerID)}...)
+	span, ctx := katatrace.Trace(ctx, nil, "CreateContainer", createTracingTags)
+	katatrace.AddTag(span, "container_id", containerID)
 	defer span.End()
 
 	ociSpec = SetEphemeralStorageType(ociSpec)
@@ -228,7 +237,7 @@ func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Sp
 		return vc.Process{}, err
 	}
 
-	span.SetAttributes(label.Key("sandbox_id").String(sandboxID))
+	katatrace.AddTag(span, "sandbox_id", sandboxID)
 
 	c, err = sandbox.CreateContainer(ctx, contConfig)
 	if err != nil {

src/runtime/pkg/katautils/hook.go

@@ -15,18 +15,25 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
-	"go.opentelemetry.io/otel/label"
 )
 
+// hookTracingTags defines tags for the trace span
+var hookTracingTags = map[string]string{
+	"source":    "runtime",
+	"package":   "katautils",
+	"subsystem": "hook",
+}
+
 // Logger returns a logrus logger appropriate for logging hook messages
 func hookLogger() *logrus.Entry {
 	return kataUtilsLogger.WithField("subsystem", "hook")
 }
 
 func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error {
-	span, _ := Trace(ctx, "runHook", []label.KeyValue{label.Key("source").String("runtime"), label.Key("package").String("katautils"), label.Key("subsystem").String("hook")}...)
+	span, _ := katatrace.Trace(ctx, hookLogger(), "runHook", hookTracingTags)
 	defer span.End()
 
 	// FIXME
@@ -88,7 +95,8 @@ func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error
 }
 
 func runHooks(ctx context.Context, hooks []specs.Hook, cid, bundlePath, hookType string) error {
-	span, ctx := Trace(ctx, "runHooks", []label.KeyValue{label.Key("source").String("runtime"), label.Key("package").String("katautils"), label.Key("subsystem").String("hook"), label.Key("type").String(hookType)}...)
+	span, _ := katatrace.Trace(ctx, hookLogger(), "runHooks", hookTracingTags)
+	katatrace.AddTag(span, "type", hookType)
 	defer span.End()
 
 	for _, hook := range hooks {

src/runtime/pkg/katautils/katatrace/tracing.go

@@ -3,12 +3,12 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-package katautils
+package katatrace
 
 import (
 	"context"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
+	"github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/exporters/trace/jaeger"
 	"go.opentelemetry.io/otel/label"
@@ -30,7 +30,7 @@ var _ export.SpanExporter = (*kataSpanExporter)(nil)
 // ExportSpans exports SpanData to Jaeger.
 func (e *kataSpanExporter) ExportSpans(ctx context.Context, spans []*export.SpanData) error {
 	for _, span := range spans {
-		kataUtilsLogger.Tracef("Reporting span %+v", span)
+		kataTraceLogger.Tracef("Reporting span %+v", span)
 	}
 	return nil
 }
@@ -43,8 +43,25 @@ func (e *kataSpanExporter) Shutdown(ctx context.Context) error {
 // is used by stopTracing().
 var tracerCloser func()
 
+var kataTraceLogger = logrus.NewEntry(logrus.New())
+
+// tracing determines whether tracing is enabled.
+var tracing bool
+
+// SetTracing turns tracing on or off. Called by the configuration.
+func SetTracing(isTracing bool) {
+	tracing = isTracing
+}
+
+// JaegerConfig defines necessary Jaeger config for exporting traces.
+type JaegerConfig struct {
+	JaegerEndpoint string
+	JaegerUser     string
+	JaegerPassword string
+}
+
 // CreateTracer create a tracer
-func CreateTracer(name string, config *oci.RuntimeConfig) (func(), error) {
+func CreateTracer(name string, config *JaegerConfig) (func(), error) {
 	if !tracing {
 		otel.SetTracerProvider(trace.NewNoopTracerProvider())
 		return func() {}, nil
@@ -109,12 +126,25 @@ func StopTracing(ctx context.Context) {
 	}
 }
 
-// Trace creates a new tracing span based on the specified name and parent
+// Trace creates a new tracing span based on the specified name and parent context.
-// context and an opentelemetry label.KeyValue slice for span attributes.
+// It also accepts a logger to record nil context errors and a map of tracing tags.
-func Trace(parent context.Context, name string, tags ...label.KeyValue) (otelTrace.Span, context.Context) {
+// Tracing tag keys and values are strings.
+func Trace(parent context.Context, logger *logrus.Entry, name string, tags map[string]string) (otelTrace.Span, context.Context) {
+	if parent == nil {
+		if logger == nil {
+			logger = kataTraceLogger
+		}
+		logger.WithField("type", "bug").Error("trace called before context set")
+		parent = context.Background()
+	}
+
+	var otelTags []label.KeyValue
+	for k, v := range tags {
+		otelTags = append(otelTags, label.Key(k).String(v))
+	}
 
 	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(tags...))
+	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelTags...))
 
 	// This is slightly confusing: when tracing is disabled, trace spans
 	// are still created - but the tracer used is a NOP. Therefore, only
@@ -122,8 +152,14 @@ func Trace(parent context.Context, name string, tags ...label.KeyValue) (otelTra
 	if tracing {
 		// This log message is *essential*: it is used by:
 		// https: //github.com/kata-containers/tests/blob/master/tracing/tracing-test.sh
-		kataUtilsLogger.Debugf("created span %v", span)
+		kataTraceLogger.Debugf("created span %v", span)
 	}
 
 	return span, ctx
 }
+
+// AddTag adds an additional key-value pair to a tracing span. This can be used to
+// provide dynamic tags that are determined at runtime.
+func AddTag(span otelTrace.Span, key string, value interface{}) {
+	span.SetAttributes(label.Any(key, value))
+}

src/runtime/virtcontainers/acrn.go

@@ -19,10 +19,8 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/label"
-	otelTrace "go.opentelemetry.io/otel/trace"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
 	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
@@ -31,6 +29,17 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
+// tracingTags defines tags for the trace span
+func (a *Acrn) tracingTags() map[string]string {
+	return map[string]string{
+		"source":     "runtime",
+		"package":    "virtcontainers",
+		"subsystem":  "hypervisor",
+		"type":       "acrn",
+		"sandbox_id": a.id,
+	}
+}
+
 // Since ACRN is using the store in a quite abnormal way, let's first draw it back from store to here
 
 // UUIDPathSuffix is the suffix used for uuid storage
@@ -150,7 +159,7 @@ func (a *Acrn) kernelParameters() string {
 
 // Adds all capabilities supported by Acrn implementation of hypervisor interface
 func (a *Acrn) capabilities(ctx context.Context) types.Capabilities {
-	span, _ := a.trace(ctx, "capabilities")
+	span, _ := katatrace.Trace(ctx, a.Logger(), "capabilities", a.tracingTags())
 	defer span.End()
 
 	return a.arch.capabilities()
@@ -207,18 +216,6 @@ func (a *Acrn) Logger() *logrus.Entry {
 	return virtLog.WithField("subsystem", "acrn")
 }
 
-func (a *Acrn) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
-	if parent == nil {
-		a.Logger().WithField("type", "bug").Error("trace called before context set")
-		parent = context.Background()
-	}
-
-	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(label.String("source", "runtime"), label.String("package", "virtcontainers"), label.String("subsystem", "hypervisor"), label.String("type", "acrn"), label.String("sandbox_id", a.id)))
-
-	return span, ctx
-}
-
 func (a *Acrn) memoryTopology() (Memory, error) {
 	memMb := uint64(a.config.MemorySize)
 
@@ -287,7 +284,7 @@ func (a *Acrn) buildDevices(ctx context.Context, imagePath string) ([]Device, er
 
 // setup sets the Acrn structure up.
 func (a *Acrn) setup(ctx context.Context, id string, hypervisorConfig *HypervisorConfig) error {
-	span, _ := a.trace(ctx, "setup")
+	span, _ := katatrace.Trace(ctx, a.Logger(), "setup", a.tracingTags())
 	defer span.End()
 
 	err := hypervisorConfig.valid()
@@ -330,7 +327,7 @@ func (a *Acrn) setup(ctx context.Context, id string, hypervisorConfig *Hyperviso
 }
 
 func (a *Acrn) createDummyVirtioBlkDev(ctx context.Context, devices []Device) ([]Device, error) {
-	span, _ := a.trace(ctx, "createDummyVirtioBlkDev")
+	span, _ := katatrace.Trace(ctx, a.Logger(), "createDummyVirtioBlkDev", a.tracingTags())
 	defer span.End()
 
 	// Since acrn doesn't support hot-plug, dummy virtio-blk
@@ -353,8 +350,7 @@ func (a *Acrn) createSandbox(ctx context.Context, id string, networkNS NetworkNa
 	// Save the tracing context
 	a.ctx = ctx
 
-	var span otelTrace.Span
+	span, ctx := katatrace.Trace(ctx, a.Logger(), "createSandbox", a.tracingTags())
-	span, ctx = a.trace(ctx, "createSandbox")
 	defer span.End()
 
 	if err := a.setup(ctx, id, hypervisorConfig); err != nil {
@@ -419,7 +415,7 @@ func (a *Acrn) createSandbox(ctx context.Context, id string, networkNS NetworkNa
 
 // startSandbox will start the Sandbox's VM.
 func (a *Acrn) startSandbox(ctx context.Context, timeoutSecs int) error {
-	span, ctx := a.trace(ctx, "startSandbox")
+	span, ctx := katatrace.Trace(ctx, a.Logger(), "startSandbox", a.tracingTags())
 	defer span.End()
 
 	if a.config.Debug {
@@ -465,7 +461,7 @@ func (a *Acrn) startSandbox(ctx context.Context, timeoutSecs int) error {
 
 // waitSandbox will wait for the Sandbox's VM to be up and running.
 func (a *Acrn) waitSandbox(ctx context.Context, timeoutSecs int) error {
-	span, _ := a.trace(ctx, "waitSandbox")
+	span, _ := katatrace.Trace(ctx, a.Logger(), "waitSandbox", a.tracingTags())
 	defer span.End()
 
 	if timeoutSecs < 0 {
@@ -479,7 +475,7 @@ func (a *Acrn) waitSandbox(ctx context.Context, timeoutSecs int) error {
 
 // stopSandbox will stop the Sandbox's VM.
 func (a *Acrn) stopSandbox(ctx context.Context, waitOnly bool) (err error) {
-	span, _ := a.trace(ctx, "stopSandbox")
+	span, _ := katatrace.Trace(ctx, a.Logger(), "stopSandbox", a.tracingTags())
 	defer span.End()
 
 	a.Logger().Info("Stopping acrn VM")
@@ -547,7 +543,7 @@ func (a *Acrn) updateBlockDevice(drive *config.BlockDrive) error {
 }
 
 func (a *Acrn) hotplugAddDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
-	span, _ := a.trace(ctx, "hotplugAddDevice")
+	span, _ := katatrace.Trace(ctx, a.Logger(), "hotplugAddDevice", a.tracingTags())
 	defer span.End()
 
 	switch devType {
@@ -561,7 +557,7 @@ func (a *Acrn) hotplugAddDevice(ctx context.Context, devInfo interface{}, devTyp
 }
 
 func (a *Acrn) hotplugRemoveDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
-	span, _ := a.trace(ctx, "hotplugRemoveDevice")
+	span, _ := katatrace.Trace(ctx, a.Logger(), "hotplugRemoveDevice", a.tracingTags())
 	defer span.End()
 
 	// Not supported. return success
@@ -570,7 +566,7 @@ func (a *Acrn) hotplugRemoveDevice(ctx context.Context, devInfo interface{}, dev
 }
 
 func (a *Acrn) pauseSandbox(ctx context.Context) error {
-	span, _ := a.trace(ctx, "pauseSandbox")
+	span, _ := katatrace.Trace(ctx, a.Logger(), "pauseSandbox", a.tracingTags())
 	defer span.End()
 
 	// Not supported. return success
@@ -579,7 +575,7 @@ func (a *Acrn) pauseSandbox(ctx context.Context) error {
 }
 
 func (a *Acrn) resumeSandbox(ctx context.Context) error {
-	span, _ := a.trace(ctx, "resumeSandbox")
+	span, _ := katatrace.Trace(ctx, a.Logger(), "resumeSandbox", a.tracingTags())
 	defer span.End()
 
 	// Not supported. return success
@@ -590,7 +586,7 @@ func (a *Acrn) resumeSandbox(ctx context.Context) error {
 // addDevice will add extra devices to acrn command line.
 func (a *Acrn) addDevice(ctx context.Context, devInfo interface{}, devType deviceType) error {
 	var err error
-	span, _ := a.trace(ctx, "addDevice")
+	span, _ := katatrace.Trace(ctx, a.Logger(), "addDevice", a.tracingTags())
 	defer span.End()
 
 	switch v := devInfo.(type) {
@@ -623,7 +619,7 @@ func (a *Acrn) addDevice(ctx context.Context, devInfo interface{}, devType devic
 // getSandboxConsole builds the path of the console where we can read
 // logs coming from the sandbox.
 func (a *Acrn) getSandboxConsole(ctx context.Context, id string) (string, string, error) {
-	span, _ := a.trace(ctx, "getSandboxConsole")
+	span, _ := katatrace.Trace(ctx, a.Logger(), "getSandboxConsole", a.tracingTags())
 	defer span.End()
 
 	consoleURL, err := utils.BuildSocketPath(a.store.RunVMStoragePath(), id, acrnConsoleSocket)
@@ -643,14 +639,14 @@ func (a *Acrn) saveSandbox() error {
 }
 
 func (a *Acrn) disconnect(ctx context.Context) {
-	span, _ := a.trace(ctx, "disconnect")
+	span, _ := katatrace.Trace(ctx, a.Logger(), "disconnect", a.tracingTags())
 	defer span.End()
 
 	// Not supported.
 }
 
 func (a *Acrn) getThreadIDs(ctx context.Context) (vcpuThreadIDs, error) {
-	span, _ := a.trace(ctx, "getThreadIDs")
+	span, _ := katatrace.Trace(ctx, a.Logger(), "getThreadIDs", a.tracingTags())
 	defer span.End()
 
 	// Not supported. return success
@@ -668,7 +664,7 @@ func (a *Acrn) resizeVCPUs(ctx context.Context, reqVCPUs uint32) (currentVCPUs u
 }
 
 func (a *Acrn) cleanup(ctx context.Context) error {
-	span, _ := a.trace(ctx, "cleanup")
+	span, _ := katatrace.Trace(ctx, a.Logger(), "cleanup", a.tracingTags())
 	defer span.End()
 
 	return nil

src/runtime/virtcontainers/api.go

@@ -9,32 +9,28 @@ import (
 	"context"
 	"runtime"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	deviceApi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
 	deviceConfig "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/cgroups"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 	"github.com/sirupsen/logrus"
-	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/label"
-	otelTrace "go.opentelemetry.io/otel/trace"
 )
 
+// apiTracingTags defines tags for the trace span
+var apiTracingTags = map[string]string{
+	"source":    "runtime",
+	"packages":  "virtcontainers",
+	"subsystem": "api",
+}
+
 func init() {
 	runtime.LockOSThread()
 }
 
 var virtLog = logrus.WithField("source", "virtcontainers")
 
-// trace creates a new tracing span based on the specified name and parent
-// context.
-func trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
-	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(label.String("source", "runtime"), label.String("packages", "virtcontainers"), label.String("subsystem", "api")))
-
-	return span, ctx
-}
-
 // SetLogger sets the logger for virtcontainers package.
 func SetLogger(ctx context.Context, logger *logrus.Entry) {
 	fields := virtLog.Data
@@ -49,7 +45,7 @@ func SetLogger(ctx context.Context, logger *logrus.Entry) {
 // CreateSandbox is the virtcontainers sandbox creation entry point.
 // CreateSandbox creates a sandbox and its containers. It does not start them.
 func CreateSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factory) (VCSandbox, error) {
-	span, ctx := trace(ctx, "CreateSandbox")
+	span, ctx := katatrace.Trace(ctx, virtLog, "CreateSandbox", apiTracingTags)
 	defer span.End()
 
 	s, err := createSandboxFromConfig(ctx, sandboxConfig, factory)
@@ -58,7 +54,7 @@ func CreateSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac
 }
 
 func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, factory Factory) (_ *Sandbox, err error) {
-	span, ctx := trace(ctx, "createSandboxFromConfig")
+	span, ctx := katatrace.Trace(ctx, virtLog, "createSandboxFromConfig", apiTracingTags)
 	defer span.End()
 
 	// Create the sandbox.
@@ -127,7 +123,7 @@ func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, f
 // in the sandbox left, do stop the sandbox and delete it. Those serial operations will be done exclusively by
 // locking the sandbox.
 func CleanupContainer(ctx context.Context, sandboxID, containerID string, force bool) error {
-	span, ctx := trace(ctx, "CleanupContainer")
+	span, ctx := katatrace.Trace(ctx, virtLog, "CleanupContainer", apiTracingTags)
 	defer span.End()
 
 	if sandboxID == "" {

src/runtime/virtcontainers/clh.go

@@ -25,16 +25,25 @@ import (
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
-	"go.opentelemetry.io/otel"
-	otelLabel "go.opentelemetry.io/otel/label"
-	otelTrace "go.opentelemetry.io/otel/trace"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
+// tracingTags defines tags for the trace span
+func (clh *cloudHypervisor) tracingTags() map[string]string {
+	return map[string]string{
+		"source":     "runtime",
+		"package":    "virtcontainers",
+		"subsystem":  "hypervisor",
+		"type":       "clh",
+		"sandbox_id": clh.id,
+	}
+}
+
 //
 // Constants and type definitions related to cloud hypervisor
 //
@@ -146,8 +155,8 @@ var clhDebugKernelParams = []Param{
 func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error {
 	clh.ctx = ctx
 
-	var span otelTrace.Span
+	span, newCtx := katatrace.Trace(clh.ctx, clh.Logger(), "createSandbox", clh.tracingTags())
-	span, clh.ctx = clh.trace(clh.ctx, "createSandbox")
+	clh.ctx = newCtx
 	defer span.End()
 
 	err := hypervisorConfig.valid()
@@ -295,7 +304,7 @@ func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networ
 
 // startSandbox will start the VMM and boot the virtual machine for the given sandbox.
 func (clh *cloudHypervisor) startSandbox(ctx context.Context, timeout int) error {
-	span, _ := clh.trace(ctx, "startSandbox")
+	span, _ := katatrace.Trace(ctx, clh.Logger(), "startSandbox", clh.tracingTags())
 	defer span.End()
 
 	ctx, cancel := context.WithTimeout(context.Background(), clhAPITimeout*time.Second)
@@ -454,7 +463,7 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device config.VFIODev) error {
 }
 
 func (clh *cloudHypervisor) hotplugAddDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
-	span, _ := clh.trace(ctx, "hotplugAddDevice")
+	span, _ := katatrace.Trace(ctx, clh.Logger(), "hotplugAddDevice", clh.tracingTags())
 	defer span.End()
 
 	switch devType {
@@ -471,7 +480,7 @@ func (clh *cloudHypervisor) hotplugAddDevice(ctx context.Context, devInfo interf
 }
 
 func (clh *cloudHypervisor) hotplugRemoveDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
-	span, _ := clh.trace(ctx, "hotplugRemoveDevice")
+	span, _ := katatrace.Trace(ctx, clh.Logger(), "hotplugRemoveDevice", clh.tracingTags())
 	defer span.End()
 
 	var deviceID string
@@ -631,7 +640,7 @@ func (clh *cloudHypervisor) resumeSandbox(ctx context.Context) error {
 
 // stopSandbox will stop the Sandbox's VM.
 func (clh *cloudHypervisor) stopSandbox(ctx context.Context, waitOnly bool) (err error) {
-	span, _ := clh.trace(ctx, "stopSandbox")
+	span, _ := katatrace.Trace(ctx, clh.Logger(), "stopSandbox", clh.tracingTags())
 	defer span.End()
 	clh.Logger().WithField("function", "stopSandbox").Info("Stop Sandbox")
 	return clh.terminate(ctx, waitOnly)
@@ -681,7 +690,7 @@ func (clh *cloudHypervisor) getVirtioFsPid() *int {
 }
 
 func (clh *cloudHypervisor) addDevice(ctx context.Context, devInfo interface{}, devType deviceType) error {
-	span, _ := clh.trace(ctx, "addDevice")
+	span, _ := katatrace.Trace(ctx, clh.Logger(), "addDevice", clh.tracingTags())
 	defer span.End()
 
 	var err error
@@ -715,7 +724,7 @@ func (clh *cloudHypervisor) Logger() *log.Entry {
 
 // Adds all capabilities supported by cloudHypervisor implementation of hypervisor interface
 func (clh *cloudHypervisor) capabilities(ctx context.Context) types.Capabilities {
-	span, _ := clh.trace(ctx, "capabilities")
+	span, _ := katatrace.Trace(ctx, clh.Logger(), "capabilities", clh.tracingTags())
 	defer span.End()
 
 	clh.Logger().WithField("function", "capabilities").Info("get Capabilities")
@@ -725,20 +734,8 @@ func (clh *cloudHypervisor) capabilities(ctx context.Context) types.Capabilities
 	return caps
 }
 
-func (clh *cloudHypervisor) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
-	if parent == nil {
-		clh.Logger().WithField("type", "bug").Error("trace called before context set")
-		parent = context.Background()
-	}
-
-	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "hypervisor"), otelLabel.String("type", "clh"), otelLabel.String("sandbox_id", clh.id)))
-
-	return span, ctx
-}
-
 func (clh *cloudHypervisor) terminate(ctx context.Context, waitOnly bool) (err error) {
-	span, _ := clh.trace(ctx, "terminate")
+	span, _ := katatrace.Trace(ctx, clh.Logger(), "terminate", clh.tracingTags())
 	defer span.End()
 
 	pid := clh.state.PID

src/runtime/virtcontainers/container.go

@@ -16,6 +16,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"
@@ -24,9 +25,6 @@ import (
 	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
-	"go.opentelemetry.io/otel"
-	otelLabel "go.opentelemetry.io/otel/label"
-	otelTrace "go.opentelemetry.io/otel/trace"
 
 	"github.com/containerd/cgroups"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -35,6 +33,16 @@ import (
 	"golang.org/x/sys/unix"
 )
 
+// tracingTags defines tags for the trace span
+func (c *Container) tracingTags() map[string]string {
+	return map[string]string{
+		"source":       "runtime",
+		"package":      "virtcontainers",
+		"subsystem":    "container",
+		"container_id": c.id,
+	}
+}
+
 // https://github.com/torvalds/linux/blob/master/include/uapi/linux/major.h
 // This file has definitions for major device numbers.
 var cdromMajors = map[int64]string{
@@ -354,18 +362,6 @@ func (c *Container) Logger() *logrus.Entry {
 	})
 }
 
-func (c *Container) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
-	if parent == nil {
-		c.Logger().WithField("type", "bug").Error("trace called before context set")
-		parent = context.Background()
-	}
-
-	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "container"), otelLabel.String("container_id", c.id)))
-
-	return span, ctx
-}
-
 // Sandbox returns the sandbox handler related to this container.
 func (c *Container) Sandbox() VCSandbox {
 	return c.sandbox
@@ -622,14 +618,13 @@ func (c *Container) mountSharedDirMounts(ctx context.Context, sharedDirMounts, i
 }
 
 func (c *Container) unmountHostMounts(ctx context.Context) error {
-	var span otelTrace.Span
+	span, ctx := katatrace.Trace(ctx, c.Logger(), "unmountHostMounts", c.tracingTags())
-	span, ctx = c.trace(ctx, "unmountHostMounts")
 	defer span.End()
 
 	for _, m := range c.mounts {
 		if m.HostPath != "" {
-			span, _ := c.trace(ctx, "unmount")
+			span, _ := katatrace.Trace(ctx, c.Logger(), "unmount", c.tracingTags())
-			span.SetAttributes(otelLabel.Key("host-path").String(m.HostPath))
+			katatrace.AddTag(span, "host-path", m.HostPath)
 
 			if err := syscall.Unmount(m.HostPath, syscall.MNT_DETACH|UmountNoFollow); err != nil {
 				c.Logger().WithFields(logrus.Fields{
@@ -751,7 +746,7 @@ func (c *Container) createBlockDevices(ctx context.Context) error {
 
 // newContainer creates a Container structure from a sandbox and a container configuration.
 func newContainer(ctx context.Context, sandbox *Sandbox, contConfig *ContainerConfig) (*Container, error) {
-	span, ctx := sandbox.trace(ctx, "newContainer")
+	span, ctx := katatrace.Trace(ctx, sandbox.Logger(), "newContainer", sandbox.tracingTags())
 	defer span.End()
 
 	if !contConfig.valid() {
@@ -1021,8 +1016,7 @@ func (c *Container) start(ctx context.Context) error {
 }
 
 func (c *Container) stop(ctx context.Context, force bool) error {
-	var span otelTrace.Span
+	span, ctx := katatrace.Trace(ctx, c.Logger(), "stop", c.tracingTags())
-	span, ctx = c.trace(ctx, "stop")
 	defer span.End()
 
 	// In case the container status has been updated implicitly because

src/runtime/virtcontainers/factory/factory.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	pb "github.com/kata-containers/kata-containers/src/runtime/protocols/cache"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/factory/base"
@@ -18,11 +19,15 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/factory/template"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 	"github.com/sirupsen/logrus"
-	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/label"
-	otelTrace "go.opentelemetry.io/otel/trace"
 )
 
+// factoryTracingTags defines tags for the trace span
+var factoryTracingTags = map[string]string{
+	"source":    "runtime",
+	"package":   "factory",
+	"subsystem": "factory",
+}
+
 var factoryLogger = logrus.FieldLogger(logrus.New())
 
 // Config is a collection of VM factory configurations.
@@ -40,16 +45,9 @@ type factory struct {
 	base base.FactoryBase
 }
 
-func trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
-	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(label.String("source", "runtime"), label.String("package", "factory"), label.String("subsystem", "factory")))
-
-	return span, ctx
-}
-
 // NewFactory returns a working factory.
 func NewFactory(ctx context.Context, config Config, fetchOnly bool) (vc.Factory, error) {
-	span, _ := trace(ctx, "NewFactory")
+	span, _ := katatrace.Trace(ctx, nil, "NewFactory", factoryTracingTags)
 	defer span.End()
 
 	err := config.VMConfig.Valid()
@@ -140,7 +138,7 @@ func (f *factory) checkConfig(config vc.VMConfig) error {
 
 // GetVM returns a working blank VM created by the factory.
 func (f *factory) GetVM(ctx context.Context, config vc.VMConfig) (*vc.VM, error) {
-	span, ctx := trace(ctx, "GetVM")
+	span, ctx := katatrace.Trace(ctx, f.log(), "GetVM", factoryTracingTags)
 	defer span.End()
 
 	hypervisorConfig := config.HypervisorConfig

src/runtime/virtcontainers/fc.go

@@ -22,6 +22,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client"
@@ -38,11 +39,19 @@ import (
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-	"go.opentelemetry.io/otel"
-	otelLabel "go.opentelemetry.io/otel/label"
-	otelTrace "go.opentelemetry.io/otel/trace"
 )
 
+// tracingTags defines tags for the trace span
+func (fc *firecracker) tracingTags() map[string]string {
+	return map[string]string{
+		"source":     "runtime",
+		"package":    "virtcontainers",
+		"subsystem":  "hypervisor",
+		"type":       "firecracker",
+		"sandbox_id": fc.id,
+	}
+}
+
 type vmmState uint8
 
 const (
@@ -168,18 +177,6 @@ func (fc *firecracker) Logger() *logrus.Entry {
 	return virtLog.WithField("subsystem", "firecracker")
 }
 
-func (fc *firecracker) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
-	if parent == nil {
-		fc.Logger().WithField("type", "bug").Error("trace called before context set")
-		parent = context.Background()
-	}
-
-	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "hypervisor"), otelLabel.String("type", "firecracker"), otelLabel.String("sandbox_id", fc.id)))
-
-	return span, ctx
-}
-
 //At some cases, when sandbox id is too long, it will incur error of overlong
 //firecracker API unix socket(fc.socketPath).
 //In Linux, sun_path could maximumly contains 108 bytes in size.
@@ -198,8 +195,7 @@ func (fc *firecracker) truncateID(id string) string {
 func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error {
 	fc.ctx = ctx
 
-	var span otelTrace.Span
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "createSandbox", fc.tracingTags())
-	span, _ = fc.trace(ctx, "createSandbox")
 	defer span.End()
 
 	//TODO: check validity of the hypervisor config provided
@@ -242,7 +238,7 @@ func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS N
 }
 
 func (fc *firecracker) newFireClient(ctx context.Context) *client.Firecracker {
-	span, _ := fc.trace(ctx, "newFireClient")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "newFireClient", fc.tracingTags())
 	defer span.End()
 	httpClient := client.NewHTTPClient(strfmt.NewFormats())
 
@@ -312,7 +308,7 @@ func (fc *firecracker) checkVersion(version string) error {
 
 // waitVMMRunning will wait for timeout seconds for the VMM to be up and running.
 func (fc *firecracker) waitVMMRunning(ctx context.Context, timeout int) error {
-	span, _ := fc.trace(ctx, "wait VMM to be running")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "wait VMM to be running", fc.tracingTags())
 	defer span.End()
 
 	if timeout < 0 {
@@ -334,7 +330,7 @@ func (fc *firecracker) waitVMMRunning(ctx context.Context, timeout int) error {
 }
 
 func (fc *firecracker) fcInit(ctx context.Context, timeout int) error {
-	span, _ := fc.trace(ctx, "fcInit")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "fcInit", fc.tracingTags())
 	defer span.End()
 
 	var err error
@@ -409,7 +405,7 @@ func (fc *firecracker) fcInit(ctx context.Context, timeout int) error {
 }
 
 func (fc *firecracker) fcEnd(ctx context.Context, waitOnly bool) (err error) {
-	span, _ := fc.trace(ctx, "fcEnd")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "fcEnd", fc.tracingTags())
 	defer span.End()
 
 	fc.Logger().Info("Stopping firecracker VM")
@@ -436,7 +432,7 @@ func (fc *firecracker) fcEnd(ctx context.Context, waitOnly bool) (err error) {
 }
 
 func (fc *firecracker) client(ctx context.Context) *client.Firecracker {
-	span, _ := fc.trace(ctx, "client")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "client", fc.tracingTags())
 	defer span.End()
 
 	if fc.connection == nil {
@@ -503,7 +499,7 @@ func (fc *firecracker) fcJailResource(src, dst string) (string, error) {
 }
 
 func (fc *firecracker) fcSetBootSource(ctx context.Context, path, params string) error {
-	span, _ := fc.trace(ctx, "fcSetBootSource")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "fcSetBootSource", fc.tracingTags())
 	defer span.End()
 	fc.Logger().WithFields(logrus.Fields{"kernel-path": path,
 		"kernel-params": params}).Debug("fcSetBootSource")
@@ -524,7 +520,7 @@ func (fc *firecracker) fcSetBootSource(ctx context.Context, path, params string)
 }
 
 func (fc *firecracker) fcSetVMRootfs(ctx context.Context, path string) error {
-	span, _ := fc.trace(ctx, "fcSetVMRootfs")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "fcSetVMRootfs", fc.tracingTags())
 	defer span.End()
 
 	jailedRootfs, err := fc.fcJailResource(path, fcRootfs)
@@ -551,7 +547,7 @@ func (fc *firecracker) fcSetVMRootfs(ctx context.Context, path string) error {
 }
 
 func (fc *firecracker) fcSetVMBaseConfig(ctx context.Context, mem int64, vcpus int64, htEnabled bool) {
-	span, _ := fc.trace(ctx, "fcSetVMBaseConfig")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "fcSetVMBaseConfig", fc.tracingTags())
 	defer span.End()
 	fc.Logger().WithFields(logrus.Fields{"mem": mem,
 		"vcpus":     vcpus,
@@ -567,7 +563,7 @@ func (fc *firecracker) fcSetVMBaseConfig(ctx context.Context, mem int64, vcpus i
 }
 
 func (fc *firecracker) fcSetLogger(ctx context.Context) error {
-	span, _ := fc.trace(ctx, "fcSetLogger")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "fcSetLogger", fc.tracingTags())
 	defer span.End()
 
 	fcLogLevel := "Error"
@@ -590,7 +586,7 @@ func (fc *firecracker) fcSetLogger(ctx context.Context) error {
 }
 
 func (fc *firecracker) fcSetMetrics(ctx context.Context) error {
-	span, _ := fc.trace(ctx, "fcSetMetrics")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "fcSetMetrics", fc.tracingTags())
 	defer span.End()
 
 	// listen to metrics file and transfer error info
@@ -745,7 +741,7 @@ func (fc *firecracker) fcInitConfiguration(ctx context.Context) error {
 // In the context of firecracker, this will start the hypervisor,
 // for configuration, but not yet start the actual virtual machine
 func (fc *firecracker) startSandbox(ctx context.Context, timeout int) error {
-	span, _ := fc.trace(ctx, "startSandbox")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "startSandbox", fc.tracingTags())
 	defer span.End()
 
 	if err := fc.fcInitConfiguration(ctx); err != nil {
@@ -797,7 +793,7 @@ func fcDriveIndexToID(i int) string {
 }
 
 func (fc *firecracker) createDiskPool(ctx context.Context) error {
-	span, _ := fc.trace(ctx, "createDiskPool")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "createDiskPool", fc.tracingTags())
 	defer span.End()
 
 	for i := 0; i < fcDiskPoolSize; i++ {
@@ -835,7 +831,7 @@ func (fc *firecracker) umountResource(jailedPath string) {
 
 // cleanup all jail artifacts
 func (fc *firecracker) cleanupJail(ctx context.Context) {
-	span, _ := fc.trace(ctx, "cleanupJail")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "cleanupJail", fc.tracingTags())
 	defer span.End()
 
 	fc.umountResource(fcKernel)
@@ -858,7 +854,7 @@ func (fc *firecracker) cleanupJail(ctx context.Context) {
 
 // stopSandbox will stop the Sandbox's VM.
 func (fc *firecracker) stopSandbox(ctx context.Context, waitOnly bool) (err error) {
-	span, _ := fc.trace(ctx, "stopSandbox")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "stopSandbox", fc.tracingTags())
 	defer span.End()
 
 	return fc.fcEnd(ctx, waitOnly)
@@ -877,7 +873,7 @@ func (fc *firecracker) resumeSandbox(ctx context.Context) error {
 }
 
 func (fc *firecracker) fcAddVsock(ctx context.Context, hvs types.HybridVSock) {
-	span, _ := fc.trace(ctx, "fcAddVsock")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "fcAddVsock", fc.tracingTags())
 	defer span.End()
 
 	udsPath := hvs.UdsPath
@@ -897,7 +893,7 @@ func (fc *firecracker) fcAddVsock(ctx context.Context, hvs types.HybridVSock) {
 }
 
 func (fc *firecracker) fcAddNetDevice(ctx context.Context, endpoint Endpoint) {
-	span, _ := fc.trace(ctx, "fcAddNetDevice")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "fcAddNetDevice", fc.tracingTags())
 	defer span.End()
 
 	ifaceID := endpoint.Name()
@@ -953,7 +949,7 @@ func (fc *firecracker) fcAddNetDevice(ctx context.Context, endpoint Endpoint) {
 }
 
 func (fc *firecracker) fcAddBlockDrive(ctx context.Context, drive config.BlockDrive) error {
-	span, _ := fc.trace(ctx, "fcAddBlockDrive")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "fcAddBlockDrive", fc.tracingTags())
 	defer span.End()
 
 	driveID := drive.ID
@@ -979,7 +975,7 @@ func (fc *firecracker) fcAddBlockDrive(ctx context.Context, drive config.BlockDr
 
 // Firecracker supports replacing the host drive used once the VM has booted up
 func (fc *firecracker) fcUpdateBlockDrive(ctx context.Context, path, id string) error {
-	span, _ := fc.trace(ctx, "fcUpdateBlockDrive")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "fcUpdateBlockDrive", fc.tracingTags())
 	defer span.End()
 
 	// Use the global block index as an index into the pool of the devices
@@ -1003,7 +999,7 @@ func (fc *firecracker) fcUpdateBlockDrive(ctx context.Context, path, id string)
 // addDevice will add extra devices to firecracker.  Limited to configure before the
 // virtual machine starts.  Devices include drivers and network interfaces only.
 func (fc *firecracker) addDevice(ctx context.Context, devInfo interface{}, devType deviceType) error {
-	span, _ := fc.trace(ctx, "addDevice")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "addDevice", fc.tracingTags())
 	defer span.End()
 
 	fc.state.RLock()
@@ -1064,7 +1060,7 @@ func (fc *firecracker) hotplugBlockDevice(ctx context.Context, drive config.Bloc
 
 // hotplugAddDevice supported in Firecracker VMM
 func (fc *firecracker) hotplugAddDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
-	span, _ := fc.trace(ctx, "hotplugAddDevice")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "hotplugAddDevice", fc.tracingTags())
 	defer span.End()
 
 	switch devType {
@@ -1080,7 +1076,7 @@ func (fc *firecracker) hotplugAddDevice(ctx context.Context, devInfo interface{}
 
 // hotplugRemoveDevice supported in Firecracker VMM
 func (fc *firecracker) hotplugRemoveDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
-	span, _ := fc.trace(ctx, "hotplugRemoveDevice")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "hotplugRemoveDevice", fc.tracingTags())
 	defer span.End()
 
 	switch devType {
@@ -1113,7 +1109,7 @@ func (fc *firecracker) disconnect(ctx context.Context) {
 
 // Adds all capabilities supported by firecracker implementation of hypervisor interface
 func (fc *firecracker) capabilities(ctx context.Context) types.Capabilities {
-	span, _ := fc.trace(ctx, "capabilities")
+	span, _ := katatrace.Trace(ctx, fc.Logger(), "capabilities", fc.tracingTags())
 	defer span.End()
 	var caps types.Capabilities
 	caps.SetBlockDeviceHotplugSupport()

src/runtime/virtcontainers/kata_agent.go

@@ -18,6 +18,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
@@ -30,10 +31,6 @@ import (
 	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
-	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/label"
-	otelLabel "go.opentelemetry.io/otel/label"
-	otelTrace "go.opentelemetry.io/otel/trace"
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/opencontainers/runtime-spec/specs-go"
@@ -44,6 +41,13 @@ import (
 	grpcStatus "google.golang.org/grpc/status"
 )
 
+// kataAgentTracingTags defines tags for the trace span
+var kataAgentTracingTags = map[string]string{
+	"source":    "runtime",
+	"package":   "virtcontainers",
+	"subsystem": "agent",
+}
+
 const (
 	// KataEphemeralDevType creates a tmpfs backed volume for sharing files between containers.
 	KataEphemeralDevType = "ephemeral"
@@ -246,18 +250,6 @@ type kataAgent struct {
 	ctx      context.Context
 }
 
-func (k *kataAgent) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
-	if parent == nil {
-		k.Logger().WithField("type", "bug").Error("trace called before context set")
-		parent = context.Background()
-	}
-
-	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(label.String("source", "runtime"), label.String("package", "virtcontainers"), label.String("subsystem", "agent")))
-
-	return span, ctx
-}
-
 func (k *kataAgent) Logger() *logrus.Entry {
 	return virtLog.WithField("subsystem", "kata_agent")
 }
@@ -341,7 +333,7 @@ func (k *kataAgent) init(ctx context.Context, sandbox *Sandbox, config KataAgent
 	// save
 	k.ctx = sandbox.ctx
 
-	span, _ := k.trace(ctx, "init")
+	span, _ := katatrace.Trace(ctx, k.Logger(), "init", kataAgentTracingTags)
 	defer span.End()
 
 	disableVMShutdown = k.handleTraceSettings(config)
@@ -375,7 +367,7 @@ func (k *kataAgent) capabilities() types.Capabilities {
 }
 
 func (k *kataAgent) internalConfigure(ctx context.Context, h hypervisor, id string, config KataAgentConfig) error {
-	span, _ := k.trace(ctx, "configure")
+	span, _ := katatrace.Trace(ctx, k.Logger(), "configure", kataAgentTracingTags)
 	defer span.End()
 
 	var err error
@@ -384,13 +376,13 @@ func (k *kataAgent) internalConfigure(ctx context.Context, h hypervisor, id stri
 	}
 	k.keepConn = config.LongLiveConn
 
-	span.SetAttributes(otelLabel.Any("socket", k.vmSocket))
+	katatrace.AddTag(span, "socket", k.vmSocket)
 
 	return nil
 }
 
 func (k *kataAgent) setupSandboxBindMounts(ctx context.Context, sandbox *Sandbox) (err error) {
-	span, ctx := k.trace(ctx, "setupSandboxBindMounts")
+	span, ctx := katatrace.Trace(ctx, k.Logger(), "setupSandboxBindMounts", kataAgentTracingTags)
 	defer span.End()
 
 	if len(sandbox.config.SandboxBindMounts) == 0 {
@@ -463,7 +455,7 @@ func (k *kataAgent) cleanupSandboxBindMounts(sandbox *Sandbox) error {
 }
 
 func (k *kataAgent) configure(ctx context.Context, h hypervisor, id, sharePath string, config KataAgentConfig) error {
-	span, ctx := k.trace(ctx, "configure")
+	span, ctx := katatrace.Trace(ctx, k.Logger(), "configure", kataAgentTracingTags)
 	defer span.End()
 
 	err := k.internalConfigure(ctx, h, id, config)
@@ -512,7 +504,7 @@ func (k *kataAgent) configureFromGrpc(ctx context.Context, h hypervisor, id stri
 }
 
 func (k *kataAgent) setupSharedPath(ctx context.Context, sandbox *Sandbox) (err error) {
-	span, ctx := k.trace(ctx, "setupSharedPath")
+	span, ctx := katatrace.Trace(ctx, k.Logger(), "setupSharedPath", kataAgentTracingTags)
 	defer span.End()
 
 	// create shared path structure
@@ -546,7 +538,7 @@ func (k *kataAgent) setupSharedPath(ctx context.Context, sandbox *Sandbox) (err
 }
 
 func (k *kataAgent) createSandbox(ctx context.Context, sandbox *Sandbox) error {
-	span, ctx := k.trace(ctx, "createSandbox")
+	span, ctx := katatrace.Trace(ctx, k.Logger(), "createSandbox", kataAgentTracingTags)
 	defer span.End()
 
 	if err := k.setupSharedPath(ctx, sandbox); err != nil {
@@ -633,7 +625,7 @@ func cmdEnvsToStringSlice(ev []types.EnvVar) []string {
 }
 
 func (k *kataAgent) exec(ctx context.Context, sandbox *Sandbox, c Container, cmd types.Cmd) (*Process, error) {
-	span, ctx := k.trace(ctx, "exec")
+	span, ctx := katatrace.Trace(ctx, k.Logger(), "exec", kataAgentTracingTags)
 	defer span.End()
 
 	var kataProcess *grpc.Process
@@ -805,7 +797,7 @@ func (k *kataAgent) getDNS(sandbox *Sandbox) ([]string, error) {
 }
 
 func (k *kataAgent) startSandbox(ctx context.Context, sandbox *Sandbox) error {
-	span, ctx := k.trace(ctx, "startSandbox")
+	span, ctx := katatrace.Trace(ctx, k.Logger(), "startSandbox", kataAgentTracingTags)
 	defer span.End()
 
 	if err := k.setAgentURL(); err != nil {
@@ -958,7 +950,7 @@ func setupStorages(ctx context.Context, sandbox *Sandbox) []*grpc.Storage {
 }
 
 func (k *kataAgent) stopSandbox(ctx context.Context, sandbox *Sandbox) error {
-	span, ctx := k.trace(ctx, "stopSandbox")
+	span, ctx := katatrace.Trace(ctx, k.Logger(), "stopSandbox", kataAgentTracingTags)
 	defer span.End()
 
 	req := &grpc.DestroySandboxRequest{}
@@ -1315,7 +1307,7 @@ func (k *kataAgent) buildContainerRootfs(ctx context.Context, sandbox *Sandbox,
 }
 
 func (k *kataAgent) createContainer(ctx context.Context, sandbox *Sandbox, c *Container) (p *Process, err error) {
-	span, ctx := k.trace(ctx, "createContainer")
+	span, ctx := katatrace.Trace(ctx, k.Logger(), "createContainer", kataAgentTracingTags)
 	defer span.End()
 
 	var ctrStorages []*grpc.Storage
@@ -1680,7 +1672,7 @@ func (k *kataAgent) handlePidNamespace(grpcSpec *grpc.Spec, sandbox *Sandbox) bo
 }
 
 func (k *kataAgent) startContainer(ctx context.Context, sandbox *Sandbox, c *Container) error {
-	span, ctx := k.trace(ctx, "startContainer")
+	span, ctx := katatrace.Trace(ctx, k.Logger(), "startContainer", kataAgentTracingTags)
 	defer span.End()
 
 	req := &grpc.StartContainerRequest{
@@ -1692,7 +1684,7 @@ func (k *kataAgent) startContainer(ctx context.Context, sandbox *Sandbox, c *Con
 }
 
 func (k *kataAgent) stopContainer(ctx context.Context, sandbox *Sandbox, c Container) error {
-	span, ctx := k.trace(ctx, "stopContainer")
+	span, ctx := katatrace.Trace(ctx, k.Logger(), "stopContainer", kataAgentTracingTags)
 	defer span.End()
 
 	_, err := k.sendReq(ctx, &grpc.RemoveContainerRequest{ContainerId: c.id})
@@ -1836,7 +1828,7 @@ func (k *kataAgent) connect(ctx context.Context) error {
 		return nil
 	}
 
-	span, _ := k.trace(ctx, "connect")
+	span, _ := katatrace.Trace(ctx, k.Logger(), "connect", kataAgentTracingTags)
 	defer span.End()
 
 	// This is for the first connection only, to prevent race
@@ -1860,7 +1852,7 @@ func (k *kataAgent) connect(ctx context.Context) error {
 }
 
 func (k *kataAgent) disconnect(ctx context.Context) error {
-	span, _ := k.trace(ctx, "disconnect")
+	span, _ := katatrace.Trace(ctx, k.Logger(), "disconnect", kataAgentTracingTags)
 	defer span.End()
 
 	k.Lock()
@@ -1890,7 +1882,7 @@ func (k *kataAgent) check(ctx context.Context) error {
 }
 
 func (k *kataAgent) waitProcess(ctx context.Context, c *Container, processID string) (int32, error) {
-	span, ctx := k.trace(ctx, "waitProcess")
+	span, ctx := katatrace.Trace(ctx, k.Logger(), "waitProcess", kataAgentTracingTags)
 	defer span.End()
 
 	resp, err := k.sendReq(ctx, &grpc.WaitProcessRequest{

src/runtime/virtcontainers/mount.go

@@ -16,6 +16,7 @@ import (
 	"syscall"
 
 	merr "github.com/hashicorp/go-multierror"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 	"github.com/sirupsen/logrus"
 	otelLabel "go.opentelemetry.io/otel/label"
@@ -239,7 +240,7 @@ func evalMountPath(source, destination string) (string, string, error) {
 // * ensure the source exists
 // * recursively create the destination
 func moveMount(ctx context.Context, source, destination string) error {
-	span, _ := trace(ctx, "moveMount")
+	span, _ := katatrace.Trace(ctx, nil, "moveMount", apiTracingTags)
 	defer span.End()
 
 	source, destination, err := evalMountPath(source, destination)
@@ -257,7 +258,7 @@ func moveMount(ctx context.Context, source, destination string) error {
 // * recursively create the destination
 // pgtypes stands for propagation types, which are shared, private, slave, and ubind.
 func bindMount(ctx context.Context, source, destination string, readonly bool, pgtypes string) error {
-	span, _ := trace(ctx, "bindMount")
+	span, _ := katatrace.Trace(ctx, nil, "bindMount", apiTracingTags)
 	defer span.End()
 	span.SetAttributes(otelLabel.String("source", source), otelLabel.String("destination", destination))
 
@@ -294,7 +295,7 @@ func bindMount(ctx context.Context, source, destination string, readonly bool, p
 // The mountflags should match the values used in the original mount() call,
 // except for those parameters that you are trying to change.
 func remount(ctx context.Context, mountflags uintptr, src string) error {
-	span, _ := trace(ctx, "remount")
+	span, _ := katatrace.Trace(ctx, nil, "remount", apiTracingTags)
 	defer span.End()
 	span.SetAttributes(otelLabel.String("source", src))
 
@@ -319,7 +320,7 @@ func remountRo(ctx context.Context, src string) error {
 // bindMountContainerRootfs bind mounts a container rootfs into a 9pfs shared
 // directory between the guest and the host.
 func bindMountContainerRootfs(ctx context.Context, shareDir, cid, cRootFs string, readonly bool) error {
-	span, _ := trace(ctx, "bindMountContainerRootfs")
+	span, _ := katatrace.Trace(ctx, nil, "bindMountContainerRootfs", apiTracingTags)
 	defer span.End()
 
 	rootfsDest := filepath.Join(shareDir, cid, rootfsDir)
@@ -359,7 +360,7 @@ func isSymlink(path string) bool {
 }
 
 func bindUnmountContainerRootfs(ctx context.Context, sharedDir, cID string) error {
-	span, _ := trace(ctx, "bindUnmountContainerRootfs")
+	span, _ := katatrace.Trace(ctx, nil, "bindUnmountContainerRootfs", apiTracingTags)
 	defer span.End()
 	span.SetAttributes(otelLabel.String("shared_dir", sharedDir), otelLabel.String("container_id", cID))
 
@@ -382,7 +383,7 @@ func bindUnmountContainerRootfs(ctx context.Context, sharedDir, cID string) erro
 }
 
 func bindUnmountAllRootfs(ctx context.Context, sharedDir string, sandbox *Sandbox) error {
-	span, ctx := trace(ctx, "bindUnmountAllRootfs")
+	span, ctx := katatrace.Trace(ctx, nil, "bindUnmountAllRootfs", apiTracingTags)
 	defer span.End()
 	span.SetAttributes(otelLabel.String("shared_dir", sharedDir), otelLabel.String("sandbox_id", sandbox.id))
 

src/runtime/virtcontainers/network.go

@@ -22,17 +22,23 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
 	"github.com/vishvananda/netns"
-	"go.opentelemetry.io/otel"
-	otelLabel "go.opentelemetry.io/otel/label"
 	otelTrace "go.opentelemetry.io/otel/trace"
 	"golang.org/x/sys/unix"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
+// networkTracingTags defines tags for the trace span
+var networkTracingTags = map[string]string{
+	"source":    "runtime",
+	"package":   "virtcontainers",
+	"subsystem": "network",
+}
+
 // NetInterworkingModel defines the network model connecting
 // the network interface to the virtual machine.
 type NetInterworkingModel int
@@ -1295,13 +1301,12 @@ func (n *Network) trace(ctx context.Context, name string) (otelTrace.Span, conte
 
 func getNetworkTrace(networkType EndpointType) func(ctx context.Context, name string, endpoint interface{}) (otelTrace.Span, context.Context) {
 	return func(ctx context.Context, name string, endpoint interface{}) (otelTrace.Span, context.Context) {
-		tracer := otel.Tracer("kata")
+		span, ctx := katatrace.Trace(ctx, networkLogger(), name, networkTracingTags)
-		ctx, span := tracer.Start(ctx, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "network")))
 		if networkType != "" {
-			span.SetAttributes(otelLabel.Any("type", string(networkType)))
+			katatrace.AddTag(span, "type", string(networkType))
 		}
 		if endpoint != nil {
-			span.SetAttributes(otelLabel.Any("endpoint", endpoint))
+			katatrace.AddTag(span, "endpoint", endpoint)
 		}
 		return span, ctx
 	}
@@ -1309,7 +1314,7 @@ func getNetworkTrace(networkType EndpointType) func(ctx context.Context, name st
 
 func closeSpan(span otelTrace.Span, err error) {
 	if err != nil {
-		span.SetAttributes(otelLabel.Any("error", err))
+		katatrace.AddTag(span, "error", err)
 	}
 	span.End()
 }
@@ -1327,15 +1332,15 @@ func (n *Network) Run(ctx context.Context, networkNSPath string, cb func() error
 // Add adds all needed interfaces inside the network namespace.
 func (n *Network) Add(ctx context.Context, config *NetworkConfig, s *Sandbox, hotplug bool) ([]Endpoint, error) {
 	span, ctx := n.trace(ctx, "Add")
-	span.SetAttributes(otelLabel.String("type", config.InterworkingModel.GetModel()))
+	katatrace.AddTag(span, "type", config.InterworkingModel.GetModel())
 	defer span.End()
 
 	endpoints, err := createEndpointsFromScan(config.NetNSPath, config)
 	if err != nil {
 		return endpoints, err
 	}
-	span.SetAttributes(otelLabel.Any("endpoints", endpoints))
+	katatrace.AddTag(span, "endpoints", endpoints)
-	span.SetAttributes(otelLabel.Bool("hotplug", hotplug))
+	katatrace.AddTag(span, "hotplug", hotplug)
 
 	err = doNetNS(config.NetNSPath, func(_ ns.NetNS) error {
 		for _, endpoint := range endpoints {

src/runtime/virtcontainers/qemu.go

@@ -26,11 +26,9 @@ import (
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-	"go.opentelemetry.io/otel"
-	otelLabel "go.opentelemetry.io/otel/label"
-	otelTrace "go.opentelemetry.io/otel/trace"
 	"golang.org/x/sys/unix"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	pkgUtils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
@@ -40,6 +38,17 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
+// tracingTags defines tags for the trace span
+func (q *qemu) tracingTags() map[string]string {
+	return map[string]string{
+		"source":     "runtime",
+		"package":    "virtcontainers",
+		"subsystem":  "hypervisor",
+		"type":       "qemu",
+		"sandbox_id": q.id,
+	}
+}
+
 // romFile is the file name of the ROM that can be used for virtio-pci devices.
 // If this file name is empty, this means we expect the firmware used by Qemu,
 // such as SeaBIOS or OVMF for instance, to handle this directly.
@@ -186,7 +195,7 @@ func (q *qemu) kernelParameters() string {
 
 // Adds all capabilities supported by qemu implementation of hypervisor interface
 func (q *qemu) capabilities(ctx context.Context) types.Capabilities {
-	span, _ := q.trace(ctx, "capabilities")
+	span, _ := katatrace.Trace(ctx, q.Logger(), "capabilities", q.tracingTags())
 	defer span.End()
 
 	return q.arch.capabilities()
@@ -214,21 +223,9 @@ func (q *qemu) qemuPath() (string, error) {
 	return p, nil
 }
 
-func (q *qemu) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
-	if parent == nil {
-		q.Logger().WithField("type", "bug").Error("trace called before context set")
-		parent = context.Background()
-	}
-
-	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "hypervisor"), otelLabel.String("type", "qemu"), otelLabel.String("sandbox_id", q.id)))
-
-	return span, ctx
-}
-
 // setup sets the Qemu structure up.
 func (q *qemu) setup(ctx context.Context, id string, hypervisorConfig *HypervisorConfig) error {
-	span, _ := q.trace(ctx, "setup")
+	span, _ := katatrace.Trace(ctx, q.Logger(), "setup", q.tracingTags())
 	defer span.End()
 
 	err := hypervisorConfig.valid()
@@ -471,7 +468,7 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa
 	// Save the tracing context
 	q.ctx = ctx
 
-	span, ctx := q.trace(ctx, "createSandbox")
+	span, ctx := katatrace.Trace(ctx, q.Logger(), "createSandbox", q.tracingTags())
 	defer span.End()
 
 	if err := q.setup(ctx, id, hypervisorConfig); err != nil {
@@ -755,7 +752,7 @@ func (q *qemu) setupVirtioMem(ctx context.Context) error {
 
 // startSandbox will start the Sandbox's VM.
 func (q *qemu) startSandbox(ctx context.Context, timeout int) error {
-	span, ctx := q.trace(ctx, "startSandbox")
+	span, ctx := katatrace.Trace(ctx, q.Logger(), "startSandbox", q.tracingTags())
 	defer span.End()
 
 	if q.config.Debug {
@@ -873,7 +870,7 @@ func (q *qemu) bootFromTemplate() error {
 
 // waitSandbox will wait for the Sandbox's VM to be up and running.
 func (q *qemu) waitSandbox(ctx context.Context, timeout int) error {
-	span, _ := q.trace(ctx, "waitSandbox")
+	span, _ := katatrace.Trace(ctx, q.Logger(), "waitSandbox", q.tracingTags())
 	defer span.End()
 
 	if timeout < 0 {
@@ -924,7 +921,7 @@ func (q *qemu) waitSandbox(ctx context.Context, timeout int) error {
 
 // stopSandbox will stop the Sandbox's VM.
 func (q *qemu) stopSandbox(ctx context.Context, waitOnly bool) error {
-	span, _ := q.trace(ctx, "stopSandbox")
+	span, _ := katatrace.Trace(ctx, q.Logger(), "stopSandbox", q.tracingTags())
 	defer span.End()
 
 	q.Logger().Info("Stopping Sandbox")
@@ -1012,7 +1009,7 @@ func (q *qemu) cleanupVM() error {
 }
 
 func (q *qemu) togglePauseSandbox(ctx context.Context, pause bool) error {
-	span, _ := q.trace(ctx, "togglePauseSandbox")
+	span, _ := katatrace.Trace(ctx, q.Logger(), "togglePauseSandbox", q.tracingTags())
 	defer span.End()
 
 	if err := q.qmpSetup(); err != nil {
@@ -1613,9 +1610,9 @@ func (q *qemu) hotplugDevice(ctx context.Context, devInfo interface{}, devType d
 }
 
 func (q *qemu) hotplugAddDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
-	span, ctx := q.trace(ctx, "hotplugAddDevice")
+	span, ctx := katatrace.Trace(ctx, q.Logger(), "hotplugAddDevice", q.tracingTags())
 	defer span.End()
-	span.SetAttributes(otelLabel.Any("device", devInfo))
+	katatrace.AddTag(span, "device", devInfo)
 
 	data, err := q.hotplugDevice(ctx, devInfo, devType, addDevice)
 	if err != nil {
@@ -1626,9 +1623,9 @@ func (q *qemu) hotplugAddDevice(ctx context.Context, devInfo interface{}, devTyp
 }
 
 func (q *qemu) hotplugRemoveDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
-	span, ctx := q.trace(ctx, "hotplugRemoveDevice")
+	span, ctx := katatrace.Trace(ctx, q.Logger(), "hotplugRemoveDevice", q.tracingTags())
 	defer span.End()
-	span.SetAttributes(otelLabel.Any("device", devInfo))
+	katatrace.AddTag(span, "device", devInfo)
 
 	data, err := q.hotplugDevice(ctx, devInfo, devType, removeDevice)
 	if err != nil {
@@ -1839,14 +1836,14 @@ func (q *qemu) hotplugAddMemory(memDev *memoryDevice) (int, error) {
 }
 
 func (q *qemu) pauseSandbox(ctx context.Context) error {
-	span, ctx := q.trace(ctx, "pauseSandbox")
+	span, ctx := katatrace.Trace(ctx, q.Logger(), "pauseSandbox", q.tracingTags())
 	defer span.End()
 
 	return q.togglePauseSandbox(ctx, true)
 }
 
 func (q *qemu) resumeSandbox(ctx context.Context) error {
-	span, ctx := q.trace(ctx, "resumeSandbox")
+	span, ctx := katatrace.Trace(ctx, q.Logger(), "resumeSandbox", q.tracingTags())
 	defer span.End()
 
 	return q.togglePauseSandbox(ctx, false)
@@ -1855,9 +1852,9 @@ func (q *qemu) resumeSandbox(ctx context.Context) error {
 // addDevice will add extra devices to Qemu command line.
 func (q *qemu) addDevice(ctx context.Context, devInfo interface{}, devType deviceType) error {
 	var err error
-	span, _ := q.trace(ctx, "addDevice")
+	span, _ := katatrace.Trace(ctx, q.Logger(), "addDevice", q.tracingTags())
 	defer span.End()
-	span.SetAttributes(otelLabel.Any("device", devInfo))
+	katatrace.AddTag(span, "device", devInfo)
 
 	switch v := devInfo.(type) {
 	case types.Volume:
@@ -1914,7 +1911,7 @@ func (q *qemu) addDevice(ctx context.Context, devInfo interface{}, devType devic
 // getSandboxConsole builds the path of the console where we can read
 // logs coming from the sandbox.
 func (q *qemu) getSandboxConsole(ctx context.Context, id string) (string, string, error) {
-	span, _ := q.trace(ctx, "getSandboxConsole")
+	span, _ := katatrace.Trace(ctx, q.Logger(), "getSandboxConsole", q.tracingTags())
 	defer span.End()
 
 	consoleURL, err := utils.BuildSocketPath(q.store.RunVMStoragePath(), id, consoleSocket)
@@ -1979,7 +1976,7 @@ func (q *qemu) waitMigration() error {
 }
 
 func (q *qemu) disconnect(ctx context.Context) {
-	span, _ := q.trace(ctx, "disconnect")
+	span, _ := katatrace.Trace(ctx, q.Logger(), "disconnect", q.tracingTags())
 	defer span.End()
 
 	q.qmpShutdown()
@@ -2183,7 +2180,7 @@ func genericAppendPCIeRootPort(devices []govmmQemu.Device, number uint32, machin
 }
 
 func (q *qemu) getThreadIDs(ctx context.Context) (vcpuThreadIDs, error) {
-	span, _ := q.trace(ctx, "getThreadIDs")
+	span, _ := katatrace.Trace(ctx, q.Logger(), "getThreadIDs", q.tracingTags())
 	defer span.End()
 
 	tid := vcpuThreadIDs{}
@@ -2248,7 +2245,7 @@ func (q *qemu) resizeVCPUs(ctx context.Context, reqVCPUs uint32) (currentVCPUs u
 }
 
 func (q *qemu) cleanup(ctx context.Context) error {
-	span, _ := q.trace(ctx, "cleanup")
+	span, _ := katatrace.Trace(ctx, q.Logger(), "cleanup", q.tracingTags())
 	defer span.End()
 
 	for _, fd := range q.fds {

src/runtime/virtcontainers/sandbox.go

@@ -26,6 +26,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers"
@@ -43,11 +44,18 @@ import (
 	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
-	"go.opentelemetry.io/otel"
-	otelLabel "go.opentelemetry.io/otel/label"
-	otelTrace "go.opentelemetry.io/otel/trace"
 )
 
+// tracingTags defines tags for the trace span
+func (s *Sandbox) tracingTags() map[string]string {
+	return map[string]string{
+		"source":     "runtime",
+		"package":    "virtcontainers",
+		"subsystem":  "sandbox",
+		"sandbox_id": s.id,
+	}
+}
+
 const (
 	// vmStartTimeout represents the time in seconds a sandbox can wait before
 	// to consider the VM starting operation failed.
@@ -127,18 +135,6 @@ type SandboxConfig struct {
 	Cgroups *configs.Cgroup
 }
 
-func (s *Sandbox) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
-	if parent == nil {
-		s.Logger().WithField("type", "bug").Error("trace called before context set")
-		parent = context.Background()
-	}
-
-	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "sandbox"), otelLabel.String("sandbox_id", s.id)))
-
-	return span, ctx
-}
-
 // valid checks that the sandbox configuration is valid.
 func (sandboxConfig *SandboxConfig) valid() bool {
 	if sandboxConfig.ID == "" {
@@ -393,8 +389,9 @@ func (s *Sandbox) IOStream(containerID, processID string) (io.WriteCloser, io.Re
 }
 
 func createAssets(ctx context.Context, sandboxConfig *SandboxConfig) error {
-	span, _ := trace(ctx, "createAssets")
+	span, _ := katatrace.Trace(ctx, nil, "createAssets", nil)
-	span.SetAttributes(otelLabel.String("sandbox_id", sandboxConfig.ID), otelLabel.String("subsystem", "sandbox"))
+	katatrace.AddTag(span, "sandbox_id", sandboxConfig.ID)
+	katatrace.AddTag(span, "subsystem", "sandbox")
 	defer span.End()
 
 	for _, name := range types.AssetTypes() {
@@ -444,8 +441,9 @@ func (s *Sandbox) getAndStoreGuestDetails(ctx context.Context) error {
 // to physically create that sandbox i.e. starts a VM for that sandbox to eventually
 // be started.
 func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factory) (*Sandbox, error) {
-	span, ctx := trace(ctx, "createSandbox")
+	span, ctx := katatrace.Trace(ctx, nil, "createSandbox", nil)
-	span.SetAttributes(otelLabel.String("sandbox_id", sandboxConfig.ID), otelLabel.String("subsystem", "sandbox"))
+	katatrace.AddTag(span, "sandbox_id", sandboxConfig.ID)
+	katatrace.AddTag(span, "subsystem", "sandbox")
 	defer span.End()
 
 	if err := createAssets(ctx, &sandboxConfig); err != nil {
@@ -483,8 +481,9 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac
 }
 
 func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factory) (sb *Sandbox, retErr error) {
-	span, ctx := trace(ctx, "newSandbox")
+	span, ctx := katatrace.Trace(ctx, nil, "newSandbox", nil)
-	span.SetAttributes(otelLabel.String("sandbox_id", sandboxConfig.ID), otelLabel.String("subsystem", "sandbox"))
+	katatrace.AddTag(span, "sandbox_id", sandboxConfig.ID)
+	katatrace.AddTag(span, "subsystem", "sandbox")
 	defer span.End()
 
 	if !sandboxConfig.valid() {
@@ -618,7 +617,7 @@ func (s *Sandbox) createCgroupManager() error {
 
 // storeSandbox stores a sandbox config.
 func (s *Sandbox) storeSandbox(ctx context.Context) error {
-	span, _ := s.trace(ctx, "storeSandbox")
+	span, _ := katatrace.Trace(ctx, s.Logger(), "storeSandbox", s.tracingTags())
 	defer span.End()
 
 	// flush data to storage
@@ -712,8 +711,7 @@ func (s *Sandbox) Delete(ctx context.Context) error {
 }
 
 func (s *Sandbox) startNetworkMonitor(ctx context.Context) error {
-	var span otelTrace.Span
+	span, ctx := katatrace.Trace(ctx, s.Logger(), "startNetworkMonitor", s.tracingTags())
-	span, ctx = s.trace(ctx, "startNetworkMonitor")
 	defer span.End()
 
 	binPath, err := os.Executable()
@@ -752,7 +750,7 @@ func (s *Sandbox) createNetwork(ctx context.Context) error {
 		return nil
 	}
 
-	span, ctx := s.trace(ctx, "createNetwork")
+	span, ctx := katatrace.Trace(ctx, s.Logger(), "createNetwork", s.tracingTags())
 	defer span.End()
 
 	s.networkNS = NetworkNamespace{
@@ -760,8 +758,8 @@ func (s *Sandbox) createNetwork(ctx context.Context) error {
 		NetNsCreated: s.config.NetworkConfig.NetNsCreated,
 	}
 
-	span.SetAttributes(otelLabel.Any("networkNS", s.networkNS))
+	katatrace.AddTag(span, "networkNS", s.networkNS)
-	span.SetAttributes(otelLabel.Any("NetworkConfig", s.config.NetworkConfig))
+	katatrace.AddTag(span, "NetworkConfig", s.config.NetworkConfig)
 
 	// In case there is a factory, network interfaces are hotplugged
 	// after vm is started.
@@ -789,8 +787,7 @@ func (s *Sandbox) postCreatedNetwork(ctx context.Context) error {
 }
 
 func (s *Sandbox) removeNetwork(ctx context.Context) error {
-	var span otelTrace.Span
+	span, ctx := katatrace.Trace(ctx, s.Logger(), "removeNetwork", s.tracingTags())
-	span, ctx = s.trace(ctx, "removeNetwork")
 	defer span.End()
 
 	if s.config.NetworkConfig.NetmonConfig.Enable {
@@ -997,7 +994,7 @@ func (cw *consoleWatcher) stop() {
 
 // startVM starts the VM.
 func (s *Sandbox) startVM(ctx context.Context) (err error) {
-	span, ctx := s.trace(ctx, "startVM")
+	span, ctx := katatrace.Trace(ctx, s.Logger(), "startVM", s.tracingTags())
 	defer span.End()
 
 	s.Logger().Info("Starting VM")
@@ -1078,7 +1075,7 @@ func (s *Sandbox) startVM(ctx context.Context) (err error) {
 
 // stopVM: stop the sandbox's VM
 func (s *Sandbox) stopVM(ctx context.Context) error {
-	span, ctx := s.trace(ctx, "stopVM")
+	span, ctx := katatrace.Trace(ctx, s.Logger(), "stopVM", s.tracingTags())
 	defer span.End()
 
 	s.Logger().Info("Stopping sandbox in the VM")
@@ -1437,7 +1434,7 @@ func (s *Sandbox) ResumeContainer(ctx context.Context, containerID string) error
 // createContainers registers all containers, create the
 // containers in the guest and starts one shim per container.
 func (s *Sandbox) createContainers(ctx context.Context) error {
-	span, ctx := s.trace(ctx, "createContainers")
+	span, ctx := katatrace.Trace(ctx, s.Logger(), "createContainers", s.tracingTags())
 	defer span.End()
 
 	for i := range s.config.Containers {
@@ -1509,7 +1506,7 @@ func (s *Sandbox) Start(ctx context.Context) error {
 // will be destroyed.
 // When force is true, ignore guest related stop failures.
 func (s *Sandbox) Stop(ctx context.Context, force bool) error {
-	span, ctx := s.trace(ctx, "Stop")
+	span, ctx := katatrace.Trace(ctx, s.Logger(), "Stop", s.tracingTags())
 	defer span.End()
 
 	if s.state.State == types.StateStopped {
@@ -1620,7 +1617,7 @@ func (s *Sandbox) unsetSandboxBlockIndex(index int) error {
 // HotplugAddDevice is used for add a device to sandbox
 // Sandbox implement DeviceReceiver interface from device/api/interface.go
 func (s *Sandbox) HotplugAddDevice(ctx context.Context, device api.Device, devType config.DeviceType) error {
-	span, ctx := s.trace(ctx, "HotplugAddDevice")
+	span, ctx := katatrace.Trace(ctx, s.Logger(), "HotplugAddDevice", s.tracingTags())
 	defer span.End()
 
 	if s.config.SandboxCgroupOnly {

src/runtime/virtcontainers/virtiofsd.go

@@ -16,14 +16,19 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
-	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/label"
-	otelTrace "go.opentelemetry.io/otel/trace"
 )
 
+// virtiofsdTracingTags defines tags for the trace span
+var virtiofsdTracingTags = map[string]string{
+	"source":    "runtime",
+	"package":   "virtcontainers",
+	"subsystem": "virtiofsd",
+}
+
 var (
 	errVirtiofsdDaemonPathEmpty    = errors.New("virtiofsd daemon path is empty")
 	errVirtiofsdSocketPathEmpty    = errors.New("virtiofsd socket path is empty")
@@ -84,7 +89,7 @@ func (v *virtiofsd) getSocketFD() (*os.File, error) {
 
 // Start the virtiofsd daemon
 func (v *virtiofsd) Start(ctx context.Context, onQuit onQuitFunc) (int, error) {
-	span, _ := v.trace(ctx, "Start")
+	span, _ := katatrace.Trace(ctx, v.Logger(), "Start", virtiofsdTracingTags)
 	defer span.End()
 	pid := 0
 
@@ -206,19 +211,8 @@ func (v *virtiofsd) Logger() *log.Entry {
 	return virtLog.WithField("subsystem", "virtiofsd")
 }
 
-func (v *virtiofsd) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
-	if parent == nil {
-		parent = context.Background()
-	}
-
-	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(label.String("source", "runtime"), label.String("package", "virtcontainers"), label.String("subsystem", "virtiofsd")))
-
-	return span, ctx
-}
-
 func (v *virtiofsd) kill(ctx context.Context) (err error) {
-	span, _ := v.trace(ctx, "kill")
+	span, _ := katatrace.Trace(ctx, v.Logger(), "kill", virtiofsdTracingTags)
 	defer span.End()
 
 	if v.PID == 0 {