tracing: Make runtime span attributes more consistent

Span attributes (tags) are not consistent in runtime tracing, so
designate and use core attributes such source, package, subsystem, and
type as span metadata for more understandable output.

Use WithAttributes() during span creation to reduce calls to
SetAttributes().

Modify Trace() in katautils to accept slice of attributes so multiple
functions using different attributes can use it.

Fixes #1852

Signed-off-by: Chelsea Mafrica <chelsea.e.mafrica@intel.com>

src/runtime/containerd-shim-v2/service.go

@@ -294,8 +294,7 @@ func trace(ctx context.Context, name string) (otelTrace.Span, context.Context) {
 		ctx = context.Background()
 	}
 	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(ctx, name)
+	ctx, span := tracer.Start(ctx, name, otelTrace.WithAttributes(label.String("source", "runtime"), label.String("package", "containerdshim")))
-	span.SetAttributes([]label.KeyValue{label.Key("source").String("runtime"), label.Key("package").String("containerdshim")}...)
 
 	return span, ctx
 }

src/runtime/pkg/katautils/create.go

@@ -104,7 +104,7 @@ func SetEphemeralStorageType(ociSpec specs.Spec) specs.Spec {
 // CreateSandbox create a sandbox container
 func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeConfig oci.RuntimeConfig, rootFs vc.RootFs,
 	containerID, bundlePath, console string, disableOutput, systemdCgroup bool) (_ vc.VCSandbox, _ vc.Process, err error) {
-	span, ctx := Trace(ctx, "createSandbox")
+	span, ctx := Trace(ctx, "CreateSandbox", []label.KeyValue{label.Key("source").String("runtime"), label.Key("package").String("katautils"), label.Key("subsystem").String("sandbox")}...)
 	defer span.End()
 
 	sandboxConfig, err := oci.SandboxConfig(ociSpec, runtimeConfig, bundlePath, containerID, console, disableOutput, systemdCgroup)
@@ -159,7 +159,7 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeCo
 
 	sid := sandbox.ID()
 	kataUtilsLogger = kataUtilsLogger.WithField("sandbox", sid)
-	span.SetAttributes(label.Key("sandbox").String(sid))
+	span.SetAttributes(label.Key("sandbox_id").String(sid))
 
 	containers := sandbox.GetAllContainers()
 	if len(containers) != 1 {
@@ -202,7 +202,7 @@ func checkForFIPS(sandboxConfig *vc.SandboxConfig) error {
 func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Spec, rootFs vc.RootFs, containerID, bundlePath, console string, disableOutput bool) (vc.Process, error) {
 	var c vc.VCContainer
 
-	span, ctx := Trace(ctx, "createContainer")
+	span, ctx := Trace(ctx, "CreateContainer", []label.KeyValue{label.Key("source").String("runtime"), label.Key("package").String("katautils"), label.Key("subsystem").String("sandbox")}...)
 	defer span.End()
 
 	ociSpec = SetEphemeralStorageType(ociSpec)
@@ -228,7 +228,7 @@ func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Sp
 		return vc.Process{}, err
 	}
 
-	span.SetAttributes(label.Key("sandbox").String(sandboxID))
+	span.SetAttributes(label.Key("sandbox_id").String(sandboxID))
 
 	c, err = sandbox.CreateContainer(ctx, contConfig)
 	if err != nil {

src/runtime/pkg/katautils/hook.go

@@ -26,11 +26,9 @@ func hookLogger() *logrus.Entry {
 }
 
 func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error {
-	span, _ := Trace(ctx, "hook")
+	span, _ := Trace(ctx, "runHook", []label.KeyValue{label.Key("source").String("runtime"), label.Key("package").String("katautils"), label.Key("subsystem").String("hook")}...)
 	defer span.End()
 
-	span.SetAttributes(label.Key("subsystem").String("runHook"))
-
 	// FIXME
 	// span.LogFields(
 	// 	log.String("hook-name", hook.Path),
@@ -90,11 +88,9 @@ func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error
 }
 
 func runHooks(ctx context.Context, hooks []specs.Hook, cid, bundlePath, hookType string) error {
-	span, _ := Trace(ctx, "hooks")
+	span, _ := Trace(ctx, "runHooks", []label.KeyValue{label.Key("source").String("runtime"), label.Key("package").String("katautils"), label.Key("subsystem").String("hook"), label.Key("type").String(hookType)}...)
 	defer span.End()
 
-	span.SetAttributes(label.Key("subsystem").String(hookType))
-
 	for _, hook := range hooks {
 		if err := runHook(ctx, hook, cid, bundlePath); err != nil {
 			hookLogger().WithFields(logrus.Fields{

src/runtime/pkg/katautils/tracing.go

@@ -110,12 +110,11 @@ func StopTracing(ctx context.Context) {
 }
 
 // Trace creates a new tracing span based on the specified name and parent
-// context.
+// context and an opentelemetry label.KeyValue slice for span attributes.
-func Trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
+func Trace(parent context.Context, name string, tags ...label.KeyValue) (otelTrace.Span, context.Context) {
 
 	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name)
+	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(tags...))
-	span.SetAttributes(label.Key("source").String("runtime"))
 
 	// This is slightly confusing: when tracing is disabled, trace spans
 	// are still created - but the tracer used is a NOP. Therefore, only

src/runtime/virtcontainers/acrn.go

@@ -214,8 +214,7 @@ func (a *Acrn) trace(parent context.Context, name string) (otelTrace.Span, conte
 	}
 
 	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name)
+	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(label.String("source", "runtime"), label.String("package", "virtcontainers"), label.String("subsystem", "hypervisor"), label.String("type", "acrn")))
-	span.SetAttributes([]label.KeyValue{label.Key("subsystem").String("hypervisor"), label.Key("type").String("acrn")}...)
 
 	return span, ctx
 }

src/runtime/virtcontainers/api.go

@@ -30,11 +30,7 @@ var virtLog = logrus.WithField("source", "virtcontainers")
 // context.
 func trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
 	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name)
+	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(label.String("source", "runtime"), label.String("packages", "virtcontainers"), label.String("subsystem", "api")))
-	span.SetAttributes([]label.KeyValue{
-		label.Key("source").String("virtcontainers"),
-		label.Key("component").String("virtcontainers"),
-		label.Key("subsystem").String("api")}...)
 
 	return span, ctx
 }

src/runtime/virtcontainers/clh.go

@@ -764,8 +764,7 @@ func (clh *cloudHypervisor) trace(parent context.Context, name string) (otelTrac
 	}
 
 	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name)
+	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "hypervisor"), otelLabel.String("type", "clh")))
-	span.SetAttributes([]otelLabel.KeyValue{otelLabel.Key("subsystem").String("hypervisor"), otelLabel.Key("type").String("clh")}...)
 
 	return span, ctx
 }

src/runtime/virtcontainers/container.go

@@ -360,8 +360,7 @@ func (c *Container) trace(parent context.Context, name string) (otelTrace.Span,
 	}
 
 	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name)
+	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "container")))
-	span.SetAttributes(otelLabel.Key("subsystem").String("container"))
 
 	return span, ctx
 }

src/runtime/virtcontainers/factory/factory.go

@@ -42,8 +42,7 @@ type factory struct {
 
 func trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
 	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name)
+	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(label.String("source", "runtime"), label.String("package", "factory"), label.String("subsystem", "factory")))
-	span.SetAttributes(label.Key("subsystem").String("factory"))
 
 	return span, ctx
 }

src/runtime/virtcontainers/fc.go

@@ -175,8 +175,7 @@ func (fc *firecracker) trace(parent context.Context, name string) (otelTrace.Spa
 	}
 
 	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name)
+	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "hypervisor"), otelLabel.String("type", "firecracker")))
-	span.SetAttributes([]otelLabel.KeyValue{otelLabel.Key("subsystem").String("hypervisor"), otelLabel.Key("type").String("firecracker")}...)
 
 	return span, ctx
 }

src/runtime/virtcontainers/kata_agent.go

@@ -251,8 +251,7 @@ func (k *kataAgent) trace(parent context.Context, name string) (otelTrace.Span,
 	}
 
 	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name)
+	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(label.String("source", "runtime"), label.String("package", "virtcontainers"), label.String("subsystem", "agent")))
-	span.SetAttributes([]label.KeyValue{label.Key("subsystem").String("agent"), label.Key("type").String("kata")}...)
 
 	return span, ctx
 }

src/runtime/virtcontainers/network.go

@@ -1255,8 +1255,7 @@ type Network struct {
 
 func (n *Network) trace(ctx context.Context, name string) (otelTrace.Span, context.Context) {
 	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(ctx, name)
+	ctx, span := tracer.Start(ctx, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "network"), otelLabel.String("type", "default")))
-	span.SetAttributes([]otelLabel.KeyValue{otelLabel.Key("subsystem").String("network"), otelLabel.Key("type").String("default")}...)
 
 	return span, ctx
 }

src/runtime/virtcontainers/qemu.go

@@ -219,8 +219,7 @@ func (q *qemu) trace(parent context.Context, name string) (otelTrace.Span, conte
 	}
 
 	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name)
+	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "hypervisor"), otelLabel.String("type", "qemu")))
-	span.SetAttributes([]otelLabel.KeyValue{otelLabel.Key("subsystem").String("hypervisor"), otelLabel.Key("type").String("qemu")}...)
 
 	return span, ctx
 }

src/runtime/virtcontainers/sandbox.go

@@ -134,8 +134,7 @@ func (s *Sandbox) trace(parent context.Context, name string) (otelTrace.Span, co
 	}
 
 	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name)
+	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "sandbox")))
-	span.SetAttributes(otelLabel.Key("subsystem").String("sandbox"))
 
 	return span, ctx
 }

src/runtime/virtcontainers/virtiofsd.go

@@ -210,8 +210,7 @@ func (v *virtiofsd) trace(parent context.Context, name string) (otelTrace.Span,
 	}
 
 	tracer := otel.Tracer("kata")
-	ctx, span := tracer.Start(parent, name)
+	ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(label.String("source", "runtime"), label.String("package", "virtcontainers"), label.String("subsystem", "virtiofsd")))
-	span.SetAttributes(label.Key("subsystem").String("virtiofds"))
 
 	return span, ctx
 }