agent: Support Kata agent tracing

Add configuration options to support the various Kata agent tracing
modes and types. See the comments in the built configuration files for
details:

- `cli/config/configuration-fc.toml`
- `cli/config/configuration-qemu.toml`

Fixes #1369.

Signed-off-by: James O. D. Hunt <james.o.hunt@intel.com>

cli/config/configuration-fc.toml.in

@@ -226,6 +226,27 @@ path = "@SHIMPATH@"
 # (default: disabled)
 #enable_debug = true
 
+# Enable agent tracing.
+#
+# If enabled, the default trace mode is "dynamic" and the
+# default trace type is "isolated". The trace mode and type are set
+# explicity with the `trace_type=` and `trace_mode=` options.
+#
+# Notes:
+#
+# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
+#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
+#   will NOT activate agent tracing.
+#
+# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
+#   full details.
+#
+# (default: disabled)
+#enable_tracing = true
+#
+#trace_mode = "dynamic"
+#trace_type = "isolated"
+
 [netmon]
 # If enabled, the network monitoring process gets started when the
 # sandbox is created. This allows for the detection of some additional

cli/config/configuration-qemu.toml.in

@@ -278,6 +278,27 @@ path = "@SHIMPATH@"
 # (default: disabled)
 #enable_debug = true
 
+# Enable agent tracing.
+#
+# If enabled, the default trace mode is "dynamic" and the
+# default trace type is "isolated". The trace mode and type are set
+# explicity with the `trace_type=` and `trace_mode=` options.
+#
+# Notes:
+#
+# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
+#   setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
+#   will NOT activate agent tracing.
+#
+# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
+#   full details.
+#
+# (default: disabled)
+#enable_tracing = true
+#
+#trace_mode = "dynamic"
+#trace_type = "isolated"
+
 [netmon]
 # If enabled, the network monitoring process gets started when the
 # sandbox is created. This allows for the detection of some additional

cli/kata-env.go

@@ -27,7 +27,7 @@ import (
 //
 // XXX: Increment for every change to the output format
 // (meaning any change to the EnvInfo type).
-const formatVersion = "1.0.22"
+const formatVersion = "1.0.23"
 
 // MetaInfo stores information on the format of the output itself
 type MetaInfo struct {
@@ -112,8 +112,11 @@ type ShimInfo struct {
 
 // AgentInfo stores agent details
 type AgentInfo struct {
-	Type  string
-	Debug bool
+	Type      string
+	Debug     bool
+	Trace     bool
+	TraceMode string
+	TraceType string
 }
 
 // DistroInfo stores host operating system distribution details.
@@ -321,6 +324,9 @@ func getAgentInfo(config oci.RuntimeConfig) (AgentInfo, error) {
 			return AgentInfo{}, errors.New("cannot determine Kata agent config")
 		}
 		agent.Debug = agentConfig.Debug
+		agent.Trace = agentConfig.Trace
+		agent.TraceMode = agentConfig.TraceMode
+		agent.TraceType = agentConfig.TraceType
 	default:
 		// Nothing useful to report for the other agent types
 	}

cli/kata-env_test.go

@@ -43,6 +43,7 @@ var (
 	shimDebug       = false
 	netmonDebug     = false
 	agentDebug      = false
+	agentTrace      = false
 )
 
 // makeVersionBinary creates a shell script with the specified file
@@ -155,6 +156,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 		ShimDebug:            shimDebug,
 		NetmonDebug:          netmonDebug,
 		AgentDebug:           agentDebug,
+		AgentTrace:           agentTrace,
 	}
 
 	runtimeConfig := katatestutils.MakeRuntimeConfigFileData(configFileOptions)
@@ -217,6 +219,11 @@ func getExpectedAgentDetails(config oci.RuntimeConfig) (AgentInfo, error) {
 	return AgentInfo{
 		Type:  string(config.AgentType),
 		Debug: agentConfig.Debug,
+		Trace: agentConfig.Trace,
+
+		// No trace mode/type set by default
+		TraceMode: "",
+		TraceType: "",
 	}, nil
 }
 
@@ -496,6 +503,7 @@ func TestEnvGetEnvInfo(t *testing.T) {
 		runtimeTrace = toggle
 		shimDebug = toggle
 		agentDebug = toggle
+		agentTrace = toggle
 
 		configFile, config, err := makeRuntimeConfig(tmpdir)
 		assert.NoError(t, err)
@@ -823,6 +831,16 @@ func TestEnvGetAgentInfo(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, agent.Debug)
 
+	agentConfig.Trace = true
+	agentConfig.TraceMode = "traceMode"
+	agentConfig.TraceType = "traceType"
+	config.AgentConfig = agentConfig
+	agent, err = getAgentInfo(config)
+	assert.NoError(t, err)
+	assert.True(t, agent.Trace)
+	assert.Equal(t, agent.TraceMode, "traceMode")
+	assert.Equal(t, agent.TraceType, "traceType")
+
 	config.AgentConfig = "I am the wrong type"
 	_, err = getAgentInfo(config)
 	assert.Error(t, err)

pkg/katautils/config.go

@@ -136,7 +136,10 @@ type shim struct {
 }
 
 type agent struct {
-	Debug bool `toml:"enable_debug"`
+	Debug     bool   `toml:"enable_debug"`
+	Tracing   bool   `toml:"enable_tracing"`
+	TraceMode string `toml:"trace_mode"`
+	TraceType string `toml:"trace_type"`
 }
 
 type netmon struct {
@@ -393,6 +396,18 @@ func (a agent) debug() bool {
 	return a.Debug
 }
 
+func (a agent) trace() bool {
+	return a.Tracing
+}
+
+func (a agent) traceMode() string {
+	return a.TraceMode
+}
+
+func (a agent) traceType() string {
+	return a.TraceType
+}
+
 func (n netmon) enable() bool {
 	return n.Enable
 }
@@ -656,8 +671,11 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc
 		case kataAgentTableType:
 			config.AgentType = vc.KataContainersAgent
 			config.AgentConfig = vc.KataAgentConfig{
-				UseVSock: config.HypervisorConfig.UseVSock,
-				Debug:    agent.debug(),
+				UseVSock:  config.HypervisorConfig.UseVSock,
+				Debug:     agent.debug(),
+				Trace:     agent.trace(),
+				TraceMode: agent.traceMode(),
+				TraceType: agent.traceType(),
 			}
 		default:
 			return fmt.Errorf("%s agent type is not supported", k)
@@ -720,6 +738,11 @@ func SetKernelParams(runtimeConfig *oci.RuntimeConfig) error {
 
 	// next, check for agent specific kernel params
 	if agentConfig, ok := runtimeConfig.AgentConfig.(vc.KataAgentConfig); ok {
+		err := vc.KataAgentSetDefaultTraceConfigOptions(&agentConfig)
+		if err != nil {
+			return err
+		}
+
 		params := vc.KataAgentKernelParams(agentConfig)
 
 		for _, p := range params {

pkg/katautils/config_test.go

@@ -34,6 +34,7 @@ var (
 	shimDebug       = false
 	netmonDebug     = false
 	agentDebug      = false
+	agentTrace      = false
 )
 
 type testRuntimeConfig struct {
@@ -111,6 +112,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		ShimDebug:            shimDebug,
 		NetmonDebug:          netmonDebug,
 		AgentDebug:           agentDebug,
+		AgentTrace:           agentTrace,
 	}
 
 	runtimeConfigFileData := katatestutils.MakeRuntimeConfigFileData(configFileOptions)
@@ -1208,6 +1210,14 @@ func TestAgentDefaults(t *testing.T) {
 
 	a.Debug = true
 	assert.Equal(a.debug(), a.Debug)
+
+	assert.Equal(a.trace(), a.Tracing)
+
+	a.Tracing = true
+	assert.Equal(a.trace(), a.Tracing)
+
+	assert.Equal(a.traceMode(), a.TraceMode)
+	assert.Equal(a.traceType(), a.TraceType)
 }
 
 func TestGetDefaultConfigFilePaths(t *testing.T) {

virtcontainers/agent.go

@@ -116,7 +116,7 @@ type agent interface {
 	// init().
 	// After init() is called, agent implementations should be initialized and ready
 	// to handle all other Agent interface methods.
-	init(ctx context.Context, sandbox *Sandbox, config interface{}) error
+	init(ctx context.Context, sandbox *Sandbox, config interface{}) (disableVMShutdown bool, err error)
 
 	// capabilities should return a structure that specifies the capabilities
 	// supported by the agent.

virtcontainers/kata_agent.go

@@ -83,12 +83,25 @@ var (
 	maxHostnameLen       = 64
 )
 
+const (
+	agentTraceModeDynamic  = "dynamic"
+	agentTraceModeStatic   = "static"
+	agentTraceTypeIsolated = "isolated"
+	agentTraceTypeCollated = "collated"
+
+	defaultAgentTraceMode = agentTraceModeDynamic
+	defaultAgentTraceType = agentTraceTypeIsolated
+)
+
 // KataAgentConfig is a structure storing information needed
 // to reach the Kata Containers agent.
 type KataAgentConfig struct {
 	LongLiveConn bool
 	UseVSock     bool
 	Debug        bool
+	Trace        bool
+	TraceMode    string
+	TraceType    string
 }
 
 type kataVSOCK struct {
@@ -116,10 +129,11 @@ type kataAgent struct {
 	sync.Mutex
 	client *kataclient.AgentClient
 
-	reqHandlers  map[string]reqFunc
-	state        KataAgentState
-	keepConn     bool
-	proxyBuiltIn bool
+	reqHandlers    map[string]reqFunc
+	state          KataAgentState
+	keepConn       bool
+	proxyBuiltIn   bool
+	dynamicTracing bool
 
 	vmSocket interface{}
 	ctx      context.Context
@@ -176,6 +190,34 @@ func (k *kataAgent) generateVMSocket(id string, c KataAgentConfig) error {
 	return nil
 }
 
+// KataAgentSetDefaultTraceConfigOptions validates agent trace options and
+// sets defaults.
+func KataAgentSetDefaultTraceConfigOptions(config *KataAgentConfig) error {
+	if !config.Trace {
+		return nil
+	}
+
+	switch config.TraceMode {
+	case agentTraceModeDynamic:
+	case agentTraceModeStatic:
+	case "":
+		config.TraceMode = defaultAgentTraceMode
+	default:
+		return fmt.Errorf("invalid kata agent trace mode: %q (need %q or %q)", config.TraceMode, agentTraceModeDynamic, agentTraceModeStatic)
+	}
+
+	switch config.TraceType {
+	case agentTraceTypeIsolated:
+	case agentTraceTypeCollated:
+	case "":
+		config.TraceType = defaultAgentTraceType
+	default:
+		return fmt.Errorf("invalid kata agent trace type: %q (need %q or %q)", config.TraceType, agentTraceTypeIsolated, agentTraceTypeCollated)
+	}
+
+	return nil
+}
+
 // KataAgentKernelParams returns a list of Kata Agent specific kernel
 // parameters.
 func KataAgentKernelParams(config KataAgentConfig) []Param {
@@ -185,10 +227,31 @@ func KataAgentKernelParams(config KataAgentConfig) []Param {
 		params = append(params, Param{Key: "agent.log", Value: "debug"})
 	}
 
+	if config.Trace && config.TraceMode == agentTraceModeStatic {
+		params = append(params, Param{Key: "agent.trace", Value: config.TraceType})
+	}
+
 	return params
 }
 
-func (k *kataAgent) init(ctx context.Context, sandbox *Sandbox, config interface{}) (err error) {
+func (k *kataAgent) handleTraceSettings(config KataAgentConfig) bool {
+	if !config.Trace {
+		return false
+	}
+
+	disableVMShutdown := false
+
+	switch config.TraceMode {
+	case agentTraceModeStatic:
+		disableVMShutdown = true
+	case agentTraceModeDynamic:
+		k.dynamicTracing = true
+	}
+
+	return disableVMShutdown
+}
+
+func (k *kataAgent) init(ctx context.Context, sandbox *Sandbox, config interface{}) (disableVMShutdown bool, err error) {
 	// save
 	k.ctx = sandbox.ctx
 
@@ -198,21 +261,23 @@ func (k *kataAgent) init(ctx context.Context, sandbox *Sandbox, config interface
 	switch c := config.(type) {
 	case KataAgentConfig:
 		if err := k.generateVMSocket(sandbox.id, c); err != nil {
-			return err
+			return false, err
 		}
+
+		disableVMShutdown = k.handleTraceSettings(c)
 		k.keepConn = c.LongLiveConn
 	default:
-		return vcTypes.ErrInvalidConfigType
+		return false, vcTypes.ErrInvalidConfigType
 	}
 
 	k.proxy, err = newProxy(sandbox.config.ProxyType)
 	if err != nil {
-		return err
+		return false, err
 	}
 
 	k.shim, err = newShim(sandbox.config.ShimType)
 	if err != nil {
-		return err
+		return false, err
 	}
 
 	k.proxyBuiltIn = isProxyBuiltIn(sandbox.config.ProxyType)
@@ -222,7 +287,7 @@ func (k *kataAgent) init(ctx context.Context, sandbox *Sandbox, config interface
 		k.Logger().Debug("Could not retrieve anything from storage")
 	}
 
-	return nil
+	return disableVMShutdown, nil
 }
 
 func (k *kataAgent) agentURL() (string, error) {
@@ -715,7 +780,18 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error {
 	}
 
 	_, err = k.sendReq(req)
-	return err
+	if err != nil {
+		return err
+	}
+
+	if k.dynamicTracing {
+		_, err = k.sendReq(&grpc.StartTracingRequest{})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
 func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
@@ -732,6 +808,13 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
 		return err
 	}
 
+	if k.dynamicTracing {
+		_, err := k.sendReq(&grpc.StopTracingRequest{})
+		if err != nil {
+			return err
+		}
+	}
+
 	if err := k.proxy.stop(k.state.ProxyPid); err != nil {
 		return err
 	}
@@ -1645,6 +1728,12 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) {
 	k.reqHandlers["grpc.SetGuestDateTimeRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
 		return k.client.SetGuestDateTime(ctx, req.(*grpc.SetGuestDateTimeRequest), opts...)
 	}
+	k.reqHandlers["grpc.StartTracingRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
+		return k.client.StartTracing(ctx, req.(*grpc.StartTracingRequest), opts...)
+	}
+	k.reqHandlers["grpc.StopTracingRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
+		return k.client.StopTracing(ctx, req.(*grpc.StopTracingRequest), opts...)
+	}
 }
 
 func (k *kataAgent) sendReq(request interface{}) (interface{}, error) {

virtcontainers/kata_agent_test.go

@@ -248,6 +248,14 @@ func (p *gRPCProxy) CopyFile(ctx context.Context, req *pb.CopyFileRequest) (*gpb
 	return &gpb.Empty{}, nil
 }
 
+func (p *gRPCProxy) StartTracing(ctx context.Context, req *pb.StartTracingRequest) (*gpb.Empty, error) {
+	return &gpb.Empty{}, nil
+}
+
+func (p *gRPCProxy) StopTracing(ctx context.Context, req *pb.StopTracingRequest) (*gpb.Empty, error) {
+	return &gpb.Empty{}, nil
+}
+
 func (p *gRPCProxy) MemHotplugByProbe(ctx context.Context, req *pb.MemHotplugByProbeRequest) (*gpb.Empty, error) {
 	return &gpb.Empty{}, nil
 }
@@ -930,22 +938,195 @@ func TestKataCleanupSandbox(t *testing.T) {
 func TestKataAgentKernelParams(t *testing.T) {
 	assert := assert.New(t)
 
-	config := KataAgentConfig{}
-
-	params := KataAgentKernelParams(config)
-	assert.Empty(params)
-
-	config.Debug = true
-
-	params = KataAgentKernelParams(config)
-	assert.NotEmpty(params)
-
-	assert.Len(params, 1)
-
-	expected := Param{
-		Key:   "agent.log",
-		Value: "debug",
+	type testData struct {
+		debug          bool
+		trace          bool
+		traceMode      string
+		traceType      string
+		expectedParams []Param
 	}
 
-	assert.Equal(params[0], expected)
+	debugParam := Param{Key: "agent.log", Value: "debug"}
+
+	traceIsolatedParam := Param{Key: "agent.trace", Value: "isolated"}
+	traceCollatedParam := Param{Key: "agent.trace", Value: "collated"}
+
+	traceFooParam := Param{Key: "agent.trace", Value: "foo"}
+
+	data := []testData{
+		{false, false, "", "", []Param{}},
+		{true, false, "", "", []Param{debugParam}},
+
+		{false, false, "foo", "", []Param{}},
+		{false, false, "foo", "", []Param{}},
+		{false, false, "", "foo", []Param{}},
+		{false, false, "", "foo", []Param{}},
+		{false, false, "foo", "foo", []Param{}},
+		{false, true, "foo", "foo", []Param{}},
+
+		{false, false, agentTraceModeDynamic, "", []Param{}},
+		{false, false, agentTraceModeStatic, "", []Param{}},
+		{false, false, "", agentTraceTypeIsolated, []Param{}},
+		{false, false, "", agentTraceTypeCollated, []Param{}},
+		{false, false, "foo", agentTraceTypeIsolated, []Param{}},
+		{false, false, "foo", agentTraceTypeCollated, []Param{}},
+
+		{false, false, agentTraceModeDynamic, agentTraceTypeIsolated, []Param{}},
+		{false, false, agentTraceModeDynamic, agentTraceTypeCollated, []Param{}},
+
+		{false, false, agentTraceModeStatic, agentTraceTypeCollated, []Param{}},
+		{false, false, agentTraceModeStatic, agentTraceTypeCollated, []Param{}},
+
+		{false, true, agentTraceModeDynamic, agentTraceTypeIsolated, []Param{}},
+		{false, true, agentTraceModeDynamic, agentTraceTypeCollated, []Param{}},
+		{true, true, agentTraceModeDynamic, agentTraceTypeCollated, []Param{debugParam}},
+
+		{false, true, "", agentTraceTypeIsolated, []Param{}},
+		{false, true, "", agentTraceTypeCollated, []Param{}},
+		{true, true, "", agentTraceTypeIsolated, []Param{debugParam}},
+		{true, true, "", agentTraceTypeCollated, []Param{debugParam}},
+		{false, true, "foo", agentTraceTypeIsolated, []Param{}},
+		{false, true, "foo", agentTraceTypeCollated, []Param{}},
+		{true, true, "foo", agentTraceTypeIsolated, []Param{debugParam}},
+		{true, true, "foo", agentTraceTypeCollated, []Param{debugParam}},
+
+		{false, true, agentTraceModeStatic, agentTraceTypeIsolated, []Param{traceIsolatedParam}},
+		{false, true, agentTraceModeStatic, agentTraceTypeCollated, []Param{traceCollatedParam}},
+		{true, true, agentTraceModeStatic, agentTraceTypeIsolated, []Param{traceIsolatedParam, debugParam}},
+		{true, true, agentTraceModeStatic, agentTraceTypeCollated, []Param{traceCollatedParam, debugParam}},
+
+		{false, true, agentTraceModeStatic, "foo", []Param{traceFooParam}},
+		{true, true, agentTraceModeStatic, "foo", []Param{debugParam, traceFooParam}},
+	}
+
+	for i, d := range data {
+		config := KataAgentConfig{
+			Debug:     d.debug,
+			Trace:     d.trace,
+			TraceMode: d.traceMode,
+			TraceType: d.traceType,
+		}
+
+		count := len(d.expectedParams)
+
+		params := KataAgentKernelParams(config)
+
+		if count == 0 {
+			assert.Emptyf(params, "test %d (%+v)", i, d)
+			continue
+		}
+
+		assert.Len(params, count)
+
+		for _, p := range d.expectedParams {
+			assert.Containsf(params, p, "test %d (%+v)", i, d)
+		}
+	}
+}
+
+func TestKataAgentHandleTraceSettings(t *testing.T) {
+	assert := assert.New(t)
+
+	type testData struct {
+		traceMode               string
+		trace                   bool
+		expectDisableVMShutdown bool
+		expectDynamicTracing    bool
+	}
+
+	data := []testData{
+		{"", false, false, false},
+		{"", true, false, false},
+		{agentTraceModeStatic, true, true, false},
+		{agentTraceModeDynamic, true, false, true},
+	}
+
+	for i, d := range data {
+		k := &kataAgent{}
+
+		config := KataAgentConfig{
+			Trace:     d.trace,
+			TraceMode: d.traceMode,
+		}
+
+		disableVMShutdown := k.handleTraceSettings(config)
+
+		if d.expectDisableVMShutdown {
+			assert.Truef(disableVMShutdown, "test %d (%+v)", i, d)
+		} else {
+			assert.Falsef(disableVMShutdown, "test %d (%+v)", i, d)
+		}
+
+		if d.expectDynamicTracing {
+			assert.Truef(k.dynamicTracing, "test %d (%+v)", i, d)
+		} else {
+			assert.Falsef(k.dynamicTracing, "test %d (%+v)", i, d)
+		}
+	}
+}
+
+func TestKataAgentSetDefaultTraceConfigOptions(t *testing.T) {
+	assert := assert.New(t)
+
+	type testData struct {
+		traceMode              string
+		traceType              string
+		trace                  bool
+		expectDefaultTraceMode bool
+		expectDefaultTraceType bool
+		expectError            bool
+	}
+
+	data := []testData{
+		{"", "", false, false, false, false},
+		{agentTraceModeDynamic, agentTraceTypeCollated, false, false, false, false},
+		{agentTraceModeDynamic, agentTraceTypeIsolated, false, false, false, false},
+		{agentTraceModeStatic, agentTraceTypeCollated, false, false, false, false},
+		{agentTraceModeStatic, agentTraceTypeIsolated, false, false, false, false},
+
+		{agentTraceModeDynamic, agentTraceTypeCollated, true, false, false, false},
+		{agentTraceModeDynamic, agentTraceTypeIsolated, true, false, false, false},
+
+		{agentTraceModeStatic, agentTraceTypeCollated, true, false, false, false},
+		{agentTraceModeStatic, agentTraceTypeIsolated, true, false, false, false},
+
+		{agentTraceModeDynamic, "", true, false, true, false},
+		{agentTraceModeDynamic, "invalid", true, false, false, true},
+
+		{agentTraceModeStatic, "", true, false, true, false},
+		{agentTraceModeStatic, "invalid", true, false, false, true},
+
+		{"", agentTraceTypeIsolated, true, true, false, false},
+		{"invalid", agentTraceTypeIsolated, true, false, false, true},
+
+		{"", agentTraceTypeCollated, true, true, false, false},
+		{"invalid", agentTraceTypeCollated, true, false, false, true},
+
+		{"", "", true, true, true, false},
+		{"invalid", "invalid", true, false, false, true},
+	}
+
+	for i, d := range data {
+		config := &KataAgentConfig{
+			Trace:     d.trace,
+			TraceMode: d.traceMode,
+			TraceType: d.traceType,
+		}
+
+		err := KataAgentSetDefaultTraceConfigOptions(config)
+		if d.expectError {
+			assert.Error(err, "test %d (%+v)", i, d)
+			continue
+		} else {
+			assert.NoError(err, "test %d (%+v)", i, d)
+		}
+
+		if d.expectDefaultTraceMode {
+			assert.Equalf(config.TraceMode, defaultAgentTraceMode, "test %d (%+v)", i, d)
+		}
+
+		if d.expectDefaultTraceType {
+			assert.Equalf(config.TraceType, defaultAgentTraceType, "test %d (%+v)", i, d)
+		}
+	}
 }

virtcontainers/noop_agent.go

@@ -27,8 +27,8 @@ func (n *noopAgent) startProxy(sandbox *Sandbox) error {
 }
 
 // init initializes the Noop agent, i.e. it does nothing.
-func (n *noopAgent) init(ctx context.Context, sandbox *Sandbox, config interface{}) error {
-	return nil
+func (n *noopAgent) init(ctx context.Context, sandbox *Sandbox, config interface{}) (bool, error) {
+	return false, nil
 }
 
 // createSandbox is the Noop agent sandbox creation implementation. It does nothing.

virtcontainers/noop_agent_test.go

@@ -40,10 +40,14 @@ func TestNoopAgentInit(t *testing.T) {
 	n := &noopAgent{}
 	sandbox := &Sandbox{}
 
-	err := n.init(context.Background(), sandbox, nil)
+	disableVMShutdown, err := n.init(context.Background(), sandbox, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
+
+	if disableVMShutdown != false {
+		t.Fatal(err)
+	}
 }
 
 func TestNoopAgentExec(t *testing.T) {

virtcontainers/sandbox.go

@@ -186,10 +186,11 @@ type Sandbox struct {
 
 	wg *sync.WaitGroup
 
-	shmSize          uint64
-	sharePidNs       bool
-	stateful         bool
-	seccompSupported bool
+	shmSize           uint64
+	sharePidNs        bool
+	stateful          bool
+	seccompSupported  bool
+	disableVMShutdown bool
 
 	ctx context.Context
 }
@@ -588,7 +589,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 		return nil, err
 	}
 
-	if err = s.agent.init(ctx, s, agentConfig); err != nil {
+	if s.disableVMShutdown, err = s.agent.init(ctx, s, agentConfig); err != nil {
 		return nil, err
 	}
 
@@ -1034,6 +1035,13 @@ func (s *Sandbox) stopVM() error {
 		s.Logger().WithError(err).WithField("sandboxid", s.id).Warning("Agent did not stop sandbox")
 	}
 
+	if s.disableVMShutdown {
+		// Do not kill the VM - allow the agent to shut it down
+		// (only used to support static agent tracing).
+		s.Logger().Info("Not stopping VM")
+		return nil
+	}
+
 	s.Logger().Info("Stopping VM")
 	return s.hypervisor.stopSandbox()
 }

virtcontainers/vm_test.go

@@ -118,7 +118,7 @@ func TestVMConfigGrpc(t *testing.T) {
 		HypervisorType:   QemuHypervisor,
 		HypervisorConfig: newQemuConfig(),
 		AgentType:        KataContainersAgent,
-		AgentConfig:      KataAgentConfig{false, true, false},
+		AgentConfig:      KataAgentConfig{false, true, false, false, "", ""},
 		ProxyType:        NoopProxyType,
 	}