diff --git a/src/runtime/cli/config/configuration-qemu.toml.in b/src/runtime/cli/config/configuration-qemu.toml.in index a5e415604..bf51d3acc 100644 --- a/src/runtime/cli/config/configuration-qemu.toml.in +++ b/src/runtime/cli/config/configuration-qemu.toml.in @@ -319,6 +319,26 @@ valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@ # Default 0-sized value means unlimited rate. #tx_rate_limiter_max_rate = 0 +# Set where to save the guest memory dump file. +# If set, when GUEST_PANICKED event occurred, +# guest memeory will be dumped to host filesystem under guest_memory_dump_path, +# This directory will be created automatically if it does not exist. +# +# The dumped file(also called vmcore) can be processed with crash or gdb. +# +# WARNING: +# Dump guest’s memory can take very long depending on the amount of guest memory +# and use much disk space. +#guest_memory_dump_path="/var/crash/kata" + +# If enable paging. +# Basically, if you want to use "gdb" rather than "crash", +# or need the guest-virtual addresses in the ELF vmcore, +# then you should enable paging. +# +# See: https://www.qemu.org/docs/master/qemu-qmp-ref.html#Dump-guest-memory for details +#guest_memory_dump_paging=false + [factory] # VM templating support. Once enabled, new VMs are created from template # using vm cloning. They will share the same initial kernel, initramfs and diff --git a/src/runtime/cli/kata-env.go b/src/runtime/cli/kata-env.go index 0c555ed31..3203f19b7 100644 --- a/src/runtime/cli/kata-env.go +++ b/src/runtime/cli/kata-env.go @@ -14,6 +14,7 @@ import ( "github.com/BurntSushi/toml" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci" @@ -292,7 +293,7 @@ func getNetmonInfo(config oci.RuntimeConfig) NetmonInfo { } func getCommandVersion(cmd string) (string, error) { - return katautils.RunCommand([]string{cmd, "--version"}) + return utils.RunCommand([]string{cmd, "--version"}) } func getAgentInfo(config oci.RuntimeConfig) (AgentInfo, error) { diff --git a/src/runtime/cli/main_test.go b/src/runtime/cli/main_test.go index cd51acda5..94140162d 100644 --- a/src/runtime/cli/main_test.go +++ b/src/runtime/cli/main_test.go @@ -24,6 +24,7 @@ import ( "github.com/dlespiau/covertool/pkg/cover" ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci" @@ -273,7 +274,7 @@ func createOCIConfig(bundleDir string) error { return errors.New("Cannot find command to generate OCI config file") } - _, err := katautils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir}) + _, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir}) if err != nil { return err } @@ -378,7 +379,7 @@ func makeOCIBundle(bundleDir string) error { } } - output, err := katautils.RunCommandFull([]string{"cp", "-a", from, to}, true) + output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true) if err != nil { return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output) } diff --git a/src/runtime/cli/utils_test.go b/src/runtime/cli/utils_test.go index c5eccddbc..a222b940b 100644 --- a/src/runtime/cli/utils_test.go +++ b/src/runtime/cli/utils_test.go @@ -13,6 +13,7 @@ import ( "testing" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" "github.com/stretchr/testify/assert" ) @@ -176,13 +177,13 @@ VERSION_ID="%s" } func TestUtilsRunCommand(t *testing.T) { - output, err := katautils.RunCommand([]string{"true"}) + output, err := utils.RunCommand([]string{"true"}) assert.NoError(t, err) assert.Equal(t, "", output) } func TestUtilsRunCommandCaptureStdout(t *testing.T) { - output, err := katautils.RunCommand([]string{"echo", "hello"}) + output, err := utils.RunCommand([]string{"echo", "hello"}) assert.NoError(t, err) assert.Equal(t, "hello", output) } @@ -190,7 +191,7 @@ func TestUtilsRunCommandCaptureStdout(t *testing.T) { func TestUtilsRunCommandIgnoreStderr(t *testing.T) { args := []string{"/bin/sh", "-c", "echo foo >&2;exit 0"} - output, err := katautils.RunCommand(args) + output, err := utils.RunCommand(args) assert.NoError(t, err) assert.Equal(t, "", output) } @@ -213,7 +214,7 @@ func TestUtilsRunCommandInvalidCmds(t *testing.T) { } for _, args := range invalidCommands { - output, err := katautils.RunCommand(args) + output, err := utils.RunCommand(args) assert.Error(t, err) assert.Equal(t, "", output) } diff --git a/src/runtime/containerd-shim-v2/utils_test.go b/src/runtime/containerd-shim-v2/utils_test.go index 7ba694af0..492a0b41c 100644 --- a/src/runtime/containerd-shim-v2/utils_test.go +++ b/src/runtime/containerd-shim-v2/utils_test.go @@ -21,6 +21,7 @@ import ( ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci" @@ -150,7 +151,7 @@ func createOCIConfig(bundleDir string) error { return errors.New("Cannot find command to generate OCI config file") } - _, err := katautils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir}) + _, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir}) if err != nil { return err } @@ -278,7 +279,7 @@ func makeOCIBundle(bundleDir string) error { } } - output, err := katautils.RunCommandFull([]string{"cp", "-a", from, to}, true) + output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true) if err != nil { return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output) } diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index a22f30d6d..e542d1475 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -125,6 +125,8 @@ type hypervisor struct { RxRateLimiterMaxRate uint64 `toml:"rx_rate_limiter_max_rate"` TxRateLimiterMaxRate uint64 `toml:"tx_rate_limiter_max_rate"` EnableAnnotations []string `toml:"enable_annotations"` + GuestMemoryDumpPath string `toml:"guest_memory_dump_path"` + GuestMemoryDumpPaging bool `toml:"guest_memory_dump_paging"` } type runtime struct { @@ -688,6 +690,8 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { RxRateLimiterMaxRate: rxRateLimiterMaxRate, TxRateLimiterMaxRate: txRateLimiterMaxRate, EnableAnnotations: h.EnableAnnotations, + GuestMemoryDumpPath: h.GuestMemoryDumpPath, + GuestMemoryDumpPaging: h.GuestMemoryDumpPaging, }, nil } diff --git a/src/runtime/pkg/katautils/container_engine.go b/src/runtime/pkg/katautils/container_engine.go index 3b0caf5ab..39f2f563f 100644 --- a/src/runtime/pkg/katautils/container_engine.go +++ b/src/runtime/pkg/katautils/container_engine.go @@ -7,6 +7,8 @@ package katautils import ( "os/exec" + + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" ) type CtrEngine struct { @@ -19,7 +21,7 @@ var ( func (e *CtrEngine) Init(name string) (string, error) { var out string - out, err := RunCommandFull([]string{name, "version"}, true) + out, err := utils.RunCommandFull([]string{name, "version"}, true) if err != nil { return out, err } @@ -30,19 +32,19 @@ func (e *CtrEngine) Init(name string) (string, error) { func (e *CtrEngine) Inspect(image string) (string, error) { // Only hit the network if the image doesn't exist locally - return RunCommand([]string{e.Name, "inspect", "--type=image", image}) + return utils.RunCommand([]string{e.Name, "inspect", "--type=image", image}) } func (e *CtrEngine) Pull(image string) (string, error) { - return RunCommand([]string{e.Name, "pull", image}) + return utils.RunCommand([]string{e.Name, "pull", image}) } func (e *CtrEngine) Create(image string) (string, error) { - return RunCommand([]string{e.Name, "create", image}) + return utils.RunCommand([]string{e.Name, "create", image}) } func (e *CtrEngine) Rm(ctrID string) (string, error) { - return RunCommand([]string{e.Name, "rm", ctrID}) + return utils.RunCommand([]string{e.Name, "rm", ctrID}) } func (e *CtrEngine) GetRootfs(ctrID string, dir string) error { diff --git a/src/runtime/pkg/katautils/create_test.go b/src/runtime/pkg/katautils/create_test.go index fb14d99b9..7f033082f 100644 --- a/src/runtime/pkg/katautils/create_test.go +++ b/src/runtime/pkg/katautils/create_test.go @@ -20,6 +20,7 @@ import ( "testing" ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci" @@ -87,7 +88,7 @@ func makeOCIBundle(bundleDir string) error { } } - output, err := RunCommandFull([]string{"cp", "-a", from, to}, true) + output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true) if err != nil { return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output) } diff --git a/src/runtime/pkg/katautils/utils.go b/src/runtime/pkg/katautils/utils.go index d89257699..02c2ea568 100644 --- a/src/runtime/pkg/katautils/utils.go +++ b/src/runtime/pkg/katautils/utils.go @@ -8,13 +8,12 @@ package katautils import ( "fmt" - "golang.org/x/sys/unix" "io/ioutil" "os" - "os/exec" "path/filepath" - "strings" "syscall" + + "golang.org/x/sys/unix" ) // FileExists test is a file exiting or not @@ -110,27 +109,3 @@ func GetFileContents(file string) (string, error) { return string(bytes), nil } - -// RunCommandFull returns the commands space-trimmed standard output and -// error on success. Note that if the command fails, the requested output will -// still be returned, along with an error. -func RunCommandFull(args []string, includeStderr bool) (string, error) { - cmd := exec.Command(args[0], args[1:]...) - var err error - var bytes []byte - - if includeStderr { - bytes, err = cmd.CombinedOutput() - } else { - bytes, err = cmd.Output() - } - - trimmed := strings.TrimSpace(string(bytes)) - - return trimmed, err -} - -// RunCommand returns the commands space-trimmed standard output on success -func RunCommand(args []string) (string, error) { - return RunCommandFull(args, false) -} diff --git a/src/runtime/pkg/katautils/utils_test.go b/src/runtime/pkg/katautils/utils_test.go index 231480622..984efc348 100644 --- a/src/runtime/pkg/katautils/utils_test.go +++ b/src/runtime/pkg/katautils/utils_test.go @@ -18,6 +18,7 @@ import ( "syscall" "testing" + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci" "github.com/stretchr/testify/assert" ) @@ -90,7 +91,7 @@ func createOCIConfig(bundleDir string) error { return errors.New("Cannot find command to generate OCI config file") } - _, err := RunCommand([]string{configCmd, "spec", "--bundle", bundleDir}) + _, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir}) if err != nil { return err } diff --git a/src/runtime/pkg/utils/utils.go b/src/runtime/pkg/utils/utils.go index 7993020c3..fd09bbd70 100644 --- a/src/runtime/pkg/utils/utils.go +++ b/src/runtime/pkg/utils/utils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 Ant Financial +// Copyright (c) 2020 Ant Group // // SPDX-License-Identifier: Apache-2.0 // @@ -6,7 +6,11 @@ package utils import ( + "fmt" "net/http" + "os" + "os/exec" + "path/filepath" "strings" ) @@ -31,3 +35,48 @@ func GzipAccepted(header http.Header) bool { func String2Pointer(s string) *string { return &s } + +// RunCommandFull returns the commands space-trimmed standard output and +// error on success. Note that if the command fails, the requested output will +// still be returned, along with an error. +func RunCommandFull(args []string, includeStderr bool) (string, error) { + cmd := exec.Command(args[0], args[1:]...) + var err error + var bytes []byte + + if includeStderr { + bytes, err = cmd.CombinedOutput() + } else { + bytes, err = cmd.Output() + } + + trimmed := strings.TrimSpace(string(bytes)) + + return trimmed, err +} + +// RunCommand returns the commands space-trimmed standard output on success +func RunCommand(args []string) (string, error) { + return RunCommandFull(args, false) +} + +// EnsureDir check if a directory exist, if not then create it +func EnsureDir(path string, mode os.FileMode) error { + if !filepath.IsAbs(path) { + return fmt.Errorf("Not an absolute path: %s", path) + } + + if fi, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + if err = os.MkdirAll(path, mode); err != nil { + return err + } + } else { + return err + } + } else if !fi.IsDir() { + return fmt.Errorf("Not a directory: %s", path) + } + + return nil +} diff --git a/src/runtime/pkg/utils/utils_test.go b/src/runtime/pkg/utils/utils_test.go index 3139b2ce1..30189521d 100644 --- a/src/runtime/pkg/utils/utils_test.go +++ b/src/runtime/pkg/utils/utils_test.go @@ -6,7 +6,10 @@ package utils import ( + "fmt" + "io/ioutil" "net/http" + "os" "testing" "github.com/stretchr/testify/assert" @@ -45,3 +48,71 @@ func TestGzipAccepted(t *testing.T) { assert.Equal(tc.result, b) } } + +func TestEnsureDir(t *testing.T) { + const testMode = 0755 + tmpdir, err := ioutil.TempDir("", "TestEnsureDir") + assert := assert.New(t) + + assert.NoError(err) + defer os.RemoveAll(tmpdir) + + testCases := []struct { + before func() + path string + err bool + msg string + }{ + { + before: nil, + path: "a/b/c", + err: true, + msg: "Not an absolute path", + }, + { + before: nil, + path: fmt.Sprintf("%s/abc/def", tmpdir), + err: false, + msg: "", + }, + { + before: nil, + path: fmt.Sprintf("%s/abc", tmpdir), + err: false, + msg: "", + }, + { + before: func() { + err := os.MkdirAll(fmt.Sprintf("%s/abc/def", tmpdir), testMode) + assert.NoError(err) + }, + path: fmt.Sprintf("%s/abc/def", tmpdir), + err: false, + msg: "", + }, + { + before: func() { + // create a regular file + err := os.MkdirAll(fmt.Sprintf("%s/abc", tmpdir), testMode) + assert.NoError(err) + _, err = os.Create(fmt.Sprintf("%s/abc/file.txt", tmpdir)) + assert.NoError(err) + }, + path: fmt.Sprintf("%s/abc/file.txt", tmpdir), + err: true, + msg: "Not a directory", + }, + } + + for _, tc := range testCases { + if tc.before != nil { + tc.before() + } + err := EnsureDir(tc.path, testMode) + if tc.err { + assert.Contains(err.Error(), tc.msg, "error msg should contains: %s, but got %s", tc.msg, err.Error()) + } else { + assert.Equal(err, nil, "failed for path: %s, except no error, but got %+v", tc.path, err) + } + } +} diff --git a/src/runtime/vendor/github.com/sirupsen/logrus/go.mod b/src/runtime/vendor/github.com/sirupsen/logrus/go.mod index 12fdf9898..e5522f30e 100644 --- a/src/runtime/vendor/github.com/sirupsen/logrus/go.mod +++ b/src/runtime/vendor/github.com/sirupsen/logrus/go.mod @@ -1,5 +1,7 @@ module github.com/sirupsen/logrus +go 1.15 + require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.1 diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index c67971001..09a21607a 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -440,6 +440,13 @@ type HypervisorConfig struct { // Enable annotations by name EnableAnnotations []string + + // GuestCoredumpPath is the path in host for saving guest memory dump + GuestMemoryDumpPath string + + // GuestMemoryDumpPaging is used to indicate if enable paging + // for QEMU dump-guest-memory command + GuestMemoryDumpPaging bool } // vcpu mapping from vcpu number to thread number @@ -607,6 +614,10 @@ func (conf *HypervisorConfig) HypervisorAssetPath() (string, error) { return conf.assetPath(types.HypervisorAsset) } +func (conf *HypervisorConfig) IfPVPanicEnabled() bool { + return conf.GuestMemoryDumpPath != "" +} + // HypervisorCtlAssetPath returns the VM hypervisor ctl path func (conf *HypervisorConfig) HypervisorCtlAssetPath() (string, error) { return conf.assetPath(types.HypervisorCtlAsset) diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 32e0739b7..1a5944b33 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -31,6 +31,7 @@ import ( "github.com/sirupsen/logrus" "golang.org/x/sys/unix" + pkgUtils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid" @@ -99,6 +100,9 @@ type qemu struct { stopped bool store persistapi.PersistDriver + + // if in memory dump progress + memoryDumpFlag sync.Mutex } const ( @@ -106,6 +110,9 @@ const ( qmpSocket = "qmp.sock" vhostFSSocket = "vhost-fs.sock" + // memory dump format will be set to elf + memoryDumpFormat = "elf" + qmpCapErrMsg = "Failed to negoatiate QMP capabilities" qmpExecCatCmd = "exec:cat" @@ -408,13 +415,17 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I } } + if q.config.IfPVPanicEnabled() { + // there should have no errors for pvpanic device + devices, _ = q.arch.appendPVPanicDevice(devices) + } + var ioThread *govmmQemu.IOThread if q.config.BlockDeviceDriver == config.VirtioSCSI { return q.arch.appendSCSIController(devices, q.config.EnableIOThreads) } return devices, ioThread, nil - } func (q *qemu) setupTemplate(knobs *govmmQemu.Knobs, memory *govmmQemu.Memory) govmmQemu.Incoming { @@ -1027,7 +1038,13 @@ func (q *qemu) qmpSetup() error { return nil } - cfg := govmmQemu.QMPConfig{Logger: newQMPLogger()} + events := make(chan govmmQemu.QMPEvent) + go q.loopQMPEvent(events) + + cfg := govmmQemu.QMPConfig{ + Logger: newQMPLogger(), + EventCh: events, + } // Auto-closed by QMPStart(). disconnectCh := make(chan struct{}) @@ -1050,6 +1067,136 @@ func (q *qemu) qmpSetup() error { return nil } +func (q *qemu) loopQMPEvent(event chan govmmQemu.QMPEvent) { + for { + select { + case e, open := <-event: + if !open { + q.Logger().Infof("QMP event channel closed") + return + } + q.Logger().WithField("event", e).Debug("got QMP event") + if e.Name == "GUEST_PANICKED" { + go q.handleGuestPanic() + } + } + } +} + +func (q *qemu) handleGuestPanic() { + if err := q.dumpGuestMemory(q.config.GuestMemoryDumpPath); err != nil { + q.Logger().WithError(err).Error("failed to dump guest memory") + } + + // TODO: how to notify the upper level sandbox to handle the error + // to do a fast fail(shutdown or others). + // tracked by https://github.com/kata-containers/kata-containers/issues/1026 +} + +// canDumpGuestMemory check if can do a guest memory dump operation. +// for now it only ensure there must be double of VM size for free disk spaces +func (q *qemu) canDumpGuestMemory(dumpSavePath string) error { + fs := unix.Statfs_t{} + if err := unix.Statfs(dumpSavePath, &fs); err != nil { + q.Logger().WithError(err).WithField("dumpSavePath", dumpSavePath).Error("failed to call Statfs") + return nil + } + availSpaceInBytes := fs.Bavail * uint64(fs.Bsize) + q.Logger().WithFields( + logrus.Fields{ + "dumpSavePath": dumpSavePath, + "availSpaceInBytes": availSpaceInBytes, + }).Info("get avail space") + + // get guest memory size + guestMemorySizeInBytes := (uint64(q.config.MemorySize) + uint64(q.state.HotpluggedMemory)) << utils.MibToBytesShift + q.Logger().WithField("guestMemorySizeInBytes", guestMemorySizeInBytes).Info("get guest memory size") + + // default we want ensure there are at least double of VM memory size free spaces available, + // this may complete one dump operation for one sandbox + exceptMemorySize := guestMemorySizeInBytes * 2 + if availSpaceInBytes >= exceptMemorySize { + return nil + } else { + return fmt.Errorf("there are not enough free space to store memory dump file. Except %d bytes, but only %d bytes available", exceptMemorySize, availSpaceInBytes) + } +} + +// dumpSandboxMetaInfo save meta information for debug purpose, includes: +// hypervisor verison, sandbox/container state, hypervisor config +func (q *qemu) dumpSandboxMetaInfo(dumpSavePath string) { + dumpStatePath := filepath.Join(dumpSavePath, "state") + + // copy state from /run/vc/sbs to memory dump directory + statePath := filepath.Join(q.store.RunStoragePath(), q.id) + command := []string{"/bin/cp", "-ar", statePath, dumpStatePath} + q.Logger().WithField("command", command).Info("try to save sandbox state") + if output, err := pkgUtils.RunCommandFull(command, true); err != nil { + q.Logger().WithError(err).WithField("output", output).Error("failed to save state") + } + // save hypervisor meta information + fileName := filepath.Join(dumpSavePath, "hypervisor.conf") + data, _ := json.MarshalIndent(q.config, "", " ") + if err := ioutil.WriteFile(fileName, data, defaultFilePerms); err != nil { + q.Logger().WithError(err).WithField("hypervisor.conf", data).Error("write to hypervisor.conf file failed") + } + + // save hypervisor version + hyperVisorVersion, err := pkgUtils.RunCommand([]string{q.config.HypervisorPath, "--version"}) + if err != nil { + q.Logger().WithError(err).WithField("HypervisorPath", data).Error("failed to get hypervisor version") + } + + fileName = filepath.Join(dumpSavePath, "hypervisor.version") + if err := ioutil.WriteFile(fileName, []byte(hyperVisorVersion), defaultFilePerms); err != nil { + q.Logger().WithError(err).WithField("hypervisor.version", data).Error("write to hypervisor.version file failed") + } +} + +func (q *qemu) dumpGuestMemory(dumpSavePath string) error { + if dumpSavePath == "" { + return nil + } + + q.memoryDumpFlag.Lock() + defer q.memoryDumpFlag.Unlock() + + q.Logger().WithField("dumpSavePath", dumpSavePath).Info("try to dump guest memory") + + dumpSavePath = filepath.Join(dumpSavePath, q.id) + dumpStatePath := filepath.Join(dumpSavePath, "state") + if err := pkgUtils.EnsureDir(dumpStatePath, DirMode); err != nil { + return err + } + + // save meta information for sandbox + q.dumpSandboxMetaInfo(dumpSavePath) + q.Logger().Info("dump sandbox meta information completed") + + // check device free space and estimated dump size + if err := q.canDumpGuestMemory(dumpSavePath); err != nil { + q.Logger().Warnf("can't dump guest memory: %s", err.Error()) + return err + } + + // dump guest memory + protocol := fmt.Sprintf("file:%s/vmcore-%s.%s", dumpSavePath, time.Now().Format("20060102150405.999"), memoryDumpFormat) + q.Logger().Infof("try to dump guest memory to %s", protocol) + + if err := q.qmpSetup(); err != nil { + q.Logger().WithError(err).Error("setup manage QMP failed") + return err + } + + if err := q.qmpMonitorCh.qmp.ExecuteDumpGuestMemory(q.qmpMonitorCh.ctx, protocol, q.config.GuestMemoryDumpPaging, memoryDumpFormat); err != nil { + q.Logger().WithError(err).Error("dump guest memory failed") + return err + } + + q.Logger().Info("dump guest memory completed") + return nil +} + func (q *qemu) qmpShutdown() { q.qmpMonitorCh.Lock() defer q.qmpMonitorCh.Unlock() @@ -2250,6 +2397,9 @@ func (q *qemu) load(s persistapi.HypervisorState) { } func (q *qemu) check() error { + q.memoryDumpFlag.Lock() + defer q.memoryDumpFlag.Unlock() + err := q.qmpSetup() if err != nil { return err diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index 4085581b2..b3580562f 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -133,6 +133,9 @@ type qemuArch interface { // append vIOMMU device appendIOMMU(devices []govmmQemu.Device) ([]govmmQemu.Device, error) + + // append pvpanic device + appendPVPanicDevice(devices []govmmQemu.Device) ([]govmmQemu.Device, error) } type qemuArchBase struct { @@ -789,3 +792,9 @@ func (q *qemuArchBase) appendIOMMU(devices []govmmQemu.Device) ([]govmmQemu.Devi return devices, fmt.Errorf("Machine Type %s does not support vIOMMU", q.qemuMachine.Type) } } + +// appendPVPanicDevice appends a pvpanic device +func (q *qemuArchBase) appendPVPanicDevice(devices []govmmQemu.Device) ([]govmmQemu.Device, error) { + devices = append(devices, govmmQemu.PVPanicDevice{NoShutdown: true}) + return devices, nil +}