diff --git a/src/runtime/cli/config/configuration-qemu.toml.in b/src/runtime/cli/config/configuration-qemu.toml.in index 5f74b427d0..9ced772869 100644 --- a/src/runtime/cli/config/configuration-qemu.toml.in +++ b/src/runtime/cli/config/configuration-qemu.toml.in @@ -319,6 +319,26 @@ valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@ # Default 0-sized value means unlimited rate. #tx_rate_limiter_max_rate = 0 +# Set where to save the guest memory dump file. +# If set, when GUEST_PANICKED event occurred, +# guest memeory will be dumped to host filesystem under guest_memory_dump_path, +# This directory will be created automatically if it does not exist. +# +# The dumped file(also called vmcore) can be processed with crash or gdb. +# +# WARNING: +# Dump guest’s memory can take very long depending on the amount of guest memory +# and use much disk space. +#guest_memory_dump_path="/var/crash/kata" + +# If enable paging. +# Basically, if you want to use "gdb" rather than "crash", +# or need the guest-virtual addresses in the ELF vmcore, +# then you should enable paging. +# +# See: https://www.qemu.org/docs/master/qemu-qmp-ref.html#Dump-guest-memory for details +#guest_memory_dump_paging=false + [factory] # VM templating support. Once enabled, new VMs are created from template # using vm cloning. They will share the same initial kernel, initramfs and diff --git a/src/runtime/cli/kata-env.go b/src/runtime/cli/kata-env.go index 01286b3b90..cb928f59e1 100644 --- a/src/runtime/cli/kata-env.go +++ b/src/runtime/cli/kata-env.go @@ -14,6 +14,7 @@ import ( "github.com/BurntSushi/toml" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci" @@ -292,7 +293,7 @@ func getNetmonInfo(config oci.RuntimeConfig) NetmonInfo { } func getCommandVersion(cmd string) (string, error) { - return katautils.RunCommand([]string{cmd, "--version"}) + return utils.RunCommand([]string{cmd, "--version"}) } func getAgentInfo(config oci.RuntimeConfig) (AgentInfo, error) { diff --git a/src/runtime/cli/main_test.go b/src/runtime/cli/main_test.go index cd51acda54..94140162da 100644 --- a/src/runtime/cli/main_test.go +++ b/src/runtime/cli/main_test.go @@ -24,6 +24,7 @@ import ( "github.com/dlespiau/covertool/pkg/cover" ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci" @@ -273,7 +274,7 @@ func createOCIConfig(bundleDir string) error { return errors.New("Cannot find command to generate OCI config file") } - _, err := katautils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir}) + _, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir}) if err != nil { return err } @@ -378,7 +379,7 @@ func makeOCIBundle(bundleDir string) error { } } - output, err := katautils.RunCommandFull([]string{"cp", "-a", from, to}, true) + output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true) if err != nil { return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output) } diff --git a/src/runtime/cli/utils_test.go b/src/runtime/cli/utils_test.go index c5eccddbc4..a222b940b0 100644 --- a/src/runtime/cli/utils_test.go +++ b/src/runtime/cli/utils_test.go @@ -13,6 +13,7 @@ import ( "testing" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" "github.com/stretchr/testify/assert" ) @@ -176,13 +177,13 @@ VERSION_ID="%s" } func TestUtilsRunCommand(t *testing.T) { - output, err := katautils.RunCommand([]string{"true"}) + output, err := utils.RunCommand([]string{"true"}) assert.NoError(t, err) assert.Equal(t, "", output) } func TestUtilsRunCommandCaptureStdout(t *testing.T) { - output, err := katautils.RunCommand([]string{"echo", "hello"}) + output, err := utils.RunCommand([]string{"echo", "hello"}) assert.NoError(t, err) assert.Equal(t, "hello", output) } @@ -190,7 +191,7 @@ func TestUtilsRunCommandCaptureStdout(t *testing.T) { func TestUtilsRunCommandIgnoreStderr(t *testing.T) { args := []string{"/bin/sh", "-c", "echo foo >&2;exit 0"} - output, err := katautils.RunCommand(args) + output, err := utils.RunCommand(args) assert.NoError(t, err) assert.Equal(t, "", output) } @@ -213,7 +214,7 @@ func TestUtilsRunCommandInvalidCmds(t *testing.T) { } for _, args := range invalidCommands { - output, err := katautils.RunCommand(args) + output, err := utils.RunCommand(args) assert.Error(t, err) assert.Equal(t, "", output) } diff --git a/src/runtime/containerd-shim-v2/utils_test.go b/src/runtime/containerd-shim-v2/utils_test.go index 7ba694af03..492a0b41cf 100644 --- a/src/runtime/containerd-shim-v2/utils_test.go +++ b/src/runtime/containerd-shim-v2/utils_test.go @@ -21,6 +21,7 @@ import ( ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci" @@ -150,7 +151,7 @@ func createOCIConfig(bundleDir string) error { return errors.New("Cannot find command to generate OCI config file") } - _, err := katautils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir}) + _, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir}) if err != nil { return err } @@ -278,7 +279,7 @@ func makeOCIBundle(bundleDir string) error { } } - output, err := katautils.RunCommandFull([]string{"cp", "-a", from, to}, true) + output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true) if err != nil { return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output) } diff --git a/src/runtime/go.mod b/src/runtime/go.mod index f755ebd4e5..a0336f87c3 100644 --- a/src/runtime/go.mod +++ b/src/runtime/go.mod @@ -32,7 +32,7 @@ require ( github.com/gogo/googleapis v1.4.0 // indirect github.com/gogo/protobuf v1.3.1 github.com/hashicorp/go-multierror v1.0.0 - github.com/intel/govmm v0.0.0-20200825065022-6042f6033126 + github.com/kata-containers/govmm v0.0.0-20201020052039-99f43ec18864 github.com/mdlayher/vsock v0.0.0-20191108225356-d9c65923cb8f github.com/opencontainers/image-spec v1.0.1 // indirect github.com/opencontainers/runc v1.0.0-rc9.0.20200102164712-2b52db75279c diff --git a/src/runtime/go.sum b/src/runtime/go.sum index ee967e7067..7a37e459af 100644 --- a/src/runtime/go.sum +++ b/src/runtime/go.sum @@ -186,8 +186,6 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/intel/govmm v0.0.0-20200825065022-6042f6033126 h1:yltaUdR0Vitnn/FEfy+JWbJ+oGhMAPP/3S7ja9S5yso= -github.com/intel/govmm v0.0.0-20200825065022-6042f6033126/go.mod h1:QKGWoQtjvkvFtzP6ybiM3lxUHqf83Sv3oLqyELUKH4g= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -198,7 +196,8 @@ github.com/juju/errors v0.0.0-20180806074554-22422dad46e1/go.mod h1:W54LbzXuIE0b github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/testing v0.0.0-20190613124551-e81189438503/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kata-containers/kata-containers v0.0.0-20201013034856-c88820454d08 h1:yk9fzLKb9RmV9xuT5mkJw4owk/K0rX5cusm2ukEEDro= +github.com/kata-containers/govmm v0.0.0-20201020052039-99f43ec18864 h1:ETwjbdr9aU/J90P5D/HAxRW8M4r0HQSPmuBDIaNr9EM= +github.com/kata-containers/govmm v0.0.0-20201020052039-99f43ec18864/go.mod h1:VmAHbsL5lLfzHW/MNL96NVLF840DNEV5i683kISgFKk= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 7ba2281ab9..e542d14757 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -14,7 +14,7 @@ import ( "strings" "github.com/BurntSushi/toml" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental" @@ -125,6 +125,8 @@ type hypervisor struct { RxRateLimiterMaxRate uint64 `toml:"rx_rate_limiter_max_rate"` TxRateLimiterMaxRate uint64 `toml:"tx_rate_limiter_max_rate"` EnableAnnotations []string `toml:"enable_annotations"` + GuestMemoryDumpPath string `toml:"guest_memory_dump_path"` + GuestMemoryDumpPaging bool `toml:"guest_memory_dump_paging"` } type runtime struct { @@ -688,6 +690,8 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { RxRateLimiterMaxRate: rxRateLimiterMaxRate, TxRateLimiterMaxRate: txRateLimiterMaxRate, EnableAnnotations: h.EnableAnnotations, + GuestMemoryDumpPath: h.GuestMemoryDumpPath, + GuestMemoryDumpPaging: h.GuestMemoryDumpPaging, }, nil } @@ -844,7 +848,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { DisableVhostNet: true, VirtioFSExtraArgs: h.VirtioFSExtraArgs, SGXEPCSize: defaultSGXEPCSize, - EnableAnnotations: h.EnableAnnotations, + EnableAnnotations: h.EnableAnnotations, }, nil } diff --git a/src/runtime/pkg/katautils/container_engine.go b/src/runtime/pkg/katautils/container_engine.go index 3b0caf5ab8..39f2f563f3 100644 --- a/src/runtime/pkg/katautils/container_engine.go +++ b/src/runtime/pkg/katautils/container_engine.go @@ -7,6 +7,8 @@ package katautils import ( "os/exec" + + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" ) type CtrEngine struct { @@ -19,7 +21,7 @@ var ( func (e *CtrEngine) Init(name string) (string, error) { var out string - out, err := RunCommandFull([]string{name, "version"}, true) + out, err := utils.RunCommandFull([]string{name, "version"}, true) if err != nil { return out, err } @@ -30,19 +32,19 @@ func (e *CtrEngine) Init(name string) (string, error) { func (e *CtrEngine) Inspect(image string) (string, error) { // Only hit the network if the image doesn't exist locally - return RunCommand([]string{e.Name, "inspect", "--type=image", image}) + return utils.RunCommand([]string{e.Name, "inspect", "--type=image", image}) } func (e *CtrEngine) Pull(image string) (string, error) { - return RunCommand([]string{e.Name, "pull", image}) + return utils.RunCommand([]string{e.Name, "pull", image}) } func (e *CtrEngine) Create(image string) (string, error) { - return RunCommand([]string{e.Name, "create", image}) + return utils.RunCommand([]string{e.Name, "create", image}) } func (e *CtrEngine) Rm(ctrID string) (string, error) { - return RunCommand([]string{e.Name, "rm", ctrID}) + return utils.RunCommand([]string{e.Name, "rm", ctrID}) } func (e *CtrEngine) GetRootfs(ctrID string, dir string) error { diff --git a/src/runtime/pkg/katautils/create_test.go b/src/runtime/pkg/katautils/create_test.go index fb14d99b9c..7f033082f6 100644 --- a/src/runtime/pkg/katautils/create_test.go +++ b/src/runtime/pkg/katautils/create_test.go @@ -20,6 +20,7 @@ import ( "testing" ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci" @@ -87,7 +88,7 @@ func makeOCIBundle(bundleDir string) error { } } - output, err := RunCommandFull([]string{"cp", "-a", from, to}, true) + output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true) if err != nil { return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output) } diff --git a/src/runtime/pkg/katautils/utils.go b/src/runtime/pkg/katautils/utils.go index d892576999..02c2ea568f 100644 --- a/src/runtime/pkg/katautils/utils.go +++ b/src/runtime/pkg/katautils/utils.go @@ -8,13 +8,12 @@ package katautils import ( "fmt" - "golang.org/x/sys/unix" "io/ioutil" "os" - "os/exec" "path/filepath" - "strings" "syscall" + + "golang.org/x/sys/unix" ) // FileExists test is a file exiting or not @@ -110,27 +109,3 @@ func GetFileContents(file string) (string, error) { return string(bytes), nil } - -// RunCommandFull returns the commands space-trimmed standard output and -// error on success. Note that if the command fails, the requested output will -// still be returned, along with an error. -func RunCommandFull(args []string, includeStderr bool) (string, error) { - cmd := exec.Command(args[0], args[1:]...) - var err error - var bytes []byte - - if includeStderr { - bytes, err = cmd.CombinedOutput() - } else { - bytes, err = cmd.Output() - } - - trimmed := strings.TrimSpace(string(bytes)) - - return trimmed, err -} - -// RunCommand returns the commands space-trimmed standard output on success -func RunCommand(args []string) (string, error) { - return RunCommandFull(args, false) -} diff --git a/src/runtime/pkg/katautils/utils_test.go b/src/runtime/pkg/katautils/utils_test.go index 2314806224..984efc3483 100644 --- a/src/runtime/pkg/katautils/utils_test.go +++ b/src/runtime/pkg/katautils/utils_test.go @@ -18,6 +18,7 @@ import ( "syscall" "testing" + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci" "github.com/stretchr/testify/assert" ) @@ -90,7 +91,7 @@ func createOCIConfig(bundleDir string) error { return errors.New("Cannot find command to generate OCI config file") } - _, err := RunCommand([]string{configCmd, "spec", "--bundle", bundleDir}) + _, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir}) if err != nil { return err } diff --git a/src/runtime/pkg/utils/utils.go b/src/runtime/pkg/utils/utils.go index 7993020c3c..fd09bbd70e 100644 --- a/src/runtime/pkg/utils/utils.go +++ b/src/runtime/pkg/utils/utils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 Ant Financial +// Copyright (c) 2020 Ant Group // // SPDX-License-Identifier: Apache-2.0 // @@ -6,7 +6,11 @@ package utils import ( + "fmt" "net/http" + "os" + "os/exec" + "path/filepath" "strings" ) @@ -31,3 +35,48 @@ func GzipAccepted(header http.Header) bool { func String2Pointer(s string) *string { return &s } + +// RunCommandFull returns the commands space-trimmed standard output and +// error on success. Note that if the command fails, the requested output will +// still be returned, along with an error. +func RunCommandFull(args []string, includeStderr bool) (string, error) { + cmd := exec.Command(args[0], args[1:]...) + var err error + var bytes []byte + + if includeStderr { + bytes, err = cmd.CombinedOutput() + } else { + bytes, err = cmd.Output() + } + + trimmed := strings.TrimSpace(string(bytes)) + + return trimmed, err +} + +// RunCommand returns the commands space-trimmed standard output on success +func RunCommand(args []string) (string, error) { + return RunCommandFull(args, false) +} + +// EnsureDir check if a directory exist, if not then create it +func EnsureDir(path string, mode os.FileMode) error { + if !filepath.IsAbs(path) { + return fmt.Errorf("Not an absolute path: %s", path) + } + + if fi, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + if err = os.MkdirAll(path, mode); err != nil { + return err + } + } else { + return err + } + } else if !fi.IsDir() { + return fmt.Errorf("Not a directory: %s", path) + } + + return nil +} diff --git a/src/runtime/pkg/utils/utils_test.go b/src/runtime/pkg/utils/utils_test.go index 3139b2ce1c..30189521d4 100644 --- a/src/runtime/pkg/utils/utils_test.go +++ b/src/runtime/pkg/utils/utils_test.go @@ -6,7 +6,10 @@ package utils import ( + "fmt" + "io/ioutil" "net/http" + "os" "testing" "github.com/stretchr/testify/assert" @@ -45,3 +48,71 @@ func TestGzipAccepted(t *testing.T) { assert.Equal(tc.result, b) } } + +func TestEnsureDir(t *testing.T) { + const testMode = 0755 + tmpdir, err := ioutil.TempDir("", "TestEnsureDir") + assert := assert.New(t) + + assert.NoError(err) + defer os.RemoveAll(tmpdir) + + testCases := []struct { + before func() + path string + err bool + msg string + }{ + { + before: nil, + path: "a/b/c", + err: true, + msg: "Not an absolute path", + }, + { + before: nil, + path: fmt.Sprintf("%s/abc/def", tmpdir), + err: false, + msg: "", + }, + { + before: nil, + path: fmt.Sprintf("%s/abc", tmpdir), + err: false, + msg: "", + }, + { + before: func() { + err := os.MkdirAll(fmt.Sprintf("%s/abc/def", tmpdir), testMode) + assert.NoError(err) + }, + path: fmt.Sprintf("%s/abc/def", tmpdir), + err: false, + msg: "", + }, + { + before: func() { + // create a regular file + err := os.MkdirAll(fmt.Sprintf("%s/abc", tmpdir), testMode) + assert.NoError(err) + _, err = os.Create(fmt.Sprintf("%s/abc/file.txt", tmpdir)) + assert.NoError(err) + }, + path: fmt.Sprintf("%s/abc/file.txt", tmpdir), + err: true, + msg: "Not a directory", + }, + } + + for _, tc := range testCases { + if tc.before != nil { + tc.before() + } + err := EnsureDir(tc.path, testMode) + if tc.err { + assert.Contains(err.Error(), tc.msg, "error msg should contains: %s, but got %s", tc.msg, err.Error()) + } else { + assert.Equal(err, nil, "failed for path: %s, except no error, but got %+v", tc.path, err) + } + } +} diff --git a/src/runtime/vendor/github.com/intel/govmm/CONTRIBUTORS.md b/src/runtime/vendor/github.com/intel/govmm/CONTRIBUTORS.md deleted file mode 100644 index 7a822adbe0..0000000000 --- a/src/runtime/vendor/github.com/intel/govmm/CONTRIBUTORS.md +++ /dev/null @@ -1,22 +0,0 @@ -This file is a partial list of contributors to the Virtual Machine -Manager for Go project. To see the full list of contributors, see the -revision history in source control. - -Contributors who wish to be recognized in this file should add -themselves (or their employer, as appropriate). - -- afrosi@de.ibm.com -- archana.m.shinde@intel.com -- caoruidong@huawei.com -- clare.chenhui@huawei.com -- eric.ernst@intel.com -- james.o.hunt@intel.com -- jose.carlos.venegas.munoz@intel.com -- julio.montes@intel.com -- manohar.r.castelino@intel.com -- mark.d.ryan@intel.com -- robert.bradford@intel.com -- sameo@linux.intel.com -- sebastien.boeuf@intel.com -- teawater@hyper.sh -- xinda.zhao@intel.com diff --git a/src/runtime/vendor/github.com/intel/govmm/COPYING b/src/runtime/vendor/github.com/kata-containers/govmm/COPYING similarity index 100% rename from src/runtime/vendor/github.com/intel/govmm/COPYING rename to src/runtime/vendor/github.com/kata-containers/govmm/COPYING diff --git a/src/runtime/vendor/github.com/intel/govmm/qemu/image.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/image.go similarity index 100% rename from src/runtime/vendor/github.com/intel/govmm/qemu/image.go rename to src/runtime/vendor/github.com/kata-containers/govmm/qemu/image.go diff --git a/src/runtime/vendor/github.com/intel/govmm/qemu/qemu.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qemu.go similarity index 97% rename from src/runtime/vendor/github.com/intel/govmm/qemu/qemu.go rename to src/runtime/vendor/github.com/kata-containers/govmm/qemu/qemu.go index 9288f59f8a..9b97aba590 100644 --- a/src/runtime/vendor/github.com/intel/govmm/qemu/qemu.go +++ b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qemu.go @@ -1084,6 +1084,8 @@ func (blkdev BlockDevice) QemuParams(config *Config) []string { deviceParams = append(deviceParams, fmt.Sprintf(",share-rw=on")) } + deviceParams = append(deviceParams, fmt.Sprintf(",serial=%s", blkdev.ID)) + blkParams = append(blkParams, fmt.Sprintf("id=%s", blkdev.ID)) blkParams = append(blkParams, fmt.Sprintf(",file=%s", blkdev.File)) blkParams = append(blkParams, fmt.Sprintf(",aio=%s", blkdev.AIO)) @@ -1118,6 +1120,24 @@ func (blkdev BlockDevice) deviceName(config *Config) string { return string(blkdev.Driver) } +// PVPanicDevice represents a qemu pvpanic device. +type PVPanicDevice struct { + NoShutdown bool +} + +// Valid always returns true for pvpanic device +func (dev PVPanicDevice) Valid() bool { + return true +} + +// QemuParams returns the qemu parameters built out of this serial device. +func (dev PVPanicDevice) QemuParams(config *Config) []string { + if dev.NoShutdown { + return []string{"-device", "pvpanic", "-no-shutdown"} + } + return []string{"-device", "pvpanic"} +} + // VhostUserDevice represents a qemu vhost-user device meant to be passed // in to the guest type VhostUserDevice struct { @@ -2103,6 +2123,56 @@ type Kernel struct { Params string } +// FwCfg allows QEMU to pass entries to the guest +// File and Str are mutually exclusive +type FwCfg struct { + Name string + File string + Str string +} + +// Valid returns true if the FwCfg structure is valid and complete. +func (fwcfg FwCfg) Valid() bool { + if fwcfg.Name == "" { + return false + } + + if fwcfg.File != "" && fwcfg.Str != "" { + return false + } + + if fwcfg.File == "" && fwcfg.Str == "" { + return false + } + + return true +} + +// QemuParams returns the qemu parameters built out of the FwCfg object +func (fwcfg FwCfg) QemuParams(config *Config) []string { + var fwcfgParams []string + var qemuParams []string + + for _, f := range config.FwCfg { + if f.Name != "" { + fwcfgParams = append(fwcfgParams, fmt.Sprintf("name=%s", f.Name)) + + if f.File != "" { + fwcfgParams = append(fwcfgParams, fmt.Sprintf(",file=%s", f.File)) + } + + if f.Str != "" { + fwcfgParams = append(fwcfgParams, fmt.Sprintf(",string=%s", f.Str)) + } + } + + qemuParams = append(qemuParams, "-fw_cfg") + qemuParams = append(qemuParams, strings.Join(fwcfgParams, "")) + } + + return qemuParams +} + // Knobs regroups a set of qemu boolean settings type Knobs struct { // NoUserConfig prevents qemu from loading user config files. @@ -2236,6 +2306,9 @@ type Config struct { // fds is a list of open file descriptors to be passed to the spawned qemu process fds []*os.File + // FwCfg is the -fw_cfg parameter + FwCfg []FwCfg + IOThreads []IOThread // PidFile is the -pidfile parameter @@ -2568,6 +2641,21 @@ func (config *Config) appendLogFile() { } } +func (config *Config) appendFwCfg(logger QMPLog) { + if logger == nil { + logger = qmpNullLogger{} + } + + for _, f := range config.FwCfg { + if !f.Valid() { + logger.Errorf("fw_cfg is not valid: %+v", config.FwCfg) + continue + } + + config.qemuParams = append(config.qemuParams, f.QemuParams(config)...) + } +} + // LaunchQemu can be used to launch a new qemu instance. // // The Config parameter contains a set of qemu parameters and settings. @@ -2595,6 +2683,7 @@ func LaunchQemu(config Config, logger QMPLog) (string, error) { config.appendIncoming() config.appendPidFile() config.appendLogFile() + config.appendFwCfg(logger) if err := config.appendCPUs(); err != nil { return "", err diff --git a/src/runtime/vendor/github.com/intel/govmm/qemu/qmp.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go similarity index 98% rename from src/runtime/vendor/github.com/intel/govmm/qemu/qmp.go rename to src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go index 4942727c42..53ba105a8e 100644 --- a/src/runtime/vendor/github.com/intel/govmm/qemu/qmp.go +++ b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go @@ -268,7 +268,7 @@ func (q *QMP) readLoop(fromVMCh chan<- []byte) { for scanner.Scan() { line := scanner.Bytes() if q.cfg.Logger.V(1) { - q.cfg.Logger.Infof("%s", string(line)) + q.cfg.Logger.Infof("read from QMP: %s", string(line)) } // Since []byte channel type transfer slice info(include slice underlying array pointer, len, cap) @@ -1639,3 +1639,29 @@ func (q *QMP) ExecQomSet(ctx context.Context, path, property string, value uint6 return q.executeCommand(ctx, "qom-set", args, nil) } + +// ExecQomGet qom-get path property +func (q *QMP) ExecQomGet(ctx context.Context, path, property string) (interface{}, error) { + args := map[string]interface{}{ + "path": path, + "property": property, + } + + response, err := q.executeCommandWithResponse(ctx, "qom-get", args, nil, nil) + if err != nil { + return "", err + } + + return response, nil +} + +// ExecuteDumpGuestMemory dump guest memory to host +func (q *QMP) ExecuteDumpGuestMemory(ctx context.Context, protocol string, paging bool, format string) error { + args := map[string]interface{}{ + "protocol": protocol, + "paging": paging, + "format": format, + } + + return q.executeCommand(ctx, "dump-guest-memory", args, nil) +} diff --git a/src/runtime/vendor/github.com/sirupsen/logrus/go.mod b/src/runtime/vendor/github.com/sirupsen/logrus/go.mod index 12fdf98984..e5522f30ef 100644 --- a/src/runtime/vendor/github.com/sirupsen/logrus/go.mod +++ b/src/runtime/vendor/github.com/sirupsen/logrus/go.mod @@ -1,5 +1,7 @@ module github.com/sirupsen/logrus +go 1.15 + require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.1 diff --git a/src/runtime/vendor/modules.txt b/src/runtime/vendor/modules.txt index 9438540fbb..8afde68e8c 100644 --- a/src/runtime/vendor/modules.txt +++ b/src/runtime/vendor/modules.txt @@ -222,9 +222,9 @@ github.com/hashicorp/errwrap # github.com/hashicorp/go-multierror v1.0.0 ## explicit github.com/hashicorp/go-multierror -# github.com/intel/govmm v0.0.0-20200825065022-6042f6033126 +# github.com/kata-containers/govmm v0.0.0-20201020052039-99f43ec18864 ## explicit -github.com/intel/govmm/qemu +github.com/kata-containers/govmm/qemu # github.com/konsorten/go-windows-terminal-sequences v1.0.1 github.com/konsorten/go-windows-terminal-sequences # github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 639808659f..ee7a2b6b73 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -440,6 +440,13 @@ type HypervisorConfig struct { // Enable annotations by name EnableAnnotations []string + + // GuestCoredumpPath is the path in host for saving guest memory dump + GuestMemoryDumpPath string + + // GuestMemoryDumpPaging is used to indicate if enable paging + // for QEMU dump-guest-memory command + GuestMemoryDumpPaging bool } // vcpu mapping from vcpu number to thread number @@ -607,6 +614,10 @@ func (conf *HypervisorConfig) HypervisorAssetPath() (string, error) { return conf.assetPath(types.HypervisorAsset) } +func (conf *HypervisorConfig) IfPVPanicEnabled() bool { + return conf.GuestMemoryDumpPath != "" +} + // HypervisorCtlAssetPath returns the VM hypervisor ctl path func (conf *HypervisorConfig) HypervisorCtlAssetPath() (string, error) { return conf.assetPath(types.HypervisorCtlAsset) diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index d503e57bd5..6dff586ac4 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -24,13 +24,14 @@ import ( "time" "unsafe" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/opencontainers/selinux/go-selinux/label" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" + pkgUtils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid" @@ -99,6 +100,9 @@ type qemu struct { stopped bool store persistapi.PersistDriver + + // if in memory dump progress + memoryDumpFlag sync.Mutex } const ( @@ -106,6 +110,9 @@ const ( qmpSocket = "qmp.sock" vhostFSSocket = "vhost-fs.sock" + // memory dump format will be set to elf + memoryDumpFormat = "elf" + qmpCapErrMsg = "Failed to negoatiate QMP capabilities" qmpExecCatCmd = "exec:cat" @@ -408,13 +415,17 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I } } + if q.config.IfPVPanicEnabled() { + // there should have no errors for pvpanic device + devices, _ = q.arch.appendPVPanicDevice(devices) + } + var ioThread *govmmQemu.IOThread if q.config.BlockDeviceDriver == config.VirtioSCSI { return q.arch.appendSCSIController(devices, q.config.EnableIOThreads) } return devices, ioThread, nil - } func (q *qemu) setupTemplate(knobs *govmmQemu.Knobs, memory *govmmQemu.Memory) govmmQemu.Incoming { @@ -1027,7 +1038,13 @@ func (q *qemu) qmpSetup() error { return nil } - cfg := govmmQemu.QMPConfig{Logger: newQMPLogger()} + events := make(chan govmmQemu.QMPEvent) + go q.loopQMPEvent(events) + + cfg := govmmQemu.QMPConfig{ + Logger: newQMPLogger(), + EventCh: events, + } // Auto-closed by QMPStart(). disconnectCh := make(chan struct{}) @@ -1050,6 +1067,136 @@ func (q *qemu) qmpSetup() error { return nil } +func (q *qemu) loopQMPEvent(event chan govmmQemu.QMPEvent) { + for { + select { + case e, open := <-event: + if !open { + q.Logger().Infof("QMP event channel closed") + return + } + q.Logger().WithField("event", e).Debug("got QMP event") + if e.Name == "GUEST_PANICKED" { + go q.handleGuestPanic() + } + } + } +} + +func (q *qemu) handleGuestPanic() { + if err := q.dumpGuestMemory(q.config.GuestMemoryDumpPath); err != nil { + q.Logger().WithError(err).Error("failed to dump guest memory") + } + + // TODO: how to notify the upper level sandbox to handle the error + // to do a fast fail(shutdown or others). + // tracked by https://github.com/kata-containers/kata-containers/issues/1026 +} + +// canDumpGuestMemory check if can do a guest memory dump operation. +// for now it only ensure there must be double of VM size for free disk spaces +func (q *qemu) canDumpGuestMemory(dumpSavePath string) error { + fs := unix.Statfs_t{} + if err := unix.Statfs(dumpSavePath, &fs); err != nil { + q.Logger().WithError(err).WithField("dumpSavePath", dumpSavePath).Error("failed to call Statfs") + return nil + } + availSpaceInBytes := fs.Bavail * uint64(fs.Bsize) + q.Logger().WithFields( + logrus.Fields{ + "dumpSavePath": dumpSavePath, + "availSpaceInBytes": availSpaceInBytes, + }).Info("get avail space") + + // get guest memory size + guestMemorySizeInBytes := (uint64(q.config.MemorySize) + uint64(q.state.HotpluggedMemory)) << utils.MibToBytesShift + q.Logger().WithField("guestMemorySizeInBytes", guestMemorySizeInBytes).Info("get guest memory size") + + // default we want ensure there are at least double of VM memory size free spaces available, + // this may complete one dump operation for one sandbox + exceptMemorySize := guestMemorySizeInBytes * 2 + if availSpaceInBytes >= exceptMemorySize { + return nil + } else { + return fmt.Errorf("there are not enough free space to store memory dump file. Except %d bytes, but only %d bytes available", exceptMemorySize, availSpaceInBytes) + } +} + +// dumpSandboxMetaInfo save meta information for debug purpose, includes: +// hypervisor verison, sandbox/container state, hypervisor config +func (q *qemu) dumpSandboxMetaInfo(dumpSavePath string) { + dumpStatePath := filepath.Join(dumpSavePath, "state") + + // copy state from /run/vc/sbs to memory dump directory + statePath := filepath.Join(q.store.RunStoragePath(), q.id) + command := []string{"/bin/cp", "-ar", statePath, dumpStatePath} + q.Logger().WithField("command", command).Info("try to save sandbox state") + if output, err := pkgUtils.RunCommandFull(command, true); err != nil { + q.Logger().WithError(err).WithField("output", output).Error("failed to save state") + } + // save hypervisor meta information + fileName := filepath.Join(dumpSavePath, "hypervisor.conf") + data, _ := json.MarshalIndent(q.config, "", " ") + if err := ioutil.WriteFile(fileName, data, defaultFilePerms); err != nil { + q.Logger().WithError(err).WithField("hypervisor.conf", data).Error("write to hypervisor.conf file failed") + } + + // save hypervisor version + hyperVisorVersion, err := pkgUtils.RunCommand([]string{q.config.HypervisorPath, "--version"}) + if err != nil { + q.Logger().WithError(err).WithField("HypervisorPath", data).Error("failed to get hypervisor version") + } + + fileName = filepath.Join(dumpSavePath, "hypervisor.version") + if err := ioutil.WriteFile(fileName, []byte(hyperVisorVersion), defaultFilePerms); err != nil { + q.Logger().WithError(err).WithField("hypervisor.version", data).Error("write to hypervisor.version file failed") + } +} + +func (q *qemu) dumpGuestMemory(dumpSavePath string) error { + if dumpSavePath == "" { + return nil + } + + q.memoryDumpFlag.Lock() + defer q.memoryDumpFlag.Unlock() + + q.Logger().WithField("dumpSavePath", dumpSavePath).Info("try to dump guest memory") + + dumpSavePath = filepath.Join(dumpSavePath, q.id) + dumpStatePath := filepath.Join(dumpSavePath, "state") + if err := pkgUtils.EnsureDir(dumpStatePath, DirMode); err != nil { + return err + } + + // save meta information for sandbox + q.dumpSandboxMetaInfo(dumpSavePath) + q.Logger().Info("dump sandbox meta information completed") + + // check device free space and estimated dump size + if err := q.canDumpGuestMemory(dumpSavePath); err != nil { + q.Logger().Warnf("can't dump guest memory: %s", err.Error()) + return err + } + + // dump guest memory + protocol := fmt.Sprintf("file:%s/vmcore-%s.%s", dumpSavePath, time.Now().Format("20060102150405.999"), memoryDumpFormat) + q.Logger().Infof("try to dump guest memory to %s", protocol) + + if err := q.qmpSetup(); err != nil { + q.Logger().WithError(err).Error("setup manage QMP failed") + return err + } + + if err := q.qmpMonitorCh.qmp.ExecuteDumpGuestMemory(q.qmpMonitorCh.ctx, protocol, q.config.GuestMemoryDumpPaging, memoryDumpFormat); err != nil { + q.Logger().WithError(err).Error("dump guest memory failed") + return err + } + + q.Logger().Info("dump guest memory completed") + return nil +} + func (q *qemu) qmpShutdown() { q.qmpMonitorCh.Lock() defer q.qmpMonitorCh.Unlock() @@ -2250,6 +2397,9 @@ func (q *qemu) load(s persistapi.HypervisorState) { } func (q *qemu) check() error { + q.memoryDumpFlag.Lock() + defer q.memoryDumpFlag.Unlock() + err := q.qmpSetup() if err != nil { return err diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go index b043a9e0c3..1fedcd7f70 100644 --- a/src/runtime/virtcontainers/qemu_amd64.go +++ b/src/runtime/virtcontainers/qemu_amd64.go @@ -11,7 +11,7 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" ) type qemuAmd64 struct { diff --git a/src/runtime/virtcontainers/qemu_amd64_test.go b/src/runtime/virtcontainers/qemu_amd64_test.go index ba810299f7..2a2b694429 100644 --- a/src/runtime/virtcontainers/qemu_amd64_test.go +++ b/src/runtime/virtcontainers/qemu_amd64_test.go @@ -11,7 +11,7 @@ import ( "os" "testing" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" ) diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index a6c83a1f10..b3580562f6 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -14,7 +14,7 @@ import ( "strconv" "strings" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" @@ -133,6 +133,9 @@ type qemuArch interface { // append vIOMMU device appendIOMMU(devices []govmmQemu.Device) ([]govmmQemu.Device, error) + + // append pvpanic device + appendPVPanicDevice(devices []govmmQemu.Device) ([]govmmQemu.Device, error) } type qemuArchBase struct { @@ -789,3 +792,9 @@ func (q *qemuArchBase) appendIOMMU(devices []govmmQemu.Device) ([]govmmQemu.Devi return devices, fmt.Errorf("Machine Type %s does not support vIOMMU", q.qemuMachine.Type) } } + +// appendPVPanicDevice appends a pvpanic device +func (q *qemuArchBase) appendPVPanicDevice(devices []govmmQemu.Device) ([]govmmQemu.Device, error) { + devices = append(devices, govmmQemu.PVPanicDevice{NoShutdown: true}) + return devices, nil +} diff --git a/src/runtime/virtcontainers/qemu_arch_base_test.go b/src/runtime/virtcontainers/qemu_arch_base_test.go index 1204545d21..31df8a7cc4 100644 --- a/src/runtime/virtcontainers/qemu_arch_base_test.go +++ b/src/runtime/virtcontainers/qemu_arch_base_test.go @@ -13,7 +13,7 @@ import ( "path/filepath" "testing" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/stretchr/testify/assert" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" diff --git a/src/runtime/virtcontainers/qemu_arm64.go b/src/runtime/virtcontainers/qemu_arm64.go index daef7fa3ff..14fdc4627e 100644 --- a/src/runtime/virtcontainers/qemu_arm64.go +++ b/src/runtime/virtcontainers/qemu_arm64.go @@ -10,7 +10,7 @@ import ( "fmt" "time" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" ) diff --git a/src/runtime/virtcontainers/qemu_arm64_test.go b/src/runtime/virtcontainers/qemu_arm64_test.go index 58e30ae33b..7b0110c5ef 100644 --- a/src/runtime/virtcontainers/qemu_arm64_test.go +++ b/src/runtime/virtcontainers/qemu_arm64_test.go @@ -11,7 +11,7 @@ import ( "os" "testing" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/stretchr/testify/assert" ) diff --git a/src/runtime/virtcontainers/qemu_ppc64le.go b/src/runtime/virtcontainers/qemu_ppc64le.go index 86fc141d79..0a8bbeb6b2 100644 --- a/src/runtime/virtcontainers/qemu_ppc64le.go +++ b/src/runtime/virtcontainers/qemu_ppc64le.go @@ -9,7 +9,7 @@ import ( "fmt" "time" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/sirupsen/logrus" ) diff --git a/src/runtime/virtcontainers/qemu_ppc64le_test.go b/src/runtime/virtcontainers/qemu_ppc64le_test.go index f48e2bae0b..c5e1038de5 100644 --- a/src/runtime/virtcontainers/qemu_ppc64le_test.go +++ b/src/runtime/virtcontainers/qemu_ppc64le_test.go @@ -9,7 +9,7 @@ import ( "fmt" "testing" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/stretchr/testify/assert" ) diff --git a/src/runtime/virtcontainers/qemu_s390x.go b/src/runtime/virtcontainers/qemu_s390x.go index a43fe4bff8..d829efa9d3 100644 --- a/src/runtime/virtcontainers/qemu_s390x.go +++ b/src/runtime/virtcontainers/qemu_s390x.go @@ -9,7 +9,7 @@ import ( "fmt" "time" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" ) diff --git a/src/runtime/virtcontainers/qemu_s390x_test.go b/src/runtime/virtcontainers/qemu_s390x_test.go index 4cdeeb245e..274329054b 100644 --- a/src/runtime/virtcontainers/qemu_s390x_test.go +++ b/src/runtime/virtcontainers/qemu_s390x_test.go @@ -9,7 +9,7 @@ import ( "fmt" "testing" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/stretchr/testify/assert" ) diff --git a/src/runtime/virtcontainers/qemu_test.go b/src/runtime/virtcontainers/qemu_test.go index 15e4de64c1..80ba64109c 100644 --- a/src/runtime/virtcontainers/qemu_test.go +++ b/src/runtime/virtcontainers/qemu_test.go @@ -14,7 +14,7 @@ import ( "strings" "testing" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"