mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-06-25 15:02:45 +00:00
runtime: add geust memory dump
When guest panic, dump guest kernel memory to host filesystem. And also includes: - hypervisor config - hypervisor version - and state of sandbox Fixes: #1012 Signed-off-by: bin liu <bin@hyper.sh>
This commit is contained in:
parent
5b065eb599
commit
40418f6d88
@ -319,6 +319,26 @@ valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@
|
||||
# Default 0-sized value means unlimited rate.
|
||||
#tx_rate_limiter_max_rate = 0
|
||||
|
||||
# Set where to save the guest memory dump file.
|
||||
# If set, when GUEST_PANICKED event occurred,
|
||||
# guest memeory will be dumped to host filesystem under guest_memory_dump_path,
|
||||
# This directory will be created automatically if it does not exist.
|
||||
#
|
||||
# The dumped file(also called vmcore) can be processed with crash or gdb.
|
||||
#
|
||||
# WARNING:
|
||||
# Dump guest’s memory can take very long depending on the amount of guest memory
|
||||
# and use much disk space.
|
||||
#guest_memory_dump_path="/var/crash/kata"
|
||||
|
||||
# If enable paging.
|
||||
# Basically, if you want to use "gdb" rather than "crash",
|
||||
# or need the guest-virtual addresses in the ELF vmcore,
|
||||
# then you should enable paging.
|
||||
#
|
||||
# See: https://www.qemu.org/docs/master/qemu-qmp-ref.html#Dump-guest-memory for details
|
||||
#guest_memory_dump_paging=false
|
||||
|
||||
[factory]
|
||||
# VM templating support. Once enabled, new VMs are created from template
|
||||
# using vm cloning. They will share the same initial kernel, initramfs and
|
||||
|
@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
|
||||
vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
|
||||
exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
|
||||
@ -292,7 +293,7 @@ func getNetmonInfo(config oci.RuntimeConfig) NetmonInfo {
|
||||
}
|
||||
|
||||
func getCommandVersion(cmd string) (string, error) {
|
||||
return katautils.RunCommand([]string{cmd, "--version"})
|
||||
return utils.RunCommand([]string{cmd, "--version"})
|
||||
}
|
||||
|
||||
func getAgentInfo(config oci.RuntimeConfig) (AgentInfo, error) {
|
||||
|
@ -24,6 +24,7 @@ import (
|
||||
"github.com/dlespiau/covertool/pkg/cover"
|
||||
ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
|
||||
vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
|
||||
@ -273,7 +274,7 @@ func createOCIConfig(bundleDir string) error {
|
||||
return errors.New("Cannot find command to generate OCI config file")
|
||||
}
|
||||
|
||||
_, err := katautils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
|
||||
_, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -378,7 +379,7 @@ func makeOCIBundle(bundleDir string) error {
|
||||
}
|
||||
}
|
||||
|
||||
output, err := katautils.RunCommandFull([]string{"cp", "-a", from, to}, true)
|
||||
output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
|
||||
}
|
||||
|
@ -13,6 +13,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
@ -176,13 +177,13 @@ VERSION_ID="%s"
|
||||
}
|
||||
|
||||
func TestUtilsRunCommand(t *testing.T) {
|
||||
output, err := katautils.RunCommand([]string{"true"})
|
||||
output, err := utils.RunCommand([]string{"true"})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "", output)
|
||||
}
|
||||
|
||||
func TestUtilsRunCommandCaptureStdout(t *testing.T) {
|
||||
output, err := katautils.RunCommand([]string{"echo", "hello"})
|
||||
output, err := utils.RunCommand([]string{"echo", "hello"})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "hello", output)
|
||||
}
|
||||
@ -190,7 +191,7 @@ func TestUtilsRunCommandCaptureStdout(t *testing.T) {
|
||||
func TestUtilsRunCommandIgnoreStderr(t *testing.T) {
|
||||
args := []string{"/bin/sh", "-c", "echo foo >&2;exit 0"}
|
||||
|
||||
output, err := katautils.RunCommand(args)
|
||||
output, err := utils.RunCommand(args)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "", output)
|
||||
}
|
||||
@ -213,7 +214,7 @@ func TestUtilsRunCommandInvalidCmds(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, args := range invalidCommands {
|
||||
output, err := katautils.RunCommand(args)
|
||||
output, err := utils.RunCommand(args)
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, "", output)
|
||||
}
|
||||
|
@ -21,6 +21,7 @@ import (
|
||||
|
||||
ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
|
||||
vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
|
||||
@ -150,7 +151,7 @@ func createOCIConfig(bundleDir string) error {
|
||||
return errors.New("Cannot find command to generate OCI config file")
|
||||
}
|
||||
|
||||
_, err := katautils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
|
||||
_, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -278,7 +279,7 @@ func makeOCIBundle(bundleDir string) error {
|
||||
}
|
||||
}
|
||||
|
||||
output, err := katautils.RunCommandFull([]string{"cp", "-a", from, to}, true)
|
||||
output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
|
||||
}
|
||||
|
@ -125,6 +125,8 @@ type hypervisor struct {
|
||||
RxRateLimiterMaxRate uint64 `toml:"rx_rate_limiter_max_rate"`
|
||||
TxRateLimiterMaxRate uint64 `toml:"tx_rate_limiter_max_rate"`
|
||||
EnableAnnotations []string `toml:"enable_annotations"`
|
||||
GuestMemoryDumpPath string `toml:"guest_memory_dump_path"`
|
||||
GuestMemoryDumpPaging bool `toml:"guest_memory_dump_paging"`
|
||||
}
|
||||
|
||||
type runtime struct {
|
||||
@ -688,6 +690,8 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||
RxRateLimiterMaxRate: rxRateLimiterMaxRate,
|
||||
TxRateLimiterMaxRate: txRateLimiterMaxRate,
|
||||
EnableAnnotations: h.EnableAnnotations,
|
||||
GuestMemoryDumpPath: h.GuestMemoryDumpPath,
|
||||
GuestMemoryDumpPaging: h.GuestMemoryDumpPaging,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
@ -7,6 +7,8 @@ package katautils
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
|
||||
"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
|
||||
)
|
||||
|
||||
type CtrEngine struct {
|
||||
@ -19,7 +21,7 @@ var (
|
||||
|
||||
func (e *CtrEngine) Init(name string) (string, error) {
|
||||
var out string
|
||||
out, err := RunCommandFull([]string{name, "version"}, true)
|
||||
out, err := utils.RunCommandFull([]string{name, "version"}, true)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
@ -30,19 +32,19 @@ func (e *CtrEngine) Init(name string) (string, error) {
|
||||
|
||||
func (e *CtrEngine) Inspect(image string) (string, error) {
|
||||
// Only hit the network if the image doesn't exist locally
|
||||
return RunCommand([]string{e.Name, "inspect", "--type=image", image})
|
||||
return utils.RunCommand([]string{e.Name, "inspect", "--type=image", image})
|
||||
}
|
||||
|
||||
func (e *CtrEngine) Pull(image string) (string, error) {
|
||||
return RunCommand([]string{e.Name, "pull", image})
|
||||
return utils.RunCommand([]string{e.Name, "pull", image})
|
||||
}
|
||||
|
||||
func (e *CtrEngine) Create(image string) (string, error) {
|
||||
return RunCommand([]string{e.Name, "create", image})
|
||||
return utils.RunCommand([]string{e.Name, "create", image})
|
||||
}
|
||||
|
||||
func (e *CtrEngine) Rm(ctrID string) (string, error) {
|
||||
return RunCommand([]string{e.Name, "rm", ctrID})
|
||||
return utils.RunCommand([]string{e.Name, "rm", ctrID})
|
||||
}
|
||||
|
||||
func (e *CtrEngine) GetRootfs(ctrID string, dir string) error {
|
||||
|
@ -20,6 +20,7 @@ import (
|
||||
"testing"
|
||||
|
||||
ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
|
||||
vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
|
||||
@ -87,7 +88,7 @@ func makeOCIBundle(bundleDir string) error {
|
||||
}
|
||||
}
|
||||
|
||||
output, err := RunCommandFull([]string{"cp", "-a", from, to}, true)
|
||||
output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
|
||||
}
|
||||
|
@ -8,13 +8,12 @@ package katautils
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"golang.org/x/sys/unix"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// FileExists test is a file exiting or not
|
||||
@ -110,27 +109,3 @@ func GetFileContents(file string) (string, error) {
|
||||
|
||||
return string(bytes), nil
|
||||
}
|
||||
|
||||
// RunCommandFull returns the commands space-trimmed standard output and
|
||||
// error on success. Note that if the command fails, the requested output will
|
||||
// still be returned, along with an error.
|
||||
func RunCommandFull(args []string, includeStderr bool) (string, error) {
|
||||
cmd := exec.Command(args[0], args[1:]...)
|
||||
var err error
|
||||
var bytes []byte
|
||||
|
||||
if includeStderr {
|
||||
bytes, err = cmd.CombinedOutput()
|
||||
} else {
|
||||
bytes, err = cmd.Output()
|
||||
}
|
||||
|
||||
trimmed := strings.TrimSpace(string(bytes))
|
||||
|
||||
return trimmed, err
|
||||
}
|
||||
|
||||
// RunCommand returns the commands space-trimmed standard output on success
|
||||
func RunCommand(args []string) (string, error) {
|
||||
return RunCommandFull(args, false)
|
||||
}
|
||||
|
@ -18,6 +18,7 @@ import (
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
@ -90,7 +91,7 @@ func createOCIConfig(bundleDir string) error {
|
||||
return errors.New("Cannot find command to generate OCI config file")
|
||||
}
|
||||
|
||||
_, err := RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
|
||||
_, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -1,4 +1,4 @@
|
||||
// Copyright (c) 2020 Ant Financial
|
||||
// Copyright (c) 2020 Ant Group
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
@ -6,7 +6,11 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@ -31,3 +35,48 @@ func GzipAccepted(header http.Header) bool {
|
||||
func String2Pointer(s string) *string {
|
||||
return &s
|
||||
}
|
||||
|
||||
// RunCommandFull returns the commands space-trimmed standard output and
|
||||
// error on success. Note that if the command fails, the requested output will
|
||||
// still be returned, along with an error.
|
||||
func RunCommandFull(args []string, includeStderr bool) (string, error) {
|
||||
cmd := exec.Command(args[0], args[1:]...)
|
||||
var err error
|
||||
var bytes []byte
|
||||
|
||||
if includeStderr {
|
||||
bytes, err = cmd.CombinedOutput()
|
||||
} else {
|
||||
bytes, err = cmd.Output()
|
||||
}
|
||||
|
||||
trimmed := strings.TrimSpace(string(bytes))
|
||||
|
||||
return trimmed, err
|
||||
}
|
||||
|
||||
// RunCommand returns the commands space-trimmed standard output on success
|
||||
func RunCommand(args []string) (string, error) {
|
||||
return RunCommandFull(args, false)
|
||||
}
|
||||
|
||||
// EnsureDir check if a directory exist, if not then create it
|
||||
func EnsureDir(path string, mode os.FileMode) error {
|
||||
if !filepath.IsAbs(path) {
|
||||
return fmt.Errorf("Not an absolute path: %s", path)
|
||||
}
|
||||
|
||||
if fi, err := os.Stat(path); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
if err = os.MkdirAll(path, mode); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
} else if !fi.IsDir() {
|
||||
return fmt.Errorf("Not a directory: %s", path)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
@ -6,7 +6,10 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@ -45,3 +48,71 @@ func TestGzipAccepted(t *testing.T) {
|
||||
assert.Equal(tc.result, b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDir(t *testing.T) {
|
||||
const testMode = 0755
|
||||
tmpdir, err := ioutil.TempDir("", "TestEnsureDir")
|
||||
assert := assert.New(t)
|
||||
|
||||
assert.NoError(err)
|
||||
defer os.RemoveAll(tmpdir)
|
||||
|
||||
testCases := []struct {
|
||||
before func()
|
||||
path string
|
||||
err bool
|
||||
msg string
|
||||
}{
|
||||
{
|
||||
before: nil,
|
||||
path: "a/b/c",
|
||||
err: true,
|
||||
msg: "Not an absolute path",
|
||||
},
|
||||
{
|
||||
before: nil,
|
||||
path: fmt.Sprintf("%s/abc/def", tmpdir),
|
||||
err: false,
|
||||
msg: "",
|
||||
},
|
||||
{
|
||||
before: nil,
|
||||
path: fmt.Sprintf("%s/abc", tmpdir),
|
||||
err: false,
|
||||
msg: "",
|
||||
},
|
||||
{
|
||||
before: func() {
|
||||
err := os.MkdirAll(fmt.Sprintf("%s/abc/def", tmpdir), testMode)
|
||||
assert.NoError(err)
|
||||
},
|
||||
path: fmt.Sprintf("%s/abc/def", tmpdir),
|
||||
err: false,
|
||||
msg: "",
|
||||
},
|
||||
{
|
||||
before: func() {
|
||||
// create a regular file
|
||||
err := os.MkdirAll(fmt.Sprintf("%s/abc", tmpdir), testMode)
|
||||
assert.NoError(err)
|
||||
_, err = os.Create(fmt.Sprintf("%s/abc/file.txt", tmpdir))
|
||||
assert.NoError(err)
|
||||
},
|
||||
path: fmt.Sprintf("%s/abc/file.txt", tmpdir),
|
||||
err: true,
|
||||
msg: "Not a directory",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
if tc.before != nil {
|
||||
tc.before()
|
||||
}
|
||||
err := EnsureDir(tc.path, testMode)
|
||||
if tc.err {
|
||||
assert.Contains(err.Error(), tc.msg, "error msg should contains: %s, but got %s", tc.msg, err.Error())
|
||||
} else {
|
||||
assert.Equal(err, nil, "failed for path: %s, except no error, but got %+v", tc.path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
2
src/runtime/vendor/github.com/sirupsen/logrus/go.mod
generated
vendored
2
src/runtime/vendor/github.com/sirupsen/logrus/go.mod
generated
vendored
@ -1,5 +1,7 @@
|
||||
module github.com/sirupsen/logrus
|
||||
|
||||
go 1.15
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1
|
||||
|
@ -440,6 +440,13 @@ type HypervisorConfig struct {
|
||||
|
||||
// Enable annotations by name
|
||||
EnableAnnotations []string
|
||||
|
||||
// GuestCoredumpPath is the path in host for saving guest memory dump
|
||||
GuestMemoryDumpPath string
|
||||
|
||||
// GuestMemoryDumpPaging is used to indicate if enable paging
|
||||
// for QEMU dump-guest-memory command
|
||||
GuestMemoryDumpPaging bool
|
||||
}
|
||||
|
||||
// vcpu mapping from vcpu number to thread number
|
||||
@ -607,6 +614,10 @@ func (conf *HypervisorConfig) HypervisorAssetPath() (string, error) {
|
||||
return conf.assetPath(types.HypervisorAsset)
|
||||
}
|
||||
|
||||
func (conf *HypervisorConfig) IfPVPanicEnabled() bool {
|
||||
return conf.GuestMemoryDumpPath != ""
|
||||
}
|
||||
|
||||
// HypervisorCtlAssetPath returns the VM hypervisor ctl path
|
||||
func (conf *HypervisorConfig) HypervisorCtlAssetPath() (string, error) {
|
||||
return conf.assetPath(types.HypervisorCtlAsset)
|
||||
|
@ -31,6 +31,7 @@ import (
|
||||
"github.com/sirupsen/logrus"
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
pkgUtils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
|
||||
persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid"
|
||||
@ -99,6 +100,9 @@ type qemu struct {
|
||||
stopped bool
|
||||
|
||||
store persistapi.PersistDriver
|
||||
|
||||
// if in memory dump progress
|
||||
memoryDumpFlag sync.Mutex
|
||||
}
|
||||
|
||||
const (
|
||||
@ -106,6 +110,9 @@ const (
|
||||
qmpSocket = "qmp.sock"
|
||||
vhostFSSocket = "vhost-fs.sock"
|
||||
|
||||
// memory dump format will be set to elf
|
||||
memoryDumpFormat = "elf"
|
||||
|
||||
qmpCapErrMsg = "Failed to negoatiate QMP capabilities"
|
||||
qmpExecCatCmd = "exec:cat"
|
||||
|
||||
@ -408,13 +415,17 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
|
||||
}
|
||||
}
|
||||
|
||||
if q.config.IfPVPanicEnabled() {
|
||||
// there should have no errors for pvpanic device
|
||||
devices, _ = q.arch.appendPVPanicDevice(devices)
|
||||
}
|
||||
|
||||
var ioThread *govmmQemu.IOThread
|
||||
if q.config.BlockDeviceDriver == config.VirtioSCSI {
|
||||
return q.arch.appendSCSIController(devices, q.config.EnableIOThreads)
|
||||
}
|
||||
|
||||
return devices, ioThread, nil
|
||||
|
||||
}
|
||||
|
||||
func (q *qemu) setupTemplate(knobs *govmmQemu.Knobs, memory *govmmQemu.Memory) govmmQemu.Incoming {
|
||||
@ -1027,7 +1038,13 @@ func (q *qemu) qmpSetup() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg := govmmQemu.QMPConfig{Logger: newQMPLogger()}
|
||||
events := make(chan govmmQemu.QMPEvent)
|
||||
go q.loopQMPEvent(events)
|
||||
|
||||
cfg := govmmQemu.QMPConfig{
|
||||
Logger: newQMPLogger(),
|
||||
EventCh: events,
|
||||
}
|
||||
|
||||
// Auto-closed by QMPStart().
|
||||
disconnectCh := make(chan struct{})
|
||||
@ -1050,6 +1067,136 @@ func (q *qemu) qmpSetup() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *qemu) loopQMPEvent(event chan govmmQemu.QMPEvent) {
|
||||
for {
|
||||
select {
|
||||
case e, open := <-event:
|
||||
if !open {
|
||||
q.Logger().Infof("QMP event channel closed")
|
||||
return
|
||||
}
|
||||
q.Logger().WithField("event", e).Debug("got QMP event")
|
||||
if e.Name == "GUEST_PANICKED" {
|
||||
go q.handleGuestPanic()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (q *qemu) handleGuestPanic() {
|
||||
if err := q.dumpGuestMemory(q.config.GuestMemoryDumpPath); err != nil {
|
||||
q.Logger().WithError(err).Error("failed to dump guest memory")
|
||||
}
|
||||
|
||||
// TODO: how to notify the upper level sandbox to handle the error
|
||||
// to do a fast fail(shutdown or others).
|
||||
// tracked by https://github.com/kata-containers/kata-containers/issues/1026
|
||||
}
|
||||
|
||||
// canDumpGuestMemory check if can do a guest memory dump operation.
|
||||
// for now it only ensure there must be double of VM size for free disk spaces
|
||||
func (q *qemu) canDumpGuestMemory(dumpSavePath string) error {
|
||||
fs := unix.Statfs_t{}
|
||||
if err := unix.Statfs(dumpSavePath, &fs); err != nil {
|
||||
q.Logger().WithError(err).WithField("dumpSavePath", dumpSavePath).Error("failed to call Statfs")
|
||||
return nil
|
||||
}
|
||||
availSpaceInBytes := fs.Bavail * uint64(fs.Bsize)
|
||||
q.Logger().WithFields(
|
||||
logrus.Fields{
|
||||
"dumpSavePath": dumpSavePath,
|
||||
"availSpaceInBytes": availSpaceInBytes,
|
||||
}).Info("get avail space")
|
||||
|
||||
// get guest memory size
|
||||
guestMemorySizeInBytes := (uint64(q.config.MemorySize) + uint64(q.state.HotpluggedMemory)) << utils.MibToBytesShift
|
||||
q.Logger().WithField("guestMemorySizeInBytes", guestMemorySizeInBytes).Info("get guest memory size")
|
||||
|
||||
// default we want ensure there are at least double of VM memory size free spaces available,
|
||||
// this may complete one dump operation for one sandbox
|
||||
exceptMemorySize := guestMemorySizeInBytes * 2
|
||||
if availSpaceInBytes >= exceptMemorySize {
|
||||
return nil
|
||||
} else {
|
||||
return fmt.Errorf("there are not enough free space to store memory dump file. Except %d bytes, but only %d bytes available", exceptMemorySize, availSpaceInBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// dumpSandboxMetaInfo save meta information for debug purpose, includes:
|
||||
// hypervisor verison, sandbox/container state, hypervisor config
|
||||
func (q *qemu) dumpSandboxMetaInfo(dumpSavePath string) {
|
||||
dumpStatePath := filepath.Join(dumpSavePath, "state")
|
||||
|
||||
// copy state from /run/vc/sbs to memory dump directory
|
||||
statePath := filepath.Join(q.store.RunStoragePath(), q.id)
|
||||
command := []string{"/bin/cp", "-ar", statePath, dumpStatePath}
|
||||
q.Logger().WithField("command", command).Info("try to save sandbox state")
|
||||
if output, err := pkgUtils.RunCommandFull(command, true); err != nil {
|
||||
q.Logger().WithError(err).WithField("output", output).Error("failed to save state")
|
||||
}
|
||||
// save hypervisor meta information
|
||||
fileName := filepath.Join(dumpSavePath, "hypervisor.conf")
|
||||
data, _ := json.MarshalIndent(q.config, "", " ")
|
||||
if err := ioutil.WriteFile(fileName, data, defaultFilePerms); err != nil {
|
||||
q.Logger().WithError(err).WithField("hypervisor.conf", data).Error("write to hypervisor.conf file failed")
|
||||
}
|
||||
|
||||
// save hypervisor version
|
||||
hyperVisorVersion, err := pkgUtils.RunCommand([]string{q.config.HypervisorPath, "--version"})
|
||||
if err != nil {
|
||||
q.Logger().WithError(err).WithField("HypervisorPath", data).Error("failed to get hypervisor version")
|
||||
}
|
||||
|
||||
fileName = filepath.Join(dumpSavePath, "hypervisor.version")
|
||||
if err := ioutil.WriteFile(fileName, []byte(hyperVisorVersion), defaultFilePerms); err != nil {
|
||||
q.Logger().WithError(err).WithField("hypervisor.version", data).Error("write to hypervisor.version file failed")
|
||||
}
|
||||
}
|
||||
|
||||
func (q *qemu) dumpGuestMemory(dumpSavePath string) error {
|
||||
if dumpSavePath == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
q.memoryDumpFlag.Lock()
|
||||
defer q.memoryDumpFlag.Unlock()
|
||||
|
||||
q.Logger().WithField("dumpSavePath", dumpSavePath).Info("try to dump guest memory")
|
||||
|
||||
dumpSavePath = filepath.Join(dumpSavePath, q.id)
|
||||
dumpStatePath := filepath.Join(dumpSavePath, "state")
|
||||
if err := pkgUtils.EnsureDir(dumpStatePath, DirMode); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// save meta information for sandbox
|
||||
q.dumpSandboxMetaInfo(dumpSavePath)
|
||||
q.Logger().Info("dump sandbox meta information completed")
|
||||
|
||||
// check device free space and estimated dump size
|
||||
if err := q.canDumpGuestMemory(dumpSavePath); err != nil {
|
||||
q.Logger().Warnf("can't dump guest memory: %s", err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// dump guest memory
|
||||
protocol := fmt.Sprintf("file:%s/vmcore-%s.%s", dumpSavePath, time.Now().Format("20060102150405.999"), memoryDumpFormat)
|
||||
q.Logger().Infof("try to dump guest memory to %s", protocol)
|
||||
|
||||
if err := q.qmpSetup(); err != nil {
|
||||
q.Logger().WithError(err).Error("setup manage QMP failed")
|
||||
return err
|
||||
}
|
||||
|
||||
if err := q.qmpMonitorCh.qmp.ExecuteDumpGuestMemory(q.qmpMonitorCh.ctx, protocol, q.config.GuestMemoryDumpPaging, memoryDumpFormat); err != nil {
|
||||
q.Logger().WithError(err).Error("dump guest memory failed")
|
||||
return err
|
||||
}
|
||||
|
||||
q.Logger().Info("dump guest memory completed")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *qemu) qmpShutdown() {
|
||||
q.qmpMonitorCh.Lock()
|
||||
defer q.qmpMonitorCh.Unlock()
|
||||
@ -2250,6 +2397,9 @@ func (q *qemu) load(s persistapi.HypervisorState) {
|
||||
}
|
||||
|
||||
func (q *qemu) check() error {
|
||||
q.memoryDumpFlag.Lock()
|
||||
defer q.memoryDumpFlag.Unlock()
|
||||
|
||||
err := q.qmpSetup()
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -133,6 +133,9 @@ type qemuArch interface {
|
||||
|
||||
// append vIOMMU device
|
||||
appendIOMMU(devices []govmmQemu.Device) ([]govmmQemu.Device, error)
|
||||
|
||||
// append pvpanic device
|
||||
appendPVPanicDevice(devices []govmmQemu.Device) ([]govmmQemu.Device, error)
|
||||
}
|
||||
|
||||
type qemuArchBase struct {
|
||||
@ -789,3 +792,9 @@ func (q *qemuArchBase) appendIOMMU(devices []govmmQemu.Device) ([]govmmQemu.Devi
|
||||
return devices, fmt.Errorf("Machine Type %s does not support vIOMMU", q.qemuMachine.Type)
|
||||
}
|
||||
}
|
||||
|
||||
// appendPVPanicDevice appends a pvpanic device
|
||||
func (q *qemuArchBase) appendPVPanicDevice(devices []govmmQemu.Device) ([]govmmQemu.Device, error) {
|
||||
devices = append(devices, govmmQemu.PVPanicDevice{NoShutdown: true})
|
||||
return devices, nil
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user