From daae1db8938d355759d49453c5643532f20692ac Mon Sep 17 00:00:00 2001 From: Penny Zheng Date: Tue, 17 Sep 2019 16:49:05 +0800 Subject: [PATCH 01/14] log: extract error info from firecracker built-in log and metrics scheme Firecracker have its own logging scheme, providing two fifo files with log and metrics info. We should extract error info for better debugging. Fixes: #2072 Signed-off-by: Penny Zheng Signed-off-by: Haibo Xu --- virtcontainers/fc.go | 78 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go index c92e63e6cc..435db2a488 100644 --- a/virtcontainers/fc.go +++ b/virtcontainers/fc.go @@ -6,6 +6,7 @@ package virtcontainers import ( + "bufio" "context" "fmt" "net" @@ -19,6 +20,7 @@ import ( "syscall" "time" + "github.com/containerd/fifo" httptransport "github.com/go-openapi/runtime/client" "github.com/go-openapi/strfmt" kataclient "github.com/kata-containers/agent/protocols/client" @@ -59,13 +61,16 @@ const ( // firecracker guest VM. // We attach a pool of placeholder drives before the guest has started, and then // patch the replace placeholder drives with drives with actual contents. - fcDiskPoolSize = 8 - + fcDiskPoolSize = 8 defaultHybridVSocketName = "kata.hvsock" // This is the first usable vsock context ID. All the vsocks can use the same // ID, since it's only used in the guest. defaultGuestVSockCID = int64(0x3) + + // This is related to firecracker logging scheme + fcLogFifo = "logs.fifo" + fcMetricsFifo = "metrics.fifo" ) // Specify the minimum version of firecracker supported @@ -602,6 +607,69 @@ func (fc *firecracker) fcSetVMBaseConfig(mem int64, vcpus int64, htEnabled bool) return err } +func (fc *firecracker) fcSetLogger() error { + span, _ := fc.trace("fcSetLogger") + defer span.Finish() + + fcLogLevel := "Error" + + // listen to log fifo file and transfer error info + jailedLogFifo, err := fc.fcListenToFifo(fcLogFifo) + if err != nil { + return fmt.Errorf("Failed setting log: %s", err) + } + + // listen to metrics file and transfer error info + jailedMetricsFifo, err := fc.fcListenToFifo(fcMetricsFifo) + if err != nil { + return fmt.Errorf("Failed setting log: %s", err) + } + + param := ops.NewPutLoggerParams() + cfg := &models.Logger{ + Level: &fcLogLevel, + LogFifo: &jailedLogFifo, + MetricsFifo: &jailedMetricsFifo, + Options: []string{}, + } + param.SetBody(cfg) + _, err = fc.client().Operations.PutLogger(param) + + return err +} + +func (fc *firecracker) fcListenToFifo(fifoName string) (string, error) { + fcFifoPath := filepath.Join(fc.vmPath, fifoName) + fcFifo, err := fifo.OpenFifo(context.Background(), fcFifoPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0) + if err != nil { + return "", fmt.Errorf("Failed to open/create fifo file %s", err) + } + + jailedFifoPath, err := fc.fcJailResource(fcFifoPath, fifoName) + if err != nil { + return "", err + } + + go func() { + scanner := bufio.NewScanner(fcFifo) + for scanner.Scan() { + fc.Logger().WithFields(logrus.Fields{ + "fifoName": fifoName, + "contents": scanner.Text()}).Error("firecracker failed") + } + + if err := scanner.Err(); err != nil { + fc.Logger().WithError(err).Errorf("Failed reading firecracker fifo file") + } + + if err := fcFifo.Close(); err != nil { + fc.Logger().WithError(err).Errorf("Failed closing firecracker fifo file") + } + }() + + return jailedFifoPath, nil +} + func (fc *firecracker) fcStartVM() error { fc.Logger().Info("start firecracker virtual machine") span, _ := fc.trace("fcStartVM") @@ -688,6 +756,10 @@ func (fc *firecracker) startSandbox(timeout int) error { } } + if err := fc.fcSetLogger(); err != nil { + return err + } + if err := fc.fcStartVM(); err != nil { return err } @@ -748,6 +820,8 @@ func (fc *firecracker) cleanupJail() { fc.umountResource(fcKernel) fc.umountResource(fcRootfs) + fc.umountResource(fcLogFifo) + fc.umountResource(fcMetricsFifo) fc.Logger().WithField("cleaningJail", fc.vmPath).Info() if err := os.RemoveAll(fc.vmPath); err != nil { From 7943dd95b499ad3a842aed3b990617a46da8f52a Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 4 Nov 2019 19:45:19 +0800 Subject: [PATCH 02/14] persistence: store configuration in newstore Fixes #803 Store the configuration data in persist.json. Signed-off-by: Wei Zhang --- cli/main.go | 24 ++ virtcontainers/api.go | 10 +- virtcontainers/container.go | 1 + virtcontainers/experimental/experimental.go | 17 ++ virtcontainers/persist.go | 231 ++++++++++++++++++++ virtcontainers/persist/api/config.go | 78 ++++--- virtcontainers/persist/fs/fs.go | 19 +- virtcontainers/sandbox.go | 71 ++++-- 8 files changed, 396 insertions(+), 55 deletions(-) diff --git a/cli/main.go b/cli/main.go index 8a624a4c8c..9e63da089d 100644 --- a/cli/main.go +++ b/cli/main.go @@ -21,6 +21,7 @@ import ( "github.com/kata-containers/runtime/pkg/rootless" "github.com/kata-containers/runtime/pkg/signals" vc "github.com/kata-containers/runtime/virtcontainers" + exp "github.com/kata-containers/runtime/virtcontainers/experimental" vf "github.com/kata-containers/runtime/virtcontainers/factory" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -346,6 +347,11 @@ func beforeSubcommands(c *cli.Context) error { "arguments": `"` + args + `"`, } + err = addExpFeatures(c, runtimeConfig) + if err != nil { + return err + } + kataLog.WithFields(fields).Info() // make the data accessible to the sub-commands. @@ -401,6 +407,24 @@ func setupTracing(context *cli.Context, rootSpanName string) error { return nil } +// add supported experimental features in context +func addExpFeatures(clictx *cli.Context, runtimeConfig oci.RuntimeConfig) error { + ctx, err := cliContextToContext(clictx) + if err != nil { + return err + } + + var exps []string + for _, e := range runtimeConfig.Experimental { + exps = append(exps, e.Name) + } + + ctx = exp.ContextWithExp(ctx, exps) + // Add tracer to metadata and update the context + clictx.App.Metadata["context"] = ctx + return nil +} + func afterSubcommands(c *cli.Context) error { ctx, err := cliContextToContext(c) if err != nil { diff --git a/virtcontainers/api.go b/virtcontainers/api.go index bbd069afff..6eb112ba87 100644 --- a/virtcontainers/api.go +++ b/virtcontainers/api.go @@ -13,6 +13,7 @@ import ( deviceApi "github.com/kata-containers/runtime/virtcontainers/device/api" deviceConfig "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/persist/fs" vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" @@ -307,7 +308,14 @@ func ListSandbox(ctx context.Context) ([]SandboxStatus, error) { span, ctx := trace(ctx, "ListSandbox") defer span.Finish() - dir, err := os.Open(store.ConfigStoragePath()) + var sbsdir string + if supportNewStore(ctx) { + sbsdir = fs.RunStoragePath() + } else { + sbsdir = store.RunStoragePath() + } + + dir, err := os.Open(sbsdir) if err != nil { if os.IsNotExist(err) { // No sandbox directory is not an error diff --git a/virtcontainers/container.go b/virtcontainers/container.go index 701d5c6788..6f6e160fa4 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -417,6 +417,7 @@ func (c *Container) storeContainer() error { if err := c.sandbox.Save(); err != nil { return err } + return nil } return c.store.Store(store.Configuration, *(c.config)) } diff --git a/virtcontainers/experimental/experimental.go b/virtcontainers/experimental/experimental.go index 4204e963e5..8eb3eddd3c 100644 --- a/virtcontainers/experimental/experimental.go +++ b/virtcontainers/experimental/experimental.go @@ -6,6 +6,7 @@ package experimental import ( + "context" "fmt" "regexp" ) @@ -22,8 +23,11 @@ type Feature struct { ExpRelease string } +type contextKey struct{} + var ( supportedFeatures = make(map[string]Feature) + expContextKey = contextKey{} ) // Register register a new experimental feature @@ -61,3 +65,16 @@ func validateFeature(feature Feature) error { return nil } + +func ContextWithExp(ctx context.Context, names []string) context.Context { + return context.WithValue(ctx, expContextKey, names) +} + +func ExpFromContext(ctx context.Context) []string { + value := ctx.Value(expContextKey) + if value == nil { + return nil + } + names := value.([]string) + return names +} diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go index c8ec42a484..6755aaee85 100644 --- a/virtcontainers/persist.go +++ b/virtcontainers/persist.go @@ -13,6 +13,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/persist" persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/mitchellh/mapstructure" ) var ( @@ -171,6 +172,119 @@ func (s *Sandbox) dumpNetwork(ss *persistapi.SandboxState) { } } +func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { + sconfig := s.config + ss.Config = persistapi.SandboxConfig{ + HypervisorType: string(sconfig.HypervisorType), + AgentType: string(sconfig.AgentType), + ProxyType: string(sconfig.ProxyType), + ProxyConfig: persistapi.ProxyConfig{ + Path: sconfig.ProxyConfig.Path, + Debug: sconfig.ProxyConfig.Debug, + }, + ShimType: string(sconfig.ShimType), + NetworkConfig: persistapi.NetworkConfig{ + NetNSPath: sconfig.NetworkConfig.NetNSPath, + NetNsCreated: sconfig.NetworkConfig.NetNsCreated, + DisableNewNetNs: sconfig.NetworkConfig.DisableNewNetNs, + InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel), + }, + + ShmSize: sconfig.ShmSize, + SharePidNs: sconfig.SharePidNs, + Stateful: sconfig.Stateful, + SystemdCgroup: sconfig.SystemdCgroup, + SandboxCgroupOnly: sconfig.SandboxCgroupOnly, + DisableGuestSeccomp: sconfig.DisableGuestSeccomp, + } + + for _, e := range sconfig.Experimental { + ss.Config.Experimental = append(ss.Config.Experimental, e.Name) + } + + ss.Config.HypervisorConfig = persistapi.HypervisorConfig{ + NumVCPUs: sconfig.HypervisorConfig.NumVCPUs, + DefaultMaxVCPUs: sconfig.HypervisorConfig.DefaultMaxVCPUs, + MemorySize: sconfig.HypervisorConfig.MemorySize, + DefaultBridges: sconfig.HypervisorConfig.DefaultBridges, + Msize9p: sconfig.HypervisorConfig.Msize9p, + MemSlots: sconfig.HypervisorConfig.MemSlots, + MemOffset: sconfig.HypervisorConfig.MemOffset, + VirtioFSCacheSize: sconfig.HypervisorConfig.VirtioFSCacheSize, + KernelPath: sconfig.HypervisorConfig.KernelPath, + ImagePath: sconfig.HypervisorConfig.ImagePath, + InitrdPath: sconfig.HypervisorConfig.InitrdPath, + FirmwarePath: sconfig.HypervisorConfig.FirmwarePath, + MachineAccelerators: sconfig.HypervisorConfig.MachineAccelerators, + HypervisorPath: sconfig.HypervisorConfig.HypervisorPath, + HypervisorCtlPath: sconfig.HypervisorConfig.HypervisorCtlPath, + JailerPath: sconfig.HypervisorConfig.JailerPath, + BlockDeviceDriver: sconfig.HypervisorConfig.BlockDeviceDriver, + HypervisorMachineType: sconfig.HypervisorConfig.HypervisorMachineType, + MemoryPath: sconfig.HypervisorConfig.MemoryPath, + DevicesStatePath: sconfig.HypervisorConfig.DevicesStatePath, + EntropySource: sconfig.HypervisorConfig.EntropySource, + SharedFS: sconfig.HypervisorConfig.SharedFS, + VirtioFSDaemon: sconfig.HypervisorConfig.VirtioFSDaemon, + VirtioFSCache: sconfig.HypervisorConfig.VirtioFSCache, + VirtioFSExtraArgs: sconfig.HypervisorConfig.VirtioFSExtraArgs[:], + BlockDeviceCacheSet: sconfig.HypervisorConfig.BlockDeviceCacheSet, + BlockDeviceCacheDirect: sconfig.HypervisorConfig.BlockDeviceCacheDirect, + BlockDeviceCacheNoflush: sconfig.HypervisorConfig.BlockDeviceCacheNoflush, + DisableBlockDeviceUse: sconfig.HypervisorConfig.DisableBlockDeviceUse, + EnableIOThreads: sconfig.HypervisorConfig.EnableIOThreads, + Debug: sconfig.HypervisorConfig.Debug, + MemPrealloc: sconfig.HypervisorConfig.MemPrealloc, + HugePages: sconfig.HypervisorConfig.HugePages, + FileBackedMemRootDir: sconfig.HypervisorConfig.FileBackedMemRootDir, + Realtime: sconfig.HypervisorConfig.Realtime, + Mlock: sconfig.HypervisorConfig.Mlock, + DisableNestingChecks: sconfig.HypervisorConfig.DisableNestingChecks, + UseVSock: sconfig.HypervisorConfig.UseVSock, + HotplugVFIOOnRootBus: sconfig.HypervisorConfig.HotplugVFIOOnRootBus, + BootToBeTemplate: sconfig.HypervisorConfig.BootToBeTemplate, + BootFromTemplate: sconfig.HypervisorConfig.BootFromTemplate, + DisableVhostNet: sconfig.HypervisorConfig.DisableVhostNet, + GuestHookPath: sconfig.HypervisorConfig.GuestHookPath, + VMid: sconfig.HypervisorConfig.VMid, + } + + if sconfig.AgentType == "kata" { + var sagent KataAgentConfig + err := mapstructure.Decode(sconfig.AgentConfig, &sagent) + if err != nil { + s.Logger().WithError(err).Error("internal error: KataAgentConfig failed to decode") + } else { + ss.Config.KataAgentConfig = &persistapi.KataAgentConfig{ + LongLiveConn: sagent.LongLiveConn, + UseVSock: sagent.UseVSock, + } + } + } + + if sconfig.ShimType == "kataShim" { + var shim ShimConfig + err := mapstructure.Decode(sconfig.ShimConfig, &shim) + if err != nil { + s.Logger().WithError(err).Error("internal error: ShimConfig failed to decode") + } else { + ss.Config.KataShimConfig = &persistapi.ShimConfig{ + Path: shim.Path, + Debug: shim.Debug, + } + } + } + + for _, contConf := range sconfig.Containers { + ss.Config.ContainerConfigs = append(ss.Config.ContainerConfigs, persistapi.ContainerConfig{ + ID: contConf.ID, + Annotations: contConf.Annotations, + RootFs: contConf.RootFs.Target, + Resources: contConf.Resources, + }) + } +} + func (s *Sandbox) Save() error { var ( ss = persistapi.SandboxState{} @@ -185,6 +299,7 @@ func (s *Sandbox) Save() error { s.dumpMounts(cs) s.dumpAgent(&ss) s.dumpNetwork(&ss) + s.dumpConfig(&ss) if err := s.newStore.ToDisk(ss, cs); err != nil { return err @@ -335,3 +450,119 @@ func (s *Sandbox) supportNewStore() bool { } return false } + +func loadSandboxConfig(id string) (*SandboxConfig, error) { + store, err := persist.GetDriver("fs") + if err != nil || store == nil { + return nil, errors.New("failed to get fs persist driver") + } + + ss, _, err := store.FromDisk(id) + if err != nil { + return nil, err + } + + savedConf := ss.Config + sconfig := &SandboxConfig{ + ID: id, + HypervisorType: HypervisorType(savedConf.HypervisorType), + AgentType: AgentType(savedConf.AgentType), + ProxyType: ProxyType(savedConf.ProxyType), + ProxyConfig: ProxyConfig{ + Path: savedConf.ProxyConfig.Path, + Debug: savedConf.ProxyConfig.Debug, + }, + ShimType: ShimType(savedConf.ShimType), + NetworkConfig: NetworkConfig{ + NetNSPath: savedConf.NetworkConfig.NetNSPath, + NetNsCreated: savedConf.NetworkConfig.NetNsCreated, + DisableNewNetNs: savedConf.NetworkConfig.DisableNewNetNs, + InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel), + }, + + ShmSize: savedConf.ShmSize, + SharePidNs: savedConf.SharePidNs, + Stateful: savedConf.Stateful, + SystemdCgroup: savedConf.SystemdCgroup, + SandboxCgroupOnly: savedConf.SandboxCgroupOnly, + DisableGuestSeccomp: savedConf.DisableGuestSeccomp, + } + + for _, name := range savedConf.Experimental { + sconfig.Experimental = append(sconfig.Experimental, *exp.Get(name)) + } + + hconf := savedConf.HypervisorConfig + sconfig.HypervisorConfig = HypervisorConfig{ + NumVCPUs: hconf.NumVCPUs, + DefaultMaxVCPUs: hconf.DefaultMaxVCPUs, + MemorySize: hconf.MemorySize, + DefaultBridges: hconf.DefaultBridges, + Msize9p: hconf.Msize9p, + MemSlots: hconf.MemSlots, + MemOffset: hconf.MemOffset, + VirtioFSCacheSize: hconf.VirtioFSCacheSize, + KernelPath: hconf.KernelPath, + ImagePath: hconf.ImagePath, + InitrdPath: hconf.InitrdPath, + FirmwarePath: hconf.FirmwarePath, + MachineAccelerators: hconf.MachineAccelerators, + HypervisorPath: hconf.HypervisorPath, + HypervisorCtlPath: hconf.HypervisorCtlPath, + JailerPath: hconf.JailerPath, + BlockDeviceDriver: hconf.BlockDeviceDriver, + HypervisorMachineType: hconf.HypervisorMachineType, + MemoryPath: hconf.MemoryPath, + DevicesStatePath: hconf.DevicesStatePath, + EntropySource: hconf.EntropySource, + SharedFS: hconf.SharedFS, + VirtioFSDaemon: hconf.VirtioFSDaemon, + VirtioFSCache: hconf.VirtioFSCache, + VirtioFSExtraArgs: hconf.VirtioFSExtraArgs[:], + BlockDeviceCacheSet: hconf.BlockDeviceCacheSet, + BlockDeviceCacheDirect: hconf.BlockDeviceCacheDirect, + BlockDeviceCacheNoflush: hconf.BlockDeviceCacheNoflush, + DisableBlockDeviceUse: hconf.DisableBlockDeviceUse, + EnableIOThreads: hconf.EnableIOThreads, + Debug: hconf.Debug, + MemPrealloc: hconf.MemPrealloc, + HugePages: hconf.HugePages, + FileBackedMemRootDir: hconf.FileBackedMemRootDir, + Realtime: hconf.Realtime, + Mlock: hconf.Mlock, + DisableNestingChecks: hconf.DisableNestingChecks, + UseVSock: hconf.UseVSock, + HotplugVFIOOnRootBus: hconf.HotplugVFIOOnRootBus, + BootToBeTemplate: hconf.BootToBeTemplate, + BootFromTemplate: hconf.BootFromTemplate, + DisableVhostNet: hconf.DisableVhostNet, + GuestHookPath: hconf.GuestHookPath, + VMid: hconf.VMid, + } + + if savedConf.AgentType == "kata" { + sconfig.AgentConfig = KataAgentConfig{ + LongLiveConn: savedConf.KataAgentConfig.LongLiveConn, + UseVSock: savedConf.KataAgentConfig.UseVSock, + } + } + + if savedConf.ShimType == "kataShim" { + sconfig.ShimConfig = ShimConfig{ + Path: savedConf.KataShimConfig.Path, + Debug: savedConf.KataShimConfig.Debug, + } + } + + for _, contConf := range savedConf.ContainerConfigs { + sconfig.Containers = append(sconfig.Containers, ContainerConfig{ + ID: contConf.ID, + Annotations: contConf.Annotations, + Resources: contConf.Resources, + RootFs: RootFs{ + Target: contConf.RootFs, + }, + }) + } + return sconfig, nil +} diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go index 1d10ed483c..d718241fa3 100644 --- a/virtcontainers/persist/api/config.go +++ b/virtcontainers/persist/api/config.go @@ -6,17 +6,9 @@ package persistapi -// Param is a key/value representation for hypervisor and kernel parameters. -type Param struct { - Key string - Value string -} - -// Asset saves hypervisor asset -type Asset struct { - Path string `json:"path"` - Custom bool `json:"bool"` -} +import ( + specs "github.com/opencontainers/runtime-spec/specs-go" +) // HypervisorConfig saves configurations of sandbox hypervisor type HypervisorConfig struct { @@ -42,11 +34,8 @@ type HypervisorConfig struct { // MemOffset specifies memory space for nvdimm device MemOffset uint32 - // KernelParams are additional guest kernel parameters. - KernelParams []Param - - // HypervisorParams are additional hypervisor parameters. - HypervisorParams []Param + // VirtioFSCacheSize is the DAX cache size in MiB + VirtioFSCacheSize uint32 // KernelPath is the guest kernel host path. KernelPath string @@ -67,6 +56,12 @@ type HypervisorConfig struct { // HypervisorPath is the hypervisor executable host path. HypervisorPath string + // HypervisorCtlPath is the hypervisor ctl executable host path. + HypervisorCtlPath string + + // JailerPath is the jailer executable host path. + JailerPath string + // BlockDeviceDriver specifies the driver to be used for block device // either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver BlockDeviceDriver string @@ -87,11 +82,19 @@ type HypervisorConfig struct { // entropy (/dev/random, /dev/urandom or real hardware RNG device) EntropySource string - // customAssets is a map of assets. - // Each value in that map takes precedence over the configured assets. - // For example, if there is a value for the "kernel" key in this map, - // it will be used for the sandbox's kernel path instead of KernelPath. - CustomAssets map[string]*Asset + // Shared file system type: + // - virtio-9p (default) + // - virtio-fs + SharedFS string + + // VirtioFSDaemon is the virtio-fs vhost-user daemon path + VirtioFSDaemon string + + // VirtioFSCache cache mode for fs version cache or "none" + VirtioFSCache string + + // VirtioFSExtraArgs passes options to virtiofsd daemon + VirtioFSExtraArgs []string // BlockDeviceCacheSet specifies cache-related options will be set to block devices or not. BlockDeviceCacheSet bool @@ -154,6 +157,10 @@ type HypervisorConfig struct { // GuestHookPath is the path within the VM that will be used for 'drop-in' hooks GuestHookPath string + + // VMid is the id of the VM that create the hypervisor if the VM is created by the factory. + // VMid is "" if the hypervisor is not created by the factory. + VMid string } // KataAgentConfig is a structure storing information needed @@ -186,6 +193,18 @@ type ShimConfig struct { // NetworkConfig is the network configuration related to a network. type NetworkConfig struct { + NetNSPath string + NetNsCreated bool + DisableNewNetNs bool + InterworkingModel int +} + +type ContainerConfig struct { + ID string + Annotations map[string]string + RootFs string + // Resources for recoding update + Resources specs.LinuxResources } // SandboxConfig is a sandbox configuration. @@ -195,17 +214,15 @@ type SandboxConfig struct { HypervisorConfig HypervisorConfig // only one agent config can be non-nil according to agent type - AgentType string - KataAgentConfig *KataAgentConfig `json:",omitempty"` - HyperstartConfig *HyperstartConfig `json:",omitempty"` + AgentType string + KataAgentConfig *KataAgentConfig `json:",omitempty"` ProxyType string ProxyConfig ProxyConfig ShimType string - KataShimConfig ShimConfig + KataShimConfig *ShimConfig - NetworkModel string NetworkConfig NetworkConfig ShmSize uint64 @@ -220,11 +237,18 @@ type SandboxConfig struct { // SystemdCgroup enables systemd cgroup support SystemdCgroup bool + // SandboxCgroupOnly enables cgroup only at podlevel in the host + SandboxCgroupOnly bool + + DisableGuestSeccomp bool + // Experimental enables experimental features - Experimental bool + Experimental []string // Information for fields not saved: // * Annotation: this is kind of casual data, we don't need casual data in persist file, // if you know this data needs to persist, please gives it // a specific field + + ContainerConfigs []ContainerConfig } diff --git a/virtcontainers/persist/fs/fs.go b/virtcontainers/persist/fs/fs.go index 1e7552785f..0b4f28bc02 100644 --- a/virtcontainers/persist/fs/fs.go +++ b/virtcontainers/persist/fs/fs.go @@ -14,6 +14,7 @@ import ( "path/filepath" "syscall" + "github.com/kata-containers/runtime/pkg/rootless" persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" "github.com/sirupsen/logrus" ) @@ -36,9 +37,15 @@ const storagePathSuffix = "vc" // sandboxPathSuffix is the suffix used for sandbox storage const sandboxPathSuffix = "sbs" -// runStoragePath is the sandbox runtime directory. +// RunStoragePath is the sandbox runtime directory. // It will contain one state.json and one lock file for each created sandbox. -var runStoragePath = filepath.Join("/run", storagePathSuffix, sandboxPathSuffix) +var RunStoragePath = func() string { + path := filepath.Join("/run", storagePathSuffix, sandboxPathSuffix) + if rootless.IsRootless() { + return filepath.Join(rootless.GetRootlessDir(), path) + } + return path +} // FS storage driver implementation type FS struct { @@ -76,7 +83,7 @@ func (fs *FS) sandboxDir() (string, error) { return "", fmt.Errorf("sandbox container id required") } - return filepath.Join(runStoragePath, id), nil + return filepath.Join(RunStoragePath(), id), nil } // ToDisk sandboxState and containerState to disk @@ -254,8 +261,10 @@ func (fs *FS) unlock() error { return nil } -// TestSetRunStoragePath set runStoragePath to path +// TestSetRunStoragePath set RunStoragePath to path // this function is only used for testing purpose func TestSetRunStoragePath(path string) { - runStoragePath = path + RunStoragePath = func() string { + return path + } } diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index d8d548ef31..a72266e4ca 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -242,7 +242,10 @@ func (s *Sandbox) SetAnnotations(annotations map[string]string) error { s.config.Annotations[k] = v } - return s.store.Store(store.Configuration, *(s.config)) + if !s.supportNewStore() { + return s.store.Store(store.Configuration, *(s.config)) + } + return nil } // GetAnnotations returns sandbox's annotations @@ -617,25 +620,24 @@ func (s *Sandbox) storeSandbox() error { span, _ := s.trace("storeSandbox") defer span.Finish() - err := s.store.Store(store.Configuration, *(s.config)) - if err != nil { - return err - } - - for _, container := range s.containers { - err = container.store.Store(store.Configuration, *(container.config)) - if err != nil { - return err - } - } - if s.supportNewStore() { // flush data to storage if err := s.Save(); err != nil { return err } - } + } else { + err := s.store.Store(store.Configuration, *(s.config)) + if err != nil { + return err + } + for _, container := range s.containers { + err = container.store.Store(store.Configuration, *(container.config)) + if err != nil { + return err + } + } + } return nil } @@ -673,6 +675,22 @@ func unlockSandbox(ctx context.Context, sandboxID, token string) error { return store.Unlock(token) } +func supportNewStore(ctx context.Context) bool { + if exp.Get("newstore") == nil { + return false + } + + // check if client context enabled "newstore" feature + exps := exp.ExpFromContext(ctx) + for _, v := range exps { + if v == "newstore" { + return true + } + } + + return false +} + // fetchSandbox fetches a sandbox config from a sandbox ID and returns a sandbox. func fetchSandbox(ctx context.Context, sandboxID string) (sandbox *Sandbox, err error) { virtLog.Info("fetch sandbox") @@ -685,15 +703,24 @@ func fetchSandbox(ctx context.Context, sandboxID string) (sandbox *Sandbox, err return sandbox, err } - // We're bootstrapping - vcStore, err := store.NewVCSandboxStore(ctx, sandboxID) - if err != nil { - return nil, err - } - var config SandboxConfig - if err := vcStore.Load(store.Configuration, &config); err != nil { - return nil, err + + if supportNewStore(ctx) { + c, err := loadSandboxConfig(sandboxID) + if err != nil { + return nil, err + } + config = *c + } else { + // We're bootstrapping + vcStore, err := store.NewVCSandboxStore(ctx, sandboxID) + if err != nil { + return nil, err + } + + if err := vcStore.Load(store.Configuration, &config); err != nil { + return nil, err + } } // fetchSandbox is not suppose to create new sandbox VM. From 618666ed8c649df66062d83b13ba0cb26ab5940d Mon Sep 17 00:00:00 2001 From: Ted Yu Date: Tue, 19 Nov 2019 05:46:20 -0800 Subject: [PATCH 03/14] vc: Remove unnecessary call to kill Fixes #2207 Signed-off-by: Ted Yu --- virtcontainers/acrn.go | 10 +++------- virtcontainers/fc.go | 9 +++------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go index a734a7ab92..5787a25f46 100644 --- a/virtcontainers/acrn.go +++ b/virtcontainers/acrn.go @@ -513,15 +513,11 @@ func (a *Acrn) stopSandbox() (err error) { pid := a.state.PID - // Check if VM process is running, in case it is not, let's - // return from here. - if err = syscall.Kill(pid, syscall.Signal(0)); err != nil { - a.Logger().Info("acrn VM already stopped") - return nil - } - // Send signal to the VM process to try to stop it properly if err = syscall.Kill(pid, syscall.SIGINT); err != nil { + if err == syscall.ESRCH { + return nil + } a.Logger().Info("Sending signal to stop acrn VM failed") return err } diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go index e6fdb0a307..77673b8632 100644 --- a/virtcontainers/fc.go +++ b/virtcontainers/fc.go @@ -467,14 +467,11 @@ func (fc *firecracker) fcEnd() (err error) { pid := fc.info.PID - // Check if VM process is running, in case it is not, let's - // return from here. - if err = syscall.Kill(pid, syscall.Signal(0)); err != nil { - return nil - } - // Send a SIGTERM to the VM process to try to stop it properly if err = syscall.Kill(pid, syscall.SIGTERM); err != nil { + if err == syscall.ESRCH { + return nil + } return err } From 1e046791a354882ae9dcc6351dc5f37b9e9886b2 Mon Sep 17 00:00:00 2001 From: Jose Carlos Venegas Munoz Date: Tue, 19 Nov 2019 14:25:12 +0000 Subject: [PATCH 04/14] versions: Add cloud-hypervisor Needed to build as part of release and CI process. Fixes: #2218 Signed-off-by: Jose Carlos Venegas Munoz --- versions.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/versions.yaml b/versions.yaml index 428ee2bc6d..14f3fc21ff 100644 --- a/versions.yaml +++ b/versions.yaml @@ -70,6 +70,13 @@ assets: hypervisor: description: "Component used to create virtual machines" + cloud_hypervisor: + description: "Cloud Hypervisor is an open source Virtual Machine Monitor" + url: "https://github.com/intel/cloud-hypervisor" + uscan-url: >- + https://github.com/intel/cloud-hypervisor/tags.*/v?(\d\S+)\.tar\.gz + version: "v0.3.0" + firecracker: description: "Firecracker micro-VMM" url: "https://github.com/firecracker-microvm/firecracker" From 1afad1c0ad95b885b6f3d9916531ea1f1aebb7af Mon Sep 17 00:00:00 2001 From: Ted Yu Date: Tue, 19 Nov 2019 13:44:32 -0800 Subject: [PATCH 05/14] vc: Sandbox#Annotations should use annotationsLock read lock Fixes #2222 Signed-off-by: Ted Yu --- virtcontainers/sandbox.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index d8d548ef31..793f2e5425 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -225,6 +225,9 @@ func (s *Sandbox) Logger() *logrus.Entry { // Annotations returns any annotation that a user could have stored through the sandbox. func (s *Sandbox) Annotations(key string) (string, error) { + s.annotationsLock.RLock() + defer s.annotationsLock.RUnlock() + value, exist := s.config.Annotations[key] if !exist { return "", fmt.Errorf("Annotations key %s does not exist", key) From df889c37e0b6d161d3c21c8f97620ac7eaa03094 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Wed, 20 Nov 2019 00:13:14 +0000 Subject: [PATCH 06/14] release: Kata Containers 1.10.0-alpha1 - vc: Remove unnecessary call to kill - versions: Add cloud-hypervisor - clh: fix build - vc: Remove extra call to Kill - FC: log out the firecracker's console when debug enabled - Add support for intel cloud hypervisor - Remove nemu - fc: config: vhost-net not supported - qemu-configs: update disable_vhost_net description - Topic/network simplify - support firecracker 0.19.0 - rootless: Fix cgroup creation logic for rootless - FIPS: Add support for starting VM in FIPS mode. - vc/qemu: add mutex to qmp monitor channel in qmpSetup() - version: Update containerd version to 1.3.0 - rootless: Fix rangeUID parsing - versions: Update kubernetes and cri-o to 1.16 1e04679 versions: Add cloud-hypervisor 618666e vc: Remove unnecessary call to kill 510f0a6 clh: fix build ee9a53c vc: Remove extra call to Kill 784066a Makefile: clh: Use virtiofs kernel 01713d5 runtime: added cloud hypervisor driver 70297c2 nemu: remove nemu support a2b6afc FC: log out the firecracker's console when debug enabled 13a00a2 virtcontainers: add a stateful to FC struct 0c482b2 qemu-configs: update disable_vhost_net description 519eff7 fc: config: vhost-net not supported 6154210 versions: bump firecracker version 78ca966 virtcontainers: bump firecracker minimum supported version 77b0dfb virtcontainers: use new firecracker API 0def9b0 virtcontainers/firecracker: update API 744ccd4 network: Set the default config to tcfilter 27433d9 config: Get rid of bridged model docs in the configuration dffc988 virtcontainers: Eliminate legacy networking models f6ffb79 rootless: Fix cgroup creation logic for rootless 0bd41b9 FIPS: Add support for starting VM in FIPS mode. 3f1a39c rootless: Fix rangeUID parsing 69ab092 versions: Add cri-tools and conmon to the versions.yaml 5b31282 vc/qemu: add mutex to qmp monitor channel in qmpSetup() 0e70b38 versions: Update kubernetes and cri-o to 1.16 62f64bb version: Update containerd version to 1.3.0 Signed-off-by: Archana Shinde --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 50326e29ec..61f71d0ffe 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.10.0-alpha0 +1.10.0-alpha1 From fab759db73e8eb9874424024abaf637732daef48 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Wed, 20 Nov 2019 09:06:49 -0800 Subject: [PATCH 07/14] config: Disable jailer by default for firecracker Comment out jailer path so that it is disabled by default. Fixes #2228 Signed-off-by: Archana Shinde --- cli/config/configuration-fc.toml.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cli/config/configuration-fc.toml.in b/cli/config/configuration-fc.toml.in index 4b4651e37c..f595245f04 100644 --- a/cli/config/configuration-fc.toml.in +++ b/cli/config/configuration-fc.toml.in @@ -16,7 +16,9 @@ path = "@FCPATH@" # If the jailer path is not set kata will launch firecracker # without a jail. If the jailer is set firecracker will be # launched in a jailed enviornment created by the jailer -jailer_path = "@FCJAILERPATH@" +# This is disabled by default as additional setup is required +# for this feature today. +#jailer_path = "@FCJAILERPATH@" kernel = "@KERNELPATH_FC@" image = "@IMAGEPATH@" From 2331e879af9ec87fec8bb283ba851e9e1e15187c Mon Sep 17 00:00:00 2001 From: Ted Yu Date: Wed, 20 Nov 2019 11:48:34 -0800 Subject: [PATCH 08/14] vc: Persist file handle may leak in FS#FromDisk Fixes #2233 Signed-off-by: Ted Yu --- virtcontainers/persist/fs/fs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtcontainers/persist/fs/fs.go b/virtcontainers/persist/fs/fs.go index 0b4f28bc02..8aca6fb9a1 100644 --- a/virtcontainers/persist/fs/fs.go +++ b/virtcontainers/persist/fs/fs.go @@ -201,11 +201,11 @@ func (fs *FS) FromDisk(sid string) (persistapi.SandboxState, map[string]persista return ss, nil, err } + defer cf.Close() var cstate persistapi.ContainerState if err := json.NewDecoder(cf).Decode(&cstate); err != nil { return ss, nil, err } - cf.Close() fs.containerState[cid] = cstate } From 93a03369ae8c646ee0b0df4fc0ef29cceaa44465 Mon Sep 17 00:00:00 2001 From: Ted Yu Date: Wed, 20 Nov 2019 18:47:32 -0800 Subject: [PATCH 09/14] vc: Restore sandbox state when there is error starting containers Fixes #2237 Signed-off-by: Ted Yu --- virtcontainers/sandbox.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 3ab7e014ce..4a6a0d6d52 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -1523,13 +1523,21 @@ func (s *Sandbox) Start() error { return err } + prevState := s.state.State + if err := s.setSandboxState(types.StateRunning); err != nil { return err } + var startErr error + defer func() { + if startErr != nil { + s.setSandboxState(prevState) + } + }() for _, c := range s.containers { - if err := c.start(); err != nil { - return err + if startErr = c.start(); startErr != nil { + return startErr } } From 51d7c23e418f9c86eb4d737d2c8f6ece13e2ee70 Mon Sep 17 00:00:00 2001 From: choury Date: Mon, 18 Nov 2019 22:16:38 +0800 Subject: [PATCH 10/14] shim: Respect containerd's debug config There are too many logs from kata shim in containerd's log. Fixes #2209 Signed-off-by: choury --- containerd-shim-v2/service.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/containerd-shim-v2/service.go b/containerd-shim-v2/service.go index 4ffccbea9b..af26d94ef5 100644 --- a/containerd-shim-v2/service.go +++ b/containerd-shim-v2/service.go @@ -65,6 +65,10 @@ func New(ctx context.Context, id string, publisher events.Publisher) (cdshim.Shi // it will output into stdio, from which containerd would like // to get the shim's socket address. logrus.SetOutput(ioutil.Discard) + opts := ctx.Value(cdshim.OptsKey{}).(cdshim.Opts) + if !opts.Debug { + logrus.SetLevel(logrus.WarnLevel) + } vci.SetLogger(ctx, logger) katautils.SetLogger(ctx, logger, logger.Logger.Level) @@ -141,7 +145,10 @@ func newCommand(ctx context.Context, containerdBinary, id, containerdAddress str "-address", containerdAddress, "-publish-binary", containerdBinary, "-id", id, - "-debug", + } + opts := ctx.Value(cdshim.OptsKey{}).(cdshim.Opts) + if opts.Debug { + args = append(args, "-debug") } cmd := sysexec.Command(self, args...) cmd.Dir = cwd From 03478d4540789b4e4c9a79c7421121cd22371306 Mon Sep 17 00:00:00 2001 From: Ted Yu Date: Thu, 21 Nov 2019 02:27:17 -0800 Subject: [PATCH 11/14] vc: Clean up directories in case MkdirAll fails Fixes #2230 Signed-off-by: Ted Yu --- virtcontainers/persist/fs/fs.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/virtcontainers/persist/fs/fs.go b/virtcontainers/persist/fs/fs.go index 0b4f28bc02..1322e53d49 100644 --- a/virtcontainers/persist/fs/fs.go +++ b/virtcontainers/persist/fs/fs.go @@ -126,12 +126,22 @@ func (fs *FS) ToDisk(ss persistapi.SandboxState, cs map[string]persistapi.Contai return err } + var dirCreationErr error + var createdDirs []string + defer func() { + if dirCreationErr != nil && len(createdDirs) > 0 { + for _, dir := range createdDirs { + os.RemoveAll(dir) + } + } + }() // persist container configuration data for cid, cstate := range fs.containerState { cdir := filepath.Join(sandboxDir, cid) - if err := os.MkdirAll(cdir, dirMode); err != nil { - return err + if dirCreationErr = os.MkdirAll(cdir, dirMode); dirCreationErr != nil { + return dirCreationErr } + createdDirs = append(createdDirs, cdir) cfile := filepath.Join(cdir, persistFile) cf, err := os.OpenFile(cfile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileMode) From b8b6733f629a2d1cbc9e3bc771eefcf37e1e7f67 Mon Sep 17 00:00:00 2001 From: Ted Yu Date: Thu, 21 Nov 2019 12:41:50 -0800 Subject: [PATCH 12/14] vc: Persist file handle may leak in FS#ToDisk Fixes #2241 Signed-off-by: Ted Yu --- virtcontainers/persist/fs/fs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtcontainers/persist/fs/fs.go b/virtcontainers/persist/fs/fs.go index 8aca6fb9a1..ec3c59823b 100644 --- a/virtcontainers/persist/fs/fs.go +++ b/virtcontainers/persist/fs/fs.go @@ -139,10 +139,10 @@ func (fs *FS) ToDisk(ss persistapi.SandboxState, cs map[string]persistapi.Contai return err } + defer cf.Close() if err := json.NewEncoder(cf).Encode(cstate); err != nil { return err } - cf.Close() } return nil From 1abe52abd7ec97da5bbc042fb2ec6e916c0eceda Mon Sep 17 00:00:00 2001 From: Johan Kuijpers Date: Tue, 19 Nov 2019 13:37:42 +0100 Subject: [PATCH 13/14] clh: removed hard-coded vsock contextid (cid) update after review 2. applied ci static checks Fixes: #2206 Signed-off-by: Johan Kuijpers --- virtcontainers/clh.go | 32 ++++++++++++++++++++------------ virtcontainers/types/sandbox.go | 5 +++-- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go index c77b98e6d4..d5491058c2 100644 --- a/virtcontainers/clh.go +++ b/virtcontainers/clh.go @@ -192,13 +192,6 @@ func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networ iommu: false, }) - // Add the hybrid vsock device to hypervisor - clh.cliBuilder.SetVsock(&CLIVsock{ - cid: 3, - socketPath: clh.socketPath, - iommu: false, - }) - // set the initial root/boot disk of hypervisor imagePath, err := clh.config.ImageAssetPath() if err != nil { @@ -436,7 +429,18 @@ func (clh *cloudHypervisor) addDevice(devInfo interface{}, devType deviceType) e device: v.Name(), mac: v.HardwareAddr(), }) - + case types.HybridVSock: + clh.Logger().WithFields(log.Fields{ + "function": "addDevice", + "path": v.UdsPath, + "cid": v.ContextID, + "port": v.Port, + }).Info("Adding HybridVSock") + clh.cliBuilder.SetVsock(&CLIVsock{ + cid: uint32(v.ContextID), + socketPath: v.UdsPath, + iommu: false, + }) default: clh.Logger().WithField("function", "addDevice").Warnf("Add device of type %v is not supported.", v) } @@ -544,7 +548,7 @@ func (clh *cloudHypervisor) reset() { func (clh *cloudHypervisor) generateSocket(id string, useVsock bool) (interface{}, error) { if !useVsock { - return nil, fmt.Errorf("Can't generate socket path for cloud-hypervisor: vsocks is disabled") + return nil, fmt.Errorf("Can't generate hybrid vsocket for cloud-hypervisor: vsocks is disabled") } udsPath, err := clh.vsockSocketPath(id) @@ -552,11 +556,15 @@ func (clh *cloudHypervisor) generateSocket(id string, useVsock bool) (interface{ clh.Logger().Info("Can't generate socket path for cloud-hypervisor") return types.HybridVSock{}, err } - clh.Logger().WithField("function", "generateSocket").Infof("Using hybrid vsock %s:%d", udsPath, vSockPort) + _, cid, err := utils.FindContextID() + if err != nil { + return nil, err + } clh.socketPath = udsPath return types.HybridVSock{ - UdsPath: udsPath, - Port: uint32(vSockPort), + UdsPath: udsPath, + ContextID: cid, + Port: uint32(vSockPort), }, nil } diff --git a/virtcontainers/types/sandbox.go b/virtcontainers/types/sandbox.go index 59570cc1eb..3938ed57ba 100644 --- a/virtcontainers/types/sandbox.go +++ b/virtcontainers/types/sandbox.go @@ -187,8 +187,9 @@ func (s *VSock) String() string { // This kind of socket is not supported in all hypervisors. // Firecracker supports it. type HybridVSock struct { - UdsPath string - Port uint32 + UdsPath string + ContextID uint64 + Port uint32 } func (s *HybridVSock) String() string { From 0ac43558c8957d995994faa66b2829eed7328eb8 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Fri, 22 Nov 2019 13:39:39 +0000 Subject: [PATCH 14/14] gitignore: include cloud hypervisor configuration toml update .gitignore to include configuration-clh.toml to the list of untracked files fixes #2249 Signed-off-by: Julio Montes --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 62465ece24..55a515bc3f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ .git-commit .git-commit.tmp /cli/config/configuration-acrn.toml +/cli/config/configuration-clh.toml /cli/config/configuration-fc.toml /cli/config/configuration-nemu.toml /cli/config/configuration-qemu.toml