diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go index 897aae3668..d3453cf1b6 100644 --- a/src/runtime/virtcontainers/persist/api/config.go +++ b/src/runtime/virtcontainers/persist/api/config.go @@ -13,31 +13,6 @@ import ( // HypervisorConfig saves configurations of sandbox hypervisor type HypervisorConfig struct { - // NumVCPUs specifies default number of vCPUs for the VM. - NumVCPUs uint32 - - //DefaultMaxVCPUs specifies the maximum number of vCPUs for the VM. - DefaultMaxVCPUs uint32 - - // DefaultMem specifies default memory size in MiB for the VM. - MemorySize uint32 - - // DefaultBridges specifies default number of bridges for the VM. - // Bridges can be used to hot plug devices - DefaultBridges uint32 - - // Msize9p is used as the msize for 9p shares - Msize9p uint32 - - // MemSlots specifies default memory slots the VM. - MemSlots uint32 - - // MemOffset specifies memory space for nvdimm device - MemOffset uint64 - - // VirtioFSCacheSize is the DAX cache size in MiB - VirtioFSCacheSize uint32 - // KernelPath is the guest kernel host path. KernelPath string @@ -60,22 +35,13 @@ type HypervisorConfig struct { // HypervisorPath is the hypervisor executable host path. HypervisorPath string - // HypervisorPathList is the list of hypervisor paths names allowed in annotations - HypervisorPathList []string - // HypervisorCtlPath is the hypervisor ctl executable host path. HypervisorCtlPath string - // HypervisorCtlPathList is the list of hypervisor control paths names allowed in annotations - HypervisorCtlPathList []string - // HypervisorCtlPath is the hypervisor ctl executable host path. // JailerPath is the jailer executable host path. JailerPath string - // JailerPathList is the list of jailer paths names allowed in annotations - JailerPathList []string - // BlockDeviceDriver specifies the driver to be used for block device // either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver BlockDeviceDriver string @@ -96,9 +62,6 @@ type HypervisorConfig struct { // entropy (/dev/random, /dev/urandom or real hardware RNG device) EntropySource string - // EntropySourceList is the list of valid entropy sources - EntropySourceList []string - // Shared file system type: // - virtio-9p (default) // - virtio-fs @@ -107,21 +70,89 @@ type HypervisorConfig struct { // VirtioFSDaemon is the virtio-fs vhost-user daemon path VirtioFSDaemon string - // VirtioFSDaemonList is the list of valid virtiofs names for annotations - VirtioFSDaemonList []string - // VirtioFSCache cache mode for fs version cache or "none" VirtioFSCache string - // VirtioFSExtraArgs passes options to virtiofsd daemon - VirtioFSExtraArgs []string - // File based memory backend root directory FileBackedMemRootDir string + // VhostUserStorePath is the directory path where vhost-user devices + // related folders, sockets and device nodes should be. + VhostUserStorePath string + + // GuestHookPath is the path within the VM that will be used for 'drop-in' hooks + GuestHookPath string + + // VMid is the id of the VM that create the hypervisor if the VM is created by the factory. + // VMid is "" if the hypervisor is not created by the factory. + VMid string + + // HypervisorPathList is the list of hypervisor paths names allowed in annotations + HypervisorPathList []string + + // HypervisorCtlPathList is the list of hypervisor control paths names allowed in annotations + HypervisorCtlPathList []string + + // JailerPathList is the list of jailer paths names allowed in annotations + JailerPathList []string + + // EntropySourceList is the list of valid entropy sources + EntropySourceList []string + + // VirtioFSDaemonList is the list of valid virtiofs names for annotations + VirtioFSDaemonList []string + + // VirtioFSExtraArgs passes options to virtiofsd daemon + VirtioFSExtraArgs []string + // FileBackedMemRootList is the list of valid root directories values for annotations FileBackedMemRootList []string + // VhostUserStorePathList is the list of valid values for vhost-user paths + VhostUserStorePathList []string + + // Enable annotations by name + EnableAnnotations []string + + // MemOffset specifies memory space for nvdimm device + MemOffset uint64 + + // RxRateLimiterMaxRate is used to control network I/O inbound bandwidth on VM level. + RxRateLimiterMaxRate uint64 + + // TxRateLimiterMaxRate is used to control network I/O outbound bandwidth on VM level. + TxRateLimiterMaxRate uint64 + + // SGXEPCSize specifies the size in bytes for the EPC Section. + // Enable SGX. Hardware-based isolation and memory encryption. + SGXEPCSize int64 + + // PCIeRootPort is used to indicate the number of PCIe Root Port devices + // The PCIe Root Port device is used to hot-plug the PCIe device + PCIeRootPort uint32 + + // NumVCPUs specifies default number of vCPUs for the VM. + NumVCPUs uint32 + + //DefaultMaxVCPUs specifies the maximum number of vCPUs for the VM. + DefaultMaxVCPUs uint32 + + // DefaultMem specifies default memory size in MiB for the VM. + MemorySize uint32 + + // DefaultBridges specifies default number of bridges for the VM. + // Bridges can be used to hot plug devices + DefaultBridges uint32 + + // Msize9p is used as the msize for 9p shares + Msize9p uint32 + + // MemSlots specifies default memory slots the VM. + MemSlots uint32 + + // VirtioFSCacheSize is the DAX cache size in MiB + VirtioFSCacheSize uint32 + // BlockDeviceCacheSet specifies cache-related options will be set to block devices or not. BlockDeviceCacheSet bool @@ -172,10 +203,6 @@ type HypervisorConfig struct { // root bus instead of a bridge. HotplugVFIOOnRootBus bool - // PCIeRootPort is used to indicate the number of PCIe Root Port devices - // The PCIe Root Port device is used to hot-plug the PCIe device - PCIeRootPort uint32 - // BootToBeTemplate used to indicate if the VM is created to be a template VM BootToBeTemplate bool @@ -187,33 +214,6 @@ type HypervisorConfig struct { // EnableVhostUserStore is used to indicate if host supports vhost-user-blk/scsi EnableVhostUserStore bool - - // VhostUserStorePath is the directory path where vhost-user devices - // related folders, sockets and device nodes should be. - VhostUserStorePath string - - // VhostUserStorePathList is the list of valid values for vhost-user paths - VhostUserStorePathList []string - - // GuestHookPath is the path within the VM that will be used for 'drop-in' hooks - GuestHookPath string - - // VMid is the id of the VM that create the hypervisor if the VM is created by the factory. - // VMid is "" if the hypervisor is not created by the factory. - VMid string - - // RxRateLimiterMaxRate is used to control network I/O inbound bandwidth on VM level. - RxRateLimiterMaxRate uint64 - - // TxRateLimiterMaxRate is used to control network I/O outbound bandwidth on VM level. - TxRateLimiterMaxRate uint64 - - // SGXEPCSize specifies the size in bytes for the EPC Section. - // Enable SGX. Hardware-based isolation and memory encryption. - SGXEPCSize int64 - - // Enable annotations by name - EnableAnnotations []string } // KataAgentConfig is a structure storing information needed @@ -238,25 +238,41 @@ type NetworkConfig struct { } type ContainerConfig struct { - ID string Annotations map[string]string - RootFs string // Resources for recoding update Resources specs.LinuxResources + ID string + RootFs string } // SandboxConfig is a sandbox configuration. // Refs: virtcontainers/sandbox.go:SandboxConfig type SandboxConfig struct { - HypervisorType string - HypervisorConfig HypervisorConfig + // Information for fields not saved: + // * Annotation: this is kind of casual data, we don't need casual data in persist file, + // if you know this data needs to persist, please gives it + // a specific field + + ContainerConfigs []ContainerConfig + + // SandboxBindMounts - list of paths to mount into guest + SandboxBindMounts []string + + // Experimental enables experimental features + Experimental []string + + // Cgroups specifies specific cgroup settings for the various subsystems that the container is + // placed into to limit the resources the container has available + Cgroups *configs.Cgroup `json:"cgroups"` // only one agent config can be non-nil according to agent type KataAgentConfig *KataAgentConfig `json:",omitempty"` KataShimConfig *ShimConfig - NetworkConfig NetworkConfig + HypervisorType string + NetworkConfig NetworkConfig + HypervisorConfig HypervisorConfig ShmSize uint64 @@ -274,21 +290,4 @@ type SandboxConfig struct { SandboxCgroupOnly bool DisableGuestSeccomp bool - - // SandboxBindMounts - list of paths to mount into guest - SandboxBindMounts []string - - // Experimental enables experimental features - Experimental []string - - // Information for fields not saved: - // * Annotation: this is kind of casual data, we don't need casual data in persist file, - // if you know this data needs to persist, please gives it - // a specific field - - ContainerConfigs []ContainerConfig - - // Cgroups specifies specific cgroup settings for the various subsystems that the container is - // placed into to limit the resources the container has available - Cgroups *configs.Cgroup `json:"cgroups"` } diff --git a/src/runtime/virtcontainers/persist/api/container.go b/src/runtime/virtcontainers/persist/api/container.go index 0418b81412..4767b79e95 100644 --- a/src/runtime/virtcontainers/persist/api/container.go +++ b/src/runtime/virtcontainers/persist/api/container.go @@ -41,19 +41,19 @@ type Mount struct { // Type specifies the type of filesystem to mount. Type string - // Options list all the mount options of the filesystem. - Options []string - // HostPath used to store host side bind mount path HostPath string - // ReadOnly specifies if the mount should be read only or not - ReadOnly bool - // BlockDeviceID represents block device that is attached to the // VM in case this mount is a block device file or a directory // backed by a block device. BlockDeviceID string + + // Options list all the mount options of the filesystem. + Options []string + + // ReadOnly specifies if the mount should be read only or not + ReadOnly bool } // RootfsState saves state of container rootfs @@ -69,6 +69,8 @@ type RootfsState struct { // Process gathers data related to a container process. // Refs: virtcontainers/container.go:Process type Process struct { + StartTime time.Time + // Token is the process execution context ID. It must be // unique per sandbox. // Token is used to manipulate processes for containers @@ -80,8 +82,6 @@ type Process struct { // stack, e.g. CRI-O, containerd. This is typically the // shim PID. Pid int - - StartTime time.Time } // ContainerState represents container state @@ -89,23 +89,23 @@ type ContainerState struct { // State is container running status State string - // Rootfs contains information of container rootfs - Rootfs RootfsState - // CgroupPath is the cgroup hierarchy where sandbox's processes // including the hypervisor are placed. CgroupPath string + // BundlePath saves container OCI config.json, which can be unmarshaled + // and translated to "CompatOCISpec" + BundlePath string + + // Rootfs contains information of container rootfs + Rootfs RootfsState + + // Process on host representing container process + Process Process + // DeviceMaps is mapping between sandbox device to dest in container DeviceMaps []DeviceMap // Mounts is mount info from OCI spec Mounts []Mount - - // Process on host representing container process - Process Process - - // BundlePath saves container OCI config.json, which can be unmarshaled - // and translated to "CompatOCISpec" - BundlePath string } diff --git a/src/runtime/virtcontainers/persist/api/device.go b/src/runtime/virtcontainers/persist/api/device.go index 5e6c6e6f73..2ac88fa8ba 100644 --- a/src/runtime/virtcontainers/persist/api/device.go +++ b/src/runtime/virtcontainers/persist/api/device.go @@ -22,15 +22,9 @@ type BlockDrive struct { // ID is used to identify this drive in the hypervisor options. ID string - // Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index - Index int - // MmioAddr is used to identify the slot at which the drive is attached (order?). MmioAddr string - // PCIPath is the PCI path used to identify the slot at which the drive is attached. - PCIPath vcTypes.PciPath - // SCSI Address of the block device, in case the device is attached using SCSI driver // SCSI address is in the format SCSI-Id:LUN SCSIAddr string @@ -44,6 +38,12 @@ type BlockDrive struct { // DevNo DevNo string + // PCIPath is the PCI path used to identify the slot at which the drive is attached. + PCIPath vcTypes.PciPath + + // Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index + Index int + // Pmem enabled persistent memory. Use File as backing file // for a nvdimm device in the guest. Pmem bool @@ -54,14 +54,14 @@ type VFIODev struct { // ID is used to identify this drive in the hypervisor options. ID string - // Type of VFIO device - Type uint32 - // BDF (Bus:Device.Function) of the PCI address BDF string // Sysfsdev of VFIO mediated device SysfsDev string + + // Type of VFIO device + Type uint32 } // VhostUserDeviceAttrs represents data shared by most vhost-user devices @@ -85,15 +85,22 @@ type VhostUserDeviceAttrs struct { // plugged to hypervisor, one Device can be shared among containers in POD // Refs: virtcontainers/device/drivers/generic.go:GenericDevice type DeviceState struct { + // DriverOptions is specific options for each device driver + // for example, for BlockDevice, we can set DriverOptions["blockDriver"]="virtio-blk" + DriverOptions map[string]string + + // VhostUserDeviceAttrs is specific for vhost-user device driver + VhostUserDev *VhostUserDeviceAttrs `json:",omitempty"` + + // BlockDrive is specific for block device driver + BlockDrive *BlockDrive `json:",omitempty"` + ID string // Type is used to specify driver type // Refs: virtcontainers/device/config/config.go:DeviceType Type string - RefCount uint - AttachCount uint - // Type of device: c, b, u or p // c , u - character(unbuffered) // p - FIFO @@ -101,6 +108,12 @@ type DeviceState struct { // More info in mknod(1). DevType string + // VFIODev is specific VFIO device driver + VFIODevs []*VFIODev `json:",omitempty"` + + RefCount uint + AttachCount uint + // Major, minor numbers for device. Major int64 Minor int64 @@ -108,19 +121,4 @@ type DeviceState struct { // ColdPlug specifies whether the device must be cold plugged (true) // or hot plugged (false). ColdPlug bool - - // DriverOptions is specific options for each device driver - // for example, for BlockDevice, we can set DriverOptions["blockDriver"]="virtio-blk" - DriverOptions map[string]string - - // ============ device driver specific data =========== - // BlockDrive is specific for block device driver - BlockDrive *BlockDrive `json:",omitempty"` - - // VFIODev is specific VFIO device driver - VFIODevs []*VFIODev `json:",omitempty"` - - // VhostUserDeviceAttrs is specific for vhost-user device driver - VhostUserDev *VhostUserDeviceAttrs `json:",omitempty"` - // ============ end device driver specific data =========== } diff --git a/src/runtime/virtcontainers/persist/api/hypervisor.go b/src/runtime/virtcontainers/persist/api/hypervisor.go index 375fd56b14..b2d41000f5 100644 --- a/src/runtime/virtcontainers/persist/api/hypervisor.go +++ b/src/runtime/virtcontainers/persist/api/hypervisor.go @@ -27,22 +27,24 @@ type CPUDevice struct { } type HypervisorState struct { - Pid int - // Type of hypervisor, E.g. qemu/firecracker/acrn. - Type string BlockIndexMap map[int]struct{} - UUID string + + // Type of hypervisor, E.g. qemu/firecracker/acrn. + Type string + UUID string + // clh sepcific: refer to 'virtcontainers/clh.go:CloudHypervisorState' + APISocket string // Belows are qemu specific // Refs: virtcontainers/qemu.go:QemuState Bridges []Bridge // HotpluggedCPUs is the list of CPUs that were hot-added - HotpluggedVCPUs []CPUDevice - HotpluggedMemory int - VirtiofsdPid int - HotplugVFIOOnRootBus bool - PCIeRootPort int + HotpluggedVCPUs []CPUDevice - // clh sepcific: refer to 'virtcontainers/clh.go:CloudHypervisorState' - APISocket string + HotpluggedMemory int + VirtiofsdPid int + Pid int + PCIeRootPort int + + HotplugVFIOOnRootBus bool } diff --git a/src/runtime/virtcontainers/persist/api/network.go b/src/runtime/virtcontainers/persist/api/network.go index 824f884539..020fca06ae 100644 --- a/src/runtime/virtcontainers/persist/api/network.go +++ b/src/runtime/virtcontainers/persist/api/network.go @@ -81,8 +81,6 @@ type VhostUserEndpoint struct { // NetworkEndpoint contains network interface information type NetworkEndpoint struct { - Type string - // One and only one of these below are not nil according to Type. Physical *PhysicalEndpoint `json:",omitempty"` Veth *VethEndpoint `json:",omitempty"` @@ -92,12 +90,14 @@ type NetworkEndpoint struct { Tap *TapEndpoint `json:",omitempty"` IPVlan *IPVlanEndpoint `json:",omitempty"` Tuntap *TuntapEndpoint `json:",omitempty"` + + Type string } // NetworkInfo contains network information of sandbox type NetworkInfo struct { NetNsPath string + Endpoints []NetworkEndpoint NetmonPID int NetNsCreated bool - Endpoints []NetworkEndpoint } diff --git a/src/runtime/virtcontainers/persist/api/sandbox.go b/src/runtime/virtcontainers/persist/api/sandbox.go index 6f6e341154..61b4afe88e 100644 --- a/src/runtime/virtcontainers/persist/api/sandbox.go +++ b/src/runtime/virtcontainers/persist/api/sandbox.go @@ -17,18 +17,16 @@ type AgentState struct { // SandboxState contains state information of sandbox // nolint: maligned type SandboxState struct { - // PersistVersion of persist data format, can be used for keeping compatibility later - PersistVersion uint + // CgroupPath is the cgroup hierarchy where sandbox's processes + // including the hypervisor are placed. + CgroupPaths map[string]string + + // Devices plugged to sandbox(hypervisor) + Devices []DeviceState // State is sandbox running status State string - // GuestMemoryBlockSizeMB is the size of memory block of guestos - GuestMemoryBlockSizeMB uint32 - - // GuestMemoryHotplugProbe determines whether guest kernel supports memory hotplug probe interface - GuestMemoryHotplugProbe bool - // SandboxContainer specifies which container is used to start the sandbox/vm SandboxContainer string @@ -36,13 +34,6 @@ type SandboxState struct { // including the hypervisor are placed. CgroupPath string - // CgroupPath is the cgroup hierarchy where sandbox's processes - // including the hypervisor are placed. - CgroupPaths map[string]string - - // Devices plugged to sandbox(hypervisor) - Devices []DeviceState - // HypervisorState saves hypervisor specific data HypervisorState HypervisorState @@ -54,4 +45,13 @@ type SandboxState struct { // Config saves config information of sandbox Config SandboxConfig + + // PersistVersion of persist data format, can be used for keeping compatibility later + PersistVersion uint + + // GuestMemoryBlockSizeMB is the size of memory block of guestos + GuestMemoryBlockSizeMB uint32 + + // GuestMemoryHotplugProbe determines whether guest kernel supports memory hotplug probe interface + GuestMemoryHotplugProbe bool }