hypervisor: decouple hypervisor from sandbox

A hypervisor implementation does not need to depend on a sandbox
structure. Decouple them in preparation for vm factory.

Signed-off-by: Peng Tao <bergwolf@gmail.com>
This commit is contained in:
Peng Tao 2018-07-13 18:12:08 +08:00
parent 4ac675453f
commit 18e6a6effc
7 changed files with 107 additions and 72 deletions

View File

@ -499,8 +499,8 @@ func RunningOnVMM(cpuInfoPath string) (bool, error) {
// hypervisor is the virtcontainers hypervisor interface. // hypervisor is the virtcontainers hypervisor interface.
// The default hypervisor implementation is Qemu. // The default hypervisor implementation is Qemu.
type hypervisor interface { type hypervisor interface {
init(sandbox *Sandbox) error init(id string, hypervisorConfig *HypervisorConfig, vmConfig Resources, storage resourceStorage) error
createSandbox(sandboxConfig SandboxConfig) error createSandbox() error
startSandbox() error startSandbox() error
waitSandbox(timeout int) error waitSandbox(timeout int) error
stopSandbox() error stopSandbox() error

View File

@ -9,8 +9,8 @@ type mockHypervisor struct {
vCPUs uint32 vCPUs uint32
} }
func (m *mockHypervisor) init(sandbox *Sandbox) error { func (m *mockHypervisor) init(id string, hypervisorConfig *HypervisorConfig, vmConfig Resources, storage resourceStorage) error {
valid, err := sandbox.config.HypervisorConfig.valid() valid, err := hypervisorConfig.valid()
if valid == false || err != nil { if valid == false || err != nil {
return err return err
} }
@ -22,7 +22,7 @@ func (m *mockHypervisor) capabilities() capabilities {
return capabilities{} return capabilities{}
} }
func (m *mockHypervisor) createSandbox(sandboxConfig SandboxConfig) error { func (m *mockHypervisor) createSandbox() error {
return nil return nil
} }

View File

@ -15,16 +15,18 @@ func TestMockHypervisorInit(t *testing.T) {
sandbox := &Sandbox{ sandbox := &Sandbox{
config: &SandboxConfig{ config: &SandboxConfig{
ID: "mock_sandbox",
HypervisorConfig: HypervisorConfig{ HypervisorConfig: HypervisorConfig{
KernelPath: "", KernelPath: "",
ImagePath: "", ImagePath: "",
HypervisorPath: "", HypervisorPath: "",
}, },
}, },
storage: &filesystem{},
} }
// wrong config // wrong config
if err := m.init(sandbox); err == nil { if err := m.init(sandbox.config.ID, &sandbox.config.HypervisorConfig, sandbox.config.VMConfig, sandbox.storage); err == nil {
t.Fatal() t.Fatal()
} }
@ -35,7 +37,7 @@ func TestMockHypervisorInit(t *testing.T) {
} }
// right config // right config
if err := m.init(sandbox); err != nil { if err := m.init(sandbox.config.ID, &sandbox.config.HypervisorConfig, sandbox.config.VMConfig, sandbox.storage); err != nil {
t.Fatal(err) t.Fatal(err)
} }
} }
@ -43,9 +45,7 @@ func TestMockHypervisorInit(t *testing.T) {
func TestMockHypervisorCreateSandbox(t *testing.T) { func TestMockHypervisorCreateSandbox(t *testing.T) {
var m *mockHypervisor var m *mockHypervisor
config := SandboxConfig{} if err := m.createSandbox(); err != nil {
if err := m.createSandbox(config); err != nil {
t.Fatal(err) t.Fatal(err)
} }
} }

View File

@ -47,16 +47,18 @@ type QemuState struct {
// qemu is an Hypervisor interface implementation for the Linux qemu hypervisor. // qemu is an Hypervisor interface implementation for the Linux qemu hypervisor.
type qemu struct { type qemu struct {
id string
vmConfig Resources vmConfig Resources
storage resourceStorage
config HypervisorConfig config HypervisorConfig
qmpMonitorCh qmpChannel qmpMonitorCh qmpChannel
qemuConfig govmmQemu.Config qemuConfig govmmQemu.Config
sandbox *Sandbox
state QemuState state QemuState
arch qemuArch arch qemuArch
@ -66,7 +68,7 @@ const qmpCapErrMsg = "Failed to negoatiate QMP capabilities"
const qmpSocket = "qmp.sock" const qmpSocket = "qmp.sock"
const defaultConsole = "console.sock" const consoleSocket = "console.sock"
var qemuMajorVersion int var qemuMajorVersion int
var qemuMinorVersion int var qemuMinorVersion int
@ -170,25 +172,26 @@ func (q *qemu) qemuPath() (string, error) {
} }
// init intializes the Qemu structure. // init intializes the Qemu structure.
func (q *qemu) init(sandbox *Sandbox) error { func (q *qemu) init(id string, hypervisorConfig *HypervisorConfig, vmConfig Resources, storage resourceStorage) error {
valid, err := sandbox.config.HypervisorConfig.valid() valid, err := hypervisorConfig.valid()
if valid == false || err != nil { if valid == false || err != nil {
return err return err
} }
q.vmConfig = sandbox.config.VMConfig q.id = id
q.config = sandbox.config.HypervisorConfig q.storage = storage
q.sandbox = sandbox q.vmConfig = vmConfig
q.config = *hypervisorConfig
q.arch = newQemuArch(q.config) q.arch = newQemuArch(q.config)
if err = sandbox.storage.fetchHypervisorState(sandbox.id, &q.state); err != nil { if err = q.storage.fetchHypervisorState(q.id, &q.state); err != nil {
q.Logger().Debug("Creating bridges") q.Logger().Debug("Creating bridges")
q.state.Bridges = q.arch.bridges(q.config.DefaultBridges) q.state.Bridges = q.arch.bridges(q.config.DefaultBridges)
q.Logger().Debug("Creating UUID") q.Logger().Debug("Creating UUID")
q.state.UUID = uuid.Generate().String() q.state.UUID = uuid.Generate().String()
if err = sandbox.storage.storeHypervisorState(sandbox.id, q.state); err != nil { if err = q.storage.storeHypervisorState(q.id, q.state); err != nil {
return err return err
} }
} }
@ -238,17 +241,17 @@ func (q *qemu) memoryTopology() (govmmQemu.Memory, error) {
return q.arch.memoryTopology(memMb, hostMemMb), nil return q.arch.memoryTopology(memMb, hostMemMb), nil
} }
func (q *qemu) qmpSocketPath(sandboxID string) (string, error) { func (q *qemu) qmpSocketPath(id string) (string, error) {
return utils.BuildSocketPath(runStoragePath, sandboxID, qmpSocket) return utils.BuildSocketPath(runStoragePath, id, qmpSocket)
} }
func (q *qemu) getQemuMachine(sandboxConfig SandboxConfig) (govmmQemu.Machine, error) { func (q *qemu) getQemuMachine() (govmmQemu.Machine, error) {
machine, err := q.arch.machine() machine, err := q.arch.machine()
if err != nil { if err != nil {
return govmmQemu.Machine{}, err return govmmQemu.Machine{}, err
} }
accelerators := sandboxConfig.HypervisorConfig.MachineAccelerators accelerators := q.config.MachineAccelerators
if accelerators != "" { if accelerators != "" {
if !strings.HasPrefix(accelerators, ",") { if !strings.HasPrefix(accelerators, ",") {
accelerators = fmt.Sprintf(",%s", accelerators) accelerators = fmt.Sprintf(",%s", accelerators)
@ -275,11 +278,65 @@ func (q *qemu) appendImage(devices []govmmQemu.Device) ([]govmmQemu.Device, erro
return devices, nil return devices, nil
} }
// createSandbox is the Hypervisor sandbox creation implementation for govmmQemu. func (q *qemu) createQmpSocket() ([]govmmQemu.QMPSocket, error) {
func (q *qemu) createSandbox(sandboxConfig SandboxConfig) error { monitorSockPath, err := q.qmpSocketPath(q.id)
if err != nil {
return nil, err
}
q.qmpMonitorCh = qmpChannel{
ctx: context.Background(),
path: monitorSockPath,
}
err = os.MkdirAll(filepath.Dir(monitorSockPath), dirMode)
if err != nil {
return nil, err
}
return []govmmQemu.QMPSocket{
{
Type: "unix",
Name: q.qmpMonitorCh.path,
Server: true,
NoWait: true,
},
}, nil
}
func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.IOThread, error) {
var devices []govmmQemu.Device var devices []govmmQemu.Device
machine, err := q.getQemuMachine(sandboxConfig) console, err := q.getSandboxConsole(q.id)
if err != nil {
return nil, nil, err
}
// Add bridges before any other devices. This way we make sure that
// bridge gets the first available PCI address i.e bridgePCIStartAddr
devices = q.arch.appendBridges(devices, q.state.Bridges)
devices = q.arch.appendConsole(devices, console)
if initrdPath == "" {
devices, err = q.appendImage(devices)
if err != nil {
return nil, nil, err
}
}
var ioThread *govmmQemu.IOThread
if q.config.BlockDeviceDriver == VirtioSCSI {
devices, ioThread = q.arch.appendSCSIController(devices, q.config.EnableIOThreads)
}
return devices, ioThread, nil
}
// createSandbox is the Hypervisor sandbox creation implementation for govmmQemu.
func (q *qemu) createSandbox() error {
machine, err := q.getQemuMachine()
if err != nil { if err != nil {
return err return err
} }
@ -314,7 +371,7 @@ func (q *qemu) createSandbox(sandboxConfig SandboxConfig) error {
// Pass the sandbox name to the agent via the kernel command-line to // Pass the sandbox name to the agent via the kernel command-line to
// allow the agent to use it in log messages. // allow the agent to use it in log messages.
params := q.kernelParameters() + " " + "agent.sandbox=" + sandboxConfig.ID params := q.kernelParameters() + " " + "agent.sandbox=" + q.id
kernel := govmmQemu.Kernel{ kernel := govmmQemu.Kernel{
Path: kernelPath, Path: kernelPath,
@ -331,7 +388,7 @@ func (q *qemu) createSandbox(sandboxConfig SandboxConfig) error {
return fmt.Errorf("UUID should not be empty") return fmt.Errorf("UUID should not be empty")
} }
monitorSockPath, err := q.qmpSocketPath(sandboxConfig.ID) monitorSockPath, err := q.qmpSocketPath(q.id)
if err != nil { if err != nil {
return err return err
} }
@ -346,41 +403,19 @@ func (q *qemu) createSandbox(sandboxConfig SandboxConfig) error {
return err return err
} }
qmpSockets := []govmmQemu.QMPSocket{ qmpSockets, err := q.createQmpSocket()
{
Type: "unix",
Name: q.qmpMonitorCh.path,
Server: true,
NoWait: true,
},
}
// Add bridges before any other devices. This way we make sure that
// bridge gets the first available PCI address i.e bridgePCIStartAddr
devices = q.arch.appendBridges(devices, q.state.Bridges)
console, err := q.getSandboxConsole(sandboxConfig.ID)
if err != nil { if err != nil {
return err return err
} }
devices = q.arch.appendConsole(devices, console) devices, ioThread, err := q.buildDevices(initrdPath)
if err != nil {
if initrdPath == "" { return err
devices, err = q.appendImage(devices)
if err != nil {
return err
}
}
var ioThread *govmmQemu.IOThread
if q.config.BlockDeviceDriver == VirtioSCSI {
devices, ioThread = q.arch.appendSCSIController(devices, q.config.EnableIOThreads)
} }
cpuModel := q.arch.cpuModel() cpuModel := q.arch.cpuModel()
firmwarePath, err := sandboxConfig.HypervisorConfig.FirmwareAssetPath() firmwarePath, err := q.config.FirmwareAssetPath()
if err != nil { if err != nil {
return err return err
} }
@ -391,7 +426,7 @@ func (q *qemu) createSandbox(sandboxConfig SandboxConfig) error {
} }
qemuConfig := govmmQemu.Config{ qemuConfig := govmmQemu.Config{
Name: fmt.Sprintf("sandbox-%s", sandboxConfig.ID), Name: fmt.Sprintf("sandbox-%s", q.id),
UUID: q.state.UUID, UUID: q.state.UUID,
Path: qemuPath, Path: qemuPath,
Ctx: q.qmpMonitorCh.ctx, Ctx: q.qmpMonitorCh.ctx,
@ -735,7 +770,7 @@ func (q *qemu) hotplugAddDevice(devInfo interface{}, devType deviceType) (interf
return data, err return data, err
} }
return data, q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state) return data, q.storage.storeHypervisorState(q.id, q.state)
} }
func (q *qemu) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { func (q *qemu) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) {
@ -744,7 +779,7 @@ func (q *qemu) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (int
return data, err return data, err
} }
return data, q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state) return data, q.storage.storeHypervisorState(q.id, q.state)
} }
func (q *qemu) hotplugCPUs(vcpus uint32, op operation) (uint32, error) { func (q *qemu) hotplugCPUs(vcpus uint32, op operation) (uint32, error) {
@ -820,12 +855,12 @@ func (q *qemu) hotplugAddCPUs(amount uint32) (uint32, error) {
hotpluggedVCPUs++ hotpluggedVCPUs++
if hotpluggedVCPUs == amount { if hotpluggedVCPUs == amount {
// All vCPUs were hotplugged // All vCPUs were hotplugged
return amount, q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state) return amount, q.storage.storeHypervisorState(q.id, q.state)
} }
} }
// All vCPUs were NOT hotplugged // All vCPUs were NOT hotplugged
if err := q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state); err != nil { if err := q.storage.storeHypervisorState(q.id, q.state); err != nil {
q.Logger().Errorf("failed to save hypervisor state after hotplug %d vCPUs: %v", hotpluggedVCPUs, err) q.Logger().Errorf("failed to save hypervisor state after hotplug %d vCPUs: %v", hotpluggedVCPUs, err)
} }
@ -845,7 +880,7 @@ func (q *qemu) hotplugRemoveCPUs(amount uint32) (uint32, error) {
// get the last vCPUs and try to remove it // get the last vCPUs and try to remove it
cpu := q.state.HotpluggedVCPUs[len(q.state.HotpluggedVCPUs)-1] cpu := q.state.HotpluggedVCPUs[len(q.state.HotpluggedVCPUs)-1]
if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, cpu.ID); err != nil { if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, cpu.ID); err != nil {
_ = q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state) _ = q.storage.storeHypervisorState(q.id, q.state)
return i, fmt.Errorf("failed to hotunplug CPUs, only %d CPUs were hotunplugged: %v", i, err) return i, fmt.Errorf("failed to hotunplug CPUs, only %d CPUs were hotunplugged: %v", i, err)
} }
@ -853,7 +888,7 @@ func (q *qemu) hotplugRemoveCPUs(amount uint32) (uint32, error) {
q.state.HotpluggedVCPUs = q.state.HotpluggedVCPUs[:len(q.state.HotpluggedVCPUs)-1] q.state.HotpluggedVCPUs = q.state.HotpluggedVCPUs[:len(q.state.HotpluggedVCPUs)-1]
} }
return amount, q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state) return amount, q.storage.storeHypervisorState(q.id, q.state)
} }
func (q *qemu) hotplugMemory(memDev *memoryDevice, op operation) error { func (q *qemu) hotplugMemory(memDev *memoryDevice, op operation) error {
@ -910,7 +945,7 @@ func (q *qemu) hotplugAddMemory(memDev *memoryDevice) error {
} }
q.state.HotpluggedMemory += memDev.sizeMB q.state.HotpluggedMemory += memDev.sizeMB
return q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state) return q.storage.storeHypervisorState(q.id, q.state)
} }
func (q *qemu) pauseSandbox() error { func (q *qemu) pauseSandbox() error {
@ -951,8 +986,8 @@ func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error {
// getSandboxConsole builds the path of the console where we can read // getSandboxConsole builds the path of the console where we can read
// logs coming from the sandbox. // logs coming from the sandbox.
func (q *qemu) getSandboxConsole(sandboxID string) (string, error) { func (q *qemu) getSandboxConsole(id string) (string, error) {
return utils.BuildSocketPath(runStoragePath, sandboxID, defaultConsole) return utils.BuildSocketPath(runStoragePath, id, consoleSocket)
} }
// genericAppendBridges appends to devices the given bridges // genericAppendBridges appends to devices the given bridges

View File

@ -222,7 +222,7 @@ func TestQemuArchBaseAppendConsoles(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
qemuArchBase := newQemuArchBase() qemuArchBase := newQemuArchBase()
path := filepath.Join(runStoragePath, sandboxID, defaultConsole) path := filepath.Join(runStoragePath, sandboxID, consoleSocket)
expectedOut := []govmmQemu.Device{ expectedOut := []govmmQemu.Device{
govmmQemu.SerialDevice{ govmmQemu.SerialDevice{

View File

@ -86,7 +86,7 @@ func TestQemuInit(t *testing.T) {
t.Fatalf("Could not create parent directory %s: %v", parentDir, err) t.Fatalf("Could not create parent directory %s: %v", parentDir, err)
} }
if err := q.init(sandbox); err != nil { if err := q.init(sandbox.id, &sandbox.config.HypervisorConfig, sandbox.config.VMConfig, sandbox.storage); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -117,7 +117,7 @@ func TestQemuInitMissingParentDirFail(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
if err := q.init(sandbox); err == nil { if err := q.init(sandbox.id, &sandbox.config.HypervisorConfig, sandbox.config.VMConfig, sandbox.storage); err == nil {
t.Fatal("Qemu init() expected to fail because of missing parent directory for storage") t.Fatal("Qemu init() expected to fail because of missing parent directory for storage")
} }
} }
@ -249,7 +249,7 @@ func TestQemuAddDeviceSerialPortDev(t *testing.T) {
func TestQemuGetSandboxConsole(t *testing.T) { func TestQemuGetSandboxConsole(t *testing.T) {
q := &qemu{} q := &qemu{}
sandboxID := "testSandboxID" sandboxID := "testSandboxID"
expected := filepath.Join(runStoragePath, sandboxID, defaultConsole) expected := filepath.Join(runStoragePath, sandboxID, consoleSocket)
result, err := q.getSandboxConsole(sandboxID) result, err := q.getSandboxConsole(sandboxID)
if err != nil { if err != nil {

View File

@ -771,11 +771,11 @@ func newSandbox(sandboxConfig SandboxConfig) (*Sandbox, error) {
} }
}() }()
if err = s.hypervisor.init(s); err != nil { if err = s.hypervisor.init(s.id, &sandboxConfig.HypervisorConfig, sandboxConfig.VMConfig, s.storage); err != nil {
return nil, err return nil, err
} }
if err = s.hypervisor.createSandbox(sandboxConfig); err != nil { if err = s.hypervisor.createSandbox(); err != nil {
return nil, err return nil, err
} }