From 18e6a6effc1880a2a7cc21c1256c60ec579beeda Mon Sep 17 00:00:00 2001 From: Peng Tao Date: Fri, 13 Jul 2018 18:12:08 +0800 Subject: [PATCH] hypervisor: decouple hypervisor from sandbox A hypervisor implementation does not need to depend on a sandbox structure. Decouple them in preparation for vm factory. Signed-off-by: Peng Tao --- virtcontainers/hypervisor.go | 4 +- virtcontainers/mock_hypervisor.go | 6 +- virtcontainers/mock_hypervisor_test.go | 10 +- virtcontainers/qemu.go | 147 +++++++++++++++---------- virtcontainers/qemu_arch_base_test.go | 2 +- virtcontainers/qemu_test.go | 6 +- virtcontainers/sandbox.go | 4 +- 7 files changed, 107 insertions(+), 72 deletions(-) diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 75d2a915d9..3950f890c6 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -499,8 +499,8 @@ func RunningOnVMM(cpuInfoPath string) (bool, error) { // hypervisor is the virtcontainers hypervisor interface. // The default hypervisor implementation is Qemu. type hypervisor interface { - init(sandbox *Sandbox) error - createSandbox(sandboxConfig SandboxConfig) error + init(id string, hypervisorConfig *HypervisorConfig, vmConfig Resources, storage resourceStorage) error + createSandbox() error startSandbox() error waitSandbox(timeout int) error stopSandbox() error diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go index 89824cdfef..f86d372534 100644 --- a/virtcontainers/mock_hypervisor.go +++ b/virtcontainers/mock_hypervisor.go @@ -9,8 +9,8 @@ type mockHypervisor struct { vCPUs uint32 } -func (m *mockHypervisor) init(sandbox *Sandbox) error { - valid, err := sandbox.config.HypervisorConfig.valid() +func (m *mockHypervisor) init(id string, hypervisorConfig *HypervisorConfig, vmConfig Resources, storage resourceStorage) error { + valid, err := hypervisorConfig.valid() if valid == false || err != nil { return err } @@ -22,7 +22,7 @@ func (m *mockHypervisor) capabilities() capabilities { return capabilities{} } -func (m *mockHypervisor) createSandbox(sandboxConfig SandboxConfig) error { +func (m *mockHypervisor) createSandbox() error { return nil } diff --git a/virtcontainers/mock_hypervisor_test.go b/virtcontainers/mock_hypervisor_test.go index 86a859035b..fae4b73cab 100644 --- a/virtcontainers/mock_hypervisor_test.go +++ b/virtcontainers/mock_hypervisor_test.go @@ -15,16 +15,18 @@ func TestMockHypervisorInit(t *testing.T) { sandbox := &Sandbox{ config: &SandboxConfig{ + ID: "mock_sandbox", HypervisorConfig: HypervisorConfig{ KernelPath: "", ImagePath: "", HypervisorPath: "", }, }, + storage: &filesystem{}, } // wrong config - if err := m.init(sandbox); err == nil { + if err := m.init(sandbox.config.ID, &sandbox.config.HypervisorConfig, sandbox.config.VMConfig, sandbox.storage); err == nil { t.Fatal() } @@ -35,7 +37,7 @@ func TestMockHypervisorInit(t *testing.T) { } // right config - if err := m.init(sandbox); err != nil { + if err := m.init(sandbox.config.ID, &sandbox.config.HypervisorConfig, sandbox.config.VMConfig, sandbox.storage); err != nil { t.Fatal(err) } } @@ -43,9 +45,7 @@ func TestMockHypervisorInit(t *testing.T) { func TestMockHypervisorCreateSandbox(t *testing.T) { var m *mockHypervisor - config := SandboxConfig{} - - if err := m.createSandbox(config); err != nil { + if err := m.createSandbox(); err != nil { t.Fatal(err) } } diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 53dbffa78a..f59aa3301d 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -47,16 +47,18 @@ type QemuState struct { // qemu is an Hypervisor interface implementation for the Linux qemu hypervisor. type qemu struct { + id string + vmConfig Resources + storage resourceStorage + config HypervisorConfig qmpMonitorCh qmpChannel qemuConfig govmmQemu.Config - sandbox *Sandbox - state QemuState arch qemuArch @@ -66,7 +68,7 @@ const qmpCapErrMsg = "Failed to negoatiate QMP capabilities" const qmpSocket = "qmp.sock" -const defaultConsole = "console.sock" +const consoleSocket = "console.sock" var qemuMajorVersion int var qemuMinorVersion int @@ -170,25 +172,26 @@ func (q *qemu) qemuPath() (string, error) { } // init intializes the Qemu structure. -func (q *qemu) init(sandbox *Sandbox) error { - valid, err := sandbox.config.HypervisorConfig.valid() +func (q *qemu) init(id string, hypervisorConfig *HypervisorConfig, vmConfig Resources, storage resourceStorage) error { + valid, err := hypervisorConfig.valid() if valid == false || err != nil { return err } - q.vmConfig = sandbox.config.VMConfig - q.config = sandbox.config.HypervisorConfig - q.sandbox = sandbox + q.id = id + q.storage = storage + q.vmConfig = vmConfig + q.config = *hypervisorConfig q.arch = newQemuArch(q.config) - if err = sandbox.storage.fetchHypervisorState(sandbox.id, &q.state); err != nil { + if err = q.storage.fetchHypervisorState(q.id, &q.state); err != nil { q.Logger().Debug("Creating bridges") q.state.Bridges = q.arch.bridges(q.config.DefaultBridges) q.Logger().Debug("Creating UUID") q.state.UUID = uuid.Generate().String() - if err = sandbox.storage.storeHypervisorState(sandbox.id, q.state); err != nil { + if err = q.storage.storeHypervisorState(q.id, q.state); err != nil { return err } } @@ -238,17 +241,17 @@ func (q *qemu) memoryTopology() (govmmQemu.Memory, error) { return q.arch.memoryTopology(memMb, hostMemMb), nil } -func (q *qemu) qmpSocketPath(sandboxID string) (string, error) { - return utils.BuildSocketPath(runStoragePath, sandboxID, qmpSocket) +func (q *qemu) qmpSocketPath(id string) (string, error) { + return utils.BuildSocketPath(runStoragePath, id, qmpSocket) } -func (q *qemu) getQemuMachine(sandboxConfig SandboxConfig) (govmmQemu.Machine, error) { +func (q *qemu) getQemuMachine() (govmmQemu.Machine, error) { machine, err := q.arch.machine() if err != nil { return govmmQemu.Machine{}, err } - accelerators := sandboxConfig.HypervisorConfig.MachineAccelerators + accelerators := q.config.MachineAccelerators if accelerators != "" { if !strings.HasPrefix(accelerators, ",") { accelerators = fmt.Sprintf(",%s", accelerators) @@ -275,11 +278,65 @@ func (q *qemu) appendImage(devices []govmmQemu.Device) ([]govmmQemu.Device, erro return devices, nil } -// createSandbox is the Hypervisor sandbox creation implementation for govmmQemu. -func (q *qemu) createSandbox(sandboxConfig SandboxConfig) error { +func (q *qemu) createQmpSocket() ([]govmmQemu.QMPSocket, error) { + monitorSockPath, err := q.qmpSocketPath(q.id) + if err != nil { + return nil, err + } + + q.qmpMonitorCh = qmpChannel{ + ctx: context.Background(), + path: monitorSockPath, + } + + err = os.MkdirAll(filepath.Dir(monitorSockPath), dirMode) + if err != nil { + return nil, err + } + + return []govmmQemu.QMPSocket{ + { + Type: "unix", + Name: q.qmpMonitorCh.path, + Server: true, + NoWait: true, + }, + }, nil +} + +func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.IOThread, error) { var devices []govmmQemu.Device - machine, err := q.getQemuMachine(sandboxConfig) + console, err := q.getSandboxConsole(q.id) + if err != nil { + return nil, nil, err + } + + // Add bridges before any other devices. This way we make sure that + // bridge gets the first available PCI address i.e bridgePCIStartAddr + devices = q.arch.appendBridges(devices, q.state.Bridges) + + devices = q.arch.appendConsole(devices, console) + + if initrdPath == "" { + devices, err = q.appendImage(devices) + if err != nil { + return nil, nil, err + } + } + + var ioThread *govmmQemu.IOThread + if q.config.BlockDeviceDriver == VirtioSCSI { + devices, ioThread = q.arch.appendSCSIController(devices, q.config.EnableIOThreads) + } + + return devices, ioThread, nil + +} + +// createSandbox is the Hypervisor sandbox creation implementation for govmmQemu. +func (q *qemu) createSandbox() error { + machine, err := q.getQemuMachine() if err != nil { return err } @@ -314,7 +371,7 @@ func (q *qemu) createSandbox(sandboxConfig SandboxConfig) error { // Pass the sandbox name to the agent via the kernel command-line to // allow the agent to use it in log messages. - params := q.kernelParameters() + " " + "agent.sandbox=" + sandboxConfig.ID + params := q.kernelParameters() + " " + "agent.sandbox=" + q.id kernel := govmmQemu.Kernel{ Path: kernelPath, @@ -331,7 +388,7 @@ func (q *qemu) createSandbox(sandboxConfig SandboxConfig) error { return fmt.Errorf("UUID should not be empty") } - monitorSockPath, err := q.qmpSocketPath(sandboxConfig.ID) + monitorSockPath, err := q.qmpSocketPath(q.id) if err != nil { return err } @@ -346,41 +403,19 @@ func (q *qemu) createSandbox(sandboxConfig SandboxConfig) error { return err } - qmpSockets := []govmmQemu.QMPSocket{ - { - Type: "unix", - Name: q.qmpMonitorCh.path, - Server: true, - NoWait: true, - }, - } - - // Add bridges before any other devices. This way we make sure that - // bridge gets the first available PCI address i.e bridgePCIStartAddr - devices = q.arch.appendBridges(devices, q.state.Bridges) - - console, err := q.getSandboxConsole(sandboxConfig.ID) + qmpSockets, err := q.createQmpSocket() if err != nil { return err } - devices = q.arch.appendConsole(devices, console) - - if initrdPath == "" { - devices, err = q.appendImage(devices) - if err != nil { - return err - } - } - - var ioThread *govmmQemu.IOThread - if q.config.BlockDeviceDriver == VirtioSCSI { - devices, ioThread = q.arch.appendSCSIController(devices, q.config.EnableIOThreads) + devices, ioThread, err := q.buildDevices(initrdPath) + if err != nil { + return err } cpuModel := q.arch.cpuModel() - firmwarePath, err := sandboxConfig.HypervisorConfig.FirmwareAssetPath() + firmwarePath, err := q.config.FirmwareAssetPath() if err != nil { return err } @@ -391,7 +426,7 @@ func (q *qemu) createSandbox(sandboxConfig SandboxConfig) error { } qemuConfig := govmmQemu.Config{ - Name: fmt.Sprintf("sandbox-%s", sandboxConfig.ID), + Name: fmt.Sprintf("sandbox-%s", q.id), UUID: q.state.UUID, Path: qemuPath, Ctx: q.qmpMonitorCh.ctx, @@ -735,7 +770,7 @@ func (q *qemu) hotplugAddDevice(devInfo interface{}, devType deviceType) (interf return data, err } - return data, q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state) + return data, q.storage.storeHypervisorState(q.id, q.state) } func (q *qemu) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { @@ -744,7 +779,7 @@ func (q *qemu) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (int return data, err } - return data, q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state) + return data, q.storage.storeHypervisorState(q.id, q.state) } func (q *qemu) hotplugCPUs(vcpus uint32, op operation) (uint32, error) { @@ -820,12 +855,12 @@ func (q *qemu) hotplugAddCPUs(amount uint32) (uint32, error) { hotpluggedVCPUs++ if hotpluggedVCPUs == amount { // All vCPUs were hotplugged - return amount, q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state) + return amount, q.storage.storeHypervisorState(q.id, q.state) } } // All vCPUs were NOT hotplugged - if err := q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state); err != nil { + if err := q.storage.storeHypervisorState(q.id, q.state); err != nil { q.Logger().Errorf("failed to save hypervisor state after hotplug %d vCPUs: %v", hotpluggedVCPUs, err) } @@ -845,7 +880,7 @@ func (q *qemu) hotplugRemoveCPUs(amount uint32) (uint32, error) { // get the last vCPUs and try to remove it cpu := q.state.HotpluggedVCPUs[len(q.state.HotpluggedVCPUs)-1] if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, cpu.ID); err != nil { - _ = q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state) + _ = q.storage.storeHypervisorState(q.id, q.state) return i, fmt.Errorf("failed to hotunplug CPUs, only %d CPUs were hotunplugged: %v", i, err) } @@ -853,7 +888,7 @@ func (q *qemu) hotplugRemoveCPUs(amount uint32) (uint32, error) { q.state.HotpluggedVCPUs = q.state.HotpluggedVCPUs[:len(q.state.HotpluggedVCPUs)-1] } - return amount, q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state) + return amount, q.storage.storeHypervisorState(q.id, q.state) } func (q *qemu) hotplugMemory(memDev *memoryDevice, op operation) error { @@ -910,7 +945,7 @@ func (q *qemu) hotplugAddMemory(memDev *memoryDevice) error { } q.state.HotpluggedMemory += memDev.sizeMB - return q.sandbox.storage.storeHypervisorState(q.sandbox.id, q.state) + return q.storage.storeHypervisorState(q.id, q.state) } func (q *qemu) pauseSandbox() error { @@ -951,8 +986,8 @@ func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error { // getSandboxConsole builds the path of the console where we can read // logs coming from the sandbox. -func (q *qemu) getSandboxConsole(sandboxID string) (string, error) { - return utils.BuildSocketPath(runStoragePath, sandboxID, defaultConsole) +func (q *qemu) getSandboxConsole(id string) (string, error) { + return utils.BuildSocketPath(runStoragePath, id, consoleSocket) } // genericAppendBridges appends to devices the given bridges diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/qemu_arch_base_test.go index 77e3f880c3..518c8135d0 100644 --- a/virtcontainers/qemu_arch_base_test.go +++ b/virtcontainers/qemu_arch_base_test.go @@ -222,7 +222,7 @@ func TestQemuArchBaseAppendConsoles(t *testing.T) { assert := assert.New(t) qemuArchBase := newQemuArchBase() - path := filepath.Join(runStoragePath, sandboxID, defaultConsole) + path := filepath.Join(runStoragePath, sandboxID, consoleSocket) expectedOut := []govmmQemu.Device{ govmmQemu.SerialDevice{ diff --git a/virtcontainers/qemu_test.go b/virtcontainers/qemu_test.go index 91e15fd4b2..6c02255150 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/qemu_test.go @@ -86,7 +86,7 @@ func TestQemuInit(t *testing.T) { t.Fatalf("Could not create parent directory %s: %v", parentDir, err) } - if err := q.init(sandbox); err != nil { + if err := q.init(sandbox.id, &sandbox.config.HypervisorConfig, sandbox.config.VMConfig, sandbox.storage); err != nil { t.Fatal(err) } @@ -117,7 +117,7 @@ func TestQemuInitMissingParentDirFail(t *testing.T) { t.Fatal(err) } - if err := q.init(sandbox); err == nil { + if err := q.init(sandbox.id, &sandbox.config.HypervisorConfig, sandbox.config.VMConfig, sandbox.storage); err == nil { t.Fatal("Qemu init() expected to fail because of missing parent directory for storage") } } @@ -249,7 +249,7 @@ func TestQemuAddDeviceSerialPortDev(t *testing.T) { func TestQemuGetSandboxConsole(t *testing.T) { q := &qemu{} sandboxID := "testSandboxID" - expected := filepath.Join(runStoragePath, sandboxID, defaultConsole) + expected := filepath.Join(runStoragePath, sandboxID, consoleSocket) result, err := q.getSandboxConsole(sandboxID) if err != nil { diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index b289db5769..5c7fb18da1 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -771,11 +771,11 @@ func newSandbox(sandboxConfig SandboxConfig) (*Sandbox, error) { } }() - if err = s.hypervisor.init(s); err != nil { + if err = s.hypervisor.init(s.id, &sandboxConfig.HypervisorConfig, sandboxConfig.VMConfig, s.storage); err != nil { return nil, err } - if err = s.hypervisor.createSandbox(sandboxConfig); err != nil { + if err = s.hypervisor.createSandbox(); err != nil { return nil, err }