github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-06-27 15:57:09 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #133 from amshinde/scsi-data-plane

Browse Source 
iothread support for virtio-scsi
This commit is contained in:
Peng Tao 2018-04-04 15:39:07 +08:00 committed by GitHub
commit 0e8b6dc049
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 84 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Gopkg.lock generated
View File

@ -80,7 +80,7 @@
[[projects]]
name = "github.com/intel/govmm"
packages = ["qemu"]
revision = "82c67ab9b21e8cd0042b6c2d3be2d3705a511603"
revision = "1509acf1862ae5154c5c096f9318bd3eb434d816"
[[projects]]
name = "github.com/kata-containers/agent"
@ -244,6 +244,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "bb2ac1696f8e90526e492a6d54362549094fa844c17fbd06c727198bff6667ef"
inputs-digest = "f7c3a1b7f5cb5f891a3badcb7323f3b5fc0fa79f69dd6532ec2e2be2baafaf98"
solver-name = "gps-cdcl"
solver-version = 1

2
Gopkg.toml
View File

@ -56,7 +56,7 @@
[[constraint]]
name = "github.com/intel/govmm"
revision = "82c67ab9b21e8cd0042b6c2d3be2d3705a511603"
revision = "1509acf1862ae5154c5c096f9318bd3eb434d816"
[[constraint]]
name = "github.com/kata-containers/agent"

4
Makefile
View File

@ -101,6 +101,7 @@ DEFNETWORKMODEL := macvtap
DEFDISABLEBLOCK := false
DEFBLOCKSTORAGEDRIVER := virtio-scsi
DEFENABLEIOTHREADS := false
DEFENABLEMEMPREALLOC := false
DEFENABLEHUGEPAGES := false
DEFENABLESWAP := false
@ -172,6 +173,7 @@ USER_VARS += DEFBRIDGES
USER_VARS += DEFNETWORKMODEL
USER_VARS += DEFDISABLEBLOCK
USER_VARS += DEFBLOCKSTORAGEDRIVER
USER_VARS += DEFENABLEIOTHREADS
USER_VARS += DEFENABLEMEMPREALLOC
USER_VARS += DEFENABLEHUGEPAGES
USER_VARS += DEFENABLESWAP
@ -263,6 +265,7 @@ const defaultBridgesCount uint32 = $(DEFBRIDGES)
const defaultInterNetworkingModel = "$(DEFNETWORKMODEL)"
const defaultDisableBlockDeviceUse bool = $(DEFDISABLEBLOCK)
const defaultBlockDeviceDriver = "$(DEFBLOCKSTORAGEDRIVER)"
const defaultEnableIOThreads bool = $(DEFENABLEIOTHREADS)
const defaultEnableMemPrealloc bool = $(DEFENABLEMEMPREALLOC)
const defaultEnableHugePages bool = $(DEFENABLEHUGEPAGES)
const defaultEnableSwap bool = $(DEFENABLESWAP)
@ -346,6 +349,7 @@ $(GENERATED_FILES): %: %.in Makefile VERSION
-e "s|@DEFNETWORKMODEL@|$(DEFNETWORKMODEL)|g" \
-e "s|@DEFDISABLEBLOCK@|$(DEFDISABLEBLOCK)|g" \
-e "s|@DEFBLOCKSTORAGEDRIVER@|$(DEFBLOCKSTORAGEDRIVER)|g" \
-e "s|@DEFENABLEIOTHREADS@|$(DEFENABLEIOTHREADS)|g" \
-e "s|@DEFENABLEMEMPREALLOC@|$(DEFENABLEMEMPREALLOC)|g" \
-e "s|@DEFENABLEHUGEPAGES@|$(DEFENABLEHUGEPAGES)|g" \
-e "s|@DEFENABLEMSWAP@|$(DEFENABLESWAP)|g" \

3
cli/config.go
View File

@ -92,6 +92,7 @@ type hypervisor struct {
Swap bool `toml:"enable_swap"`
Debug bool `toml:"enable_debug"`
DisableNestingChecks bool `toml:"disable_nesting_checks"`
EnableIOThreads bool `toml:"enable_iothreads"`
}
type proxy struct {
@ -321,6 +322,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
Debug: h.Debug,
DisableNestingChecks: h.DisableNestingChecks,
BlockDeviceDriver: blockDriver,
EnableIOThreads: h.EnableIOThreads,
}, nil
}
@ -423,6 +425,7 @@ func loadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
Debug: defaultEnableDebug,
DisableNestingChecks: defaultDisableNestingChecks,
BlockDeviceDriver: defaultBlockDeviceDriver,
EnableIOThreads: defaultEnableIOThreads,
}
err = config.InterNetworkModel.SetModel(defaultInterNetworkingModel)

6
cli/config/configuration.toml.in
View File

@ -70,6 +70,12 @@ disable_block_device_use = @DEFDISABLEBLOCK@
# virtio-blk.
block_device_driver = "@DEFBLOCKSTORAGEDRIVER@"
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented
# for SCSI.
#
enable_iothreads = @DEFENABLEIOTHREADS@
# Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density
# as all of the memory will be allocated and locked

13
cli/config_test.go
View File

@ -41,7 +41,7 @@ type testRuntimeConfig struct {
LogPath string
}
func makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath, kernelParams, machineType, shimPath, proxyPath, logPath string, disableBlock bool, blockDeviceDriver string) string {
func makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath, kernelParams, machineType, shimPath, proxyPath, logPath string, disableBlock bool, blockDeviceDriver string, enableIOThreads bool) string {
return `
# Runtime configuration file
@ -55,6 +55,7 @@ func makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath
default_vcpus = ` + strconv.FormatUint(uint64(defaultVCPUCount), 10) + `
default_memory = ` + strconv.FormatUint(uint64(defaultMemSize), 10) + `
disable_block_device_use = ` + strconv.FormatBool(disableBlock) + `
enable_iothreads = ` + strconv.FormatBool(enableIOThreads) + `
[proxy.kata]
path = "` + proxyPath + `"
@ -101,8 +102,9 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
machineType := "machineType"
disableBlockDevice := true
blockDeviceDriver := "virtio-scsi"
enableIOThreads := true
runtimeConfigFileData := makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath, kernelParams, machineType, shimPath, proxyPath, logPath, disableBlockDevice, blockDeviceDriver)
runtimeConfigFileData := makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath, kernelParams, machineType, shimPath, proxyPath, logPath, disableBlockDevice, blockDeviceDriver, enableIOThreads)
configPath := path.Join(dir, "runtime.toml")
err = createConfig(configPath, runtimeConfigFileData)
@ -140,6 +142,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
BlockDeviceDriver: defaultBlockDeviceDriver,
DefaultBridges: defaultBridgesCount,
Mlock: !defaultEnableSwap,
EnableIOThreads: enableIOThreads,
}
agentConfig := vc.KataAgentConfig{}
@ -569,6 +572,7 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
imagePath := path.Join(dir, "image")
machineType := "machineType"
disableBlock := true
enableIOThreads := true
hypervisor := hypervisor{
Path: hypervisorPath,
@ -576,6 +580,7 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
Image: imagePath,
MachineType: machineType,
DisableBlockDeviceUse: disableBlock,
EnableIOThreads: enableIOThreads,
}
files := []string{hypervisorPath, kernelPath, imagePath}
@ -617,6 +622,10 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
t.Errorf("Expected value for disable block usage %v, got %v", disableBlock, config.DisableBlockDeviceUse)
}
if config.EnableIOThreads != enableIOThreads {
t.Errorf("Expected value for enable IOThreads %v, got %v", enableIOThreads, config.EnableIOThreads)
}
}
func TestNewShimConfig(t *testing.T) {

5
cli/kata-env_test.go
View File

@ -69,6 +69,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
proxyPath := filepath.Join(prefixDir, "proxy")
disableBlock := true
blockStorageDriver := "virtio-scsi"
enableIOThreads := true
// override
defaultProxyPath = proxyPath
@ -112,7 +113,9 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
testProxyURL,
logPath,
disableBlock,
blockStorageDriver)
blockStorageDriver,
enableIOThreads,
)
configFile = path.Join(prefixDir, "runtime.toml")
err = createConfig(configFile, runtimeConfig)

23
vendor/github.com/intel/govmm/qemu/qemu.go generated vendored
View File

@ -841,6 +841,9 @@ type SCSIController struct {
// DisableModern prevents qemu from relying on fast MMIO.
DisableModern bool
// IOThread is the IO thread on which IO will be handled
IOThread string
}
// Valid returns true if the SCSIController structure is valid and complete.
@ -867,6 +870,9 @@ func (scsiCon SCSIController) QemuParams(config *Config) []string {
if scsiCon.DisableModern {
devParams = append(devParams, fmt.Sprintf("disable-modern=true"))
}
if scsiCon.IOThread != "" {
devParams = append(devParams, fmt.Sprintf("iothread=%s", scsiCon.IOThread))
}
qemuParams = append(qemuParams, "-device")
qemuParams = append(qemuParams, strings.Join(devParams, ","))
@ -1159,6 +1165,11 @@ type Knobs struct {
Realtime bool
}
// IOThread allows IO to be performed on a separate thread.
type IOThread struct {
ID string
}
// Config is the qemu configuration structure.
// It allows for passing custom settings and parameters to the qemu API.
type Config struct {
@ -1213,6 +1224,8 @@ type Config struct {
// fds is a list of open file descriptors to be passed to the spawned qemu process
fds []*os.File
IOThreads []IOThread
qemuParams []string
}
@ -1481,6 +1494,15 @@ func (config *Config) appendBios() {
}
}
func (config *Config) appendIOThreads() {
for _, t := range config.IOThreads {
if t.ID != "" {
config.qemuParams = append(config.qemuParams, "-object")
config.qemuParams = append(config.qemuParams, fmt.Sprintf("iothread,id=%s", t.ID))
}
}
}
// LaunchQemu can be used to launch a new qemu instance.
//
// The Config parameter contains a set of qemu parameters and settings.
@ -1504,6 +1526,7 @@ func LaunchQemu(config Config, logger QMPLog) (string, error) {
config.appendKnobs()
config.appendKernel()
config.appendBios()
config.appendIOThreads()
if err := config.appendCPUs(); err != nil {
return "", err

4
virtcontainers/hypervisor.go
View File

@ -201,6 +201,10 @@ type HypervisorConfig struct {
// DisableBlockDeviceUse disallows a block device from being used.
DisableBlockDeviceUse bool
// EnableIOThreads enables IO to be processed in a separate thread.
// Supported currently for virtio-scsi driver.
EnableIOThreads bool
// Debug changes the default hypervisor and kernel parameters to
// enable debug output where available.
Debug bool

7
virtcontainers/qemu.go
View File

@ -379,8 +379,9 @@ func (q *qemu) createPod(podConfig PodConfig) error {
return err
}
var ioThread *govmmQemu.IOThread
if q.config.BlockDeviceDriver == VirtioSCSI {
devices = q.arch.appendSCSIController(devices)
devices, ioThread = q.arch.appendSCSIController(devices, q.config.EnableIOThreads)
}
cpuModel := q.arch.cpuModel()
@ -414,6 +415,10 @@ func (q *qemu) createPod(podConfig PodConfig) error {
Bios: firmwarePath,
}
if ioThread != nil {
qemuConfig.IOThreads = []govmmQemu.IOThread{*ioThread}
}
q.qemuConfig = qemuConfig
return nil

18
virtcontainers/qemu_arch_base.go
View File

@ -66,7 +66,7 @@ type qemuArch interface {
appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error)
// appendSCSIController appens a SCSI controller to devices
appendSCSIController(devices []govmmQemu.Device) []govmmQemu.Device
appendSCSIController(devices []govmmQemu.Device, enableIOThreads bool) ([]govmmQemu.Device, *govmmQemu.IOThread)
// appendBridges appends bridges to devices
appendBridges(devices []govmmQemu.Device, bridges []Bridge) []govmmQemu.Device
@ -300,15 +300,27 @@ func (q *qemuArchBase) appendImage(devices []govmmQemu.Device, path string) ([]g
return q.appendBlockDevice(devices, drive), nil
}
func (q *qemuArchBase) appendSCSIController(devices []govmmQemu.Device) []govmmQemu.Device {
func (q *qemuArchBase) appendSCSIController(devices []govmmQemu.Device, enableIOThreads bool) ([]govmmQemu.Device, *govmmQemu.IOThread) {
scsiController := govmmQemu.SCSIController{
ID: scsiControllerID,
DisableModern: q.nestedRun,
}
var t *govmmQemu.IOThread
if enableIOThreads {
randBytes, _ := generateRandomBytes(8)
t = &govmmQemu.IOThread{
ID: fmt.Sprintf("%s-%s", "iothread", hex.EncodeToString(randBytes)),
}
scsiController.IOThread = t.ID
}
devices = append(devices, scsiController)
return devices
return devices, t
}
// appendBridges appends to devices the given bridges

6
virtcontainers/qemu_arch_base_test.go
View File

@ -470,6 +470,10 @@ func TestQemuArchBaseAppendSCSIController(t *testing.T) {
},
}
devices = qemuArchBase.appendSCSIController(devices)
devices, ioThread := qemuArchBase.appendSCSIController(devices, false)
assert.Equal(expectedOut, devices)
assert.Nil(ioThread)
_, ioThread = qemuArchBase.appendSCSIController(devices, true)
assert.NotNil(ioThread)
}