Merge pull request #133 from amshinde/scsi-data-plane

iothread support for virtio-scsi

Gopkg.lock

@@ -80,7 +80,7 @@
 [[projects]]
   name = "github.com/intel/govmm"
   packages = ["qemu"]
-  revision = "82c67ab9b21e8cd0042b6c2d3be2d3705a511603"
+  revision = "1509acf1862ae5154c5c096f9318bd3eb434d816"
 
 [[projects]]
   name = "github.com/kata-containers/agent"
@@ -244,6 +244,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "bb2ac1696f8e90526e492a6d54362549094fa844c17fbd06c727198bff6667ef"
+  inputs-digest = "f7c3a1b7f5cb5f891a3badcb7323f3b5fc0fa79f69dd6532ec2e2be2baafaf98"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -56,7 +56,7 @@
 
 [[constraint]]
   name = "github.com/intel/govmm"
-  revision = "82c67ab9b21e8cd0042b6c2d3be2d3705a511603"
+  revision = "1509acf1862ae5154c5c096f9318bd3eb434d816"
 
 [[constraint]]
   name = "github.com/kata-containers/agent"

Makefile

@@ -101,6 +101,7 @@ DEFNETWORKMODEL := macvtap
 
 DEFDISABLEBLOCK := false
 DEFBLOCKSTORAGEDRIVER := virtio-scsi
+DEFENABLEIOTHREADS := false
 DEFENABLEMEMPREALLOC := false
 DEFENABLEHUGEPAGES := false
 DEFENABLESWAP := false
@@ -172,6 +173,7 @@ USER_VARS += DEFBRIDGES
 USER_VARS += DEFNETWORKMODEL
 USER_VARS += DEFDISABLEBLOCK
 USER_VARS += DEFBLOCKSTORAGEDRIVER
+USER_VARS += DEFENABLEIOTHREADS
 USER_VARS += DEFENABLEMEMPREALLOC
 USER_VARS += DEFENABLEHUGEPAGES
 USER_VARS += DEFENABLESWAP
@@ -263,6 +265,7 @@ const defaultBridgesCount uint32 = $(DEFBRIDGES)
 const defaultInterNetworkingModel = "$(DEFNETWORKMODEL)"
 const defaultDisableBlockDeviceUse bool = $(DEFDISABLEBLOCK)
 const defaultBlockDeviceDriver = "$(DEFBLOCKSTORAGEDRIVER)"
+const defaultEnableIOThreads bool = $(DEFENABLEIOTHREADS)
 const defaultEnableMemPrealloc bool = $(DEFENABLEMEMPREALLOC)
 const defaultEnableHugePages bool = $(DEFENABLEHUGEPAGES)
 const defaultEnableSwap bool = $(DEFENABLESWAP)
@@ -346,6 +349,7 @@ $(GENERATED_FILES): %: %.in Makefile VERSION
 		-e "s|@DEFNETWORKMODEL@|$(DEFNETWORKMODEL)|g" \
 		-e "s|@DEFDISABLEBLOCK@|$(DEFDISABLEBLOCK)|g" \
 		-e "s|@DEFBLOCKSTORAGEDRIVER@|$(DEFBLOCKSTORAGEDRIVER)|g" \
+		-e "s|@DEFENABLEIOTHREADS@|$(DEFENABLEIOTHREADS)|g" \
 		-e "s|@DEFENABLEMEMPREALLOC@|$(DEFENABLEMEMPREALLOC)|g" \
 		-e "s|@DEFENABLEHUGEPAGES@|$(DEFENABLEHUGEPAGES)|g" \
 		-e "s|@DEFENABLEMSWAP@|$(DEFENABLESWAP)|g" \

cli/config.go

@@ -92,6 +92,7 @@ type hypervisor struct {
 	Swap                  bool   `toml:"enable_swap"`
 	Debug                 bool   `toml:"enable_debug"`
 	DisableNestingChecks  bool   `toml:"disable_nesting_checks"`
+	EnableIOThreads       bool   `toml:"enable_iothreads"`
 }
 
 type proxy struct {
@@ -321,6 +322,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		Debug:                 h.Debug,
 		DisableNestingChecks:  h.DisableNestingChecks,
 		BlockDeviceDriver:     blockDriver,
+		EnableIOThreads:       h.EnableIOThreads,
 	}, nil
 }
 
@@ -423,6 +425,7 @@ func loadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
 		Debug:                 defaultEnableDebug,
 		DisableNestingChecks:  defaultDisableNestingChecks,
 		BlockDeviceDriver:     defaultBlockDeviceDriver,
+		EnableIOThreads:       defaultEnableIOThreads,
 	}
 
 	err = config.InterNetworkModel.SetModel(defaultInterNetworkingModel)

cli/config/configuration.toml.in

@@ -70,6 +70,12 @@ disable_block_device_use = @DEFDISABLEBLOCK@
 # virtio-blk.
 block_device_driver = "@DEFBLOCKSTORAGEDRIVER@"
 
+# Enable iothreads (data-plane) to be used. This causes IO to be
+# handled in a separate IO thread. This is currently only implemented
+# for SCSI.
+#
+enable_iothreads = @DEFENABLEIOTHREADS@
+
 # Enable pre allocation of VM RAM, default false
 # Enabling this will result in lower container density
 # as all of the memory will be allocated and locked

cli/config_test.go

@@ -41,7 +41,7 @@ type testRuntimeConfig struct {
 	LogPath           string
 }
 
-func makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath, kernelParams, machineType, shimPath, proxyPath, logPath string, disableBlock bool, blockDeviceDriver string) string {
+func makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath, kernelParams, machineType, shimPath, proxyPath, logPath string, disableBlock bool, blockDeviceDriver string, enableIOThreads bool) string {
 	return `
 	# Runtime configuration file
 
@@ -55,6 +55,7 @@ func makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath
 	default_vcpus = ` + strconv.FormatUint(uint64(defaultVCPUCount), 10) + `
 	default_memory = ` + strconv.FormatUint(uint64(defaultMemSize), 10) + `
 	disable_block_device_use =  ` + strconv.FormatBool(disableBlock) + `
+	enable_iothreads =  ` + strconv.FormatBool(enableIOThreads) + `
 
 	[proxy.kata]
 	path = "` + proxyPath + `"
@@ -101,8 +102,9 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 	machineType := "machineType"
 	disableBlockDevice := true
 	blockDeviceDriver := "virtio-scsi"
+	enableIOThreads := true
 
-	runtimeConfigFileData := makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath, kernelParams, machineType, shimPath, proxyPath, logPath, disableBlockDevice, blockDeviceDriver)
+	runtimeConfigFileData := makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath, kernelParams, machineType, shimPath, proxyPath, logPath, disableBlockDevice, blockDeviceDriver, enableIOThreads)
 
 	configPath := path.Join(dir, "runtime.toml")
 	err = createConfig(configPath, runtimeConfigFileData)
@@ -140,6 +142,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		BlockDeviceDriver:     defaultBlockDeviceDriver,
 		DefaultBridges:        defaultBridgesCount,
 		Mlock:                 !defaultEnableSwap,
+		EnableIOThreads:       enableIOThreads,
 	}
 
 	agentConfig := vc.KataAgentConfig{}
@@ -569,6 +572,7 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 	imagePath := path.Join(dir, "image")
 	machineType := "machineType"
 	disableBlock := true
+	enableIOThreads := true
 
 	hypervisor := hypervisor{
 		Path:                  hypervisorPath,
@@ -576,6 +580,7 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 		Image:                 imagePath,
 		MachineType:           machineType,
 		DisableBlockDeviceUse: disableBlock,
+		EnableIOThreads:       enableIOThreads,
 	}
 
 	files := []string{hypervisorPath, kernelPath, imagePath}
@@ -617,6 +622,10 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 		t.Errorf("Expected value for disable block usage %v, got %v", disableBlock, config.DisableBlockDeviceUse)
 	}
 
+	if config.EnableIOThreads != enableIOThreads {
+		t.Errorf("Expected value for enable IOThreads  %v, got %v", enableIOThreads, config.EnableIOThreads)
+	}
+
 }
 
 func TestNewShimConfig(t *testing.T) {

cli/kata-env_test.go

@@ -69,6 +69,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 	proxyPath := filepath.Join(prefixDir, "proxy")
 	disableBlock := true
 	blockStorageDriver := "virtio-scsi"
+	enableIOThreads := true
 
 	// override
 	defaultProxyPath = proxyPath
@@ -112,7 +113,9 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 		testProxyURL,
 		logPath,
 		disableBlock,
-		blockStorageDriver)
+		blockStorageDriver,
+		enableIOThreads,
+	)
 
 	configFile = path.Join(prefixDir, "runtime.toml")
 	err = createConfig(configFile, runtimeConfig)

vendor/github.com/intel/govmm/qemu/qemu.go

@@ -841,6 +841,9 @@ type SCSIController struct {
 
 	// DisableModern prevents qemu from relying on fast MMIO.
 	DisableModern bool
+
+	// IOThread is the IO thread on which IO will be handled
+	IOThread string
 }
 
 // Valid returns true if the SCSIController structure is valid and complete.
@@ -867,6 +870,9 @@ func (scsiCon SCSIController) QemuParams(config *Config) []string {
 	if scsiCon.DisableModern {
 		devParams = append(devParams, fmt.Sprintf("disable-modern=true"))
 	}
+	if scsiCon.IOThread != "" {
+		devParams = append(devParams, fmt.Sprintf("iothread=%s", scsiCon.IOThread))
+	}
 
 	qemuParams = append(qemuParams, "-device")
 	qemuParams = append(qemuParams, strings.Join(devParams, ","))
@@ -1159,6 +1165,11 @@ type Knobs struct {
 	Realtime bool
 }
 
+// IOThread allows IO to be performed on a separate thread.
+type IOThread struct {
+	ID string
+}
+
 // Config is the qemu configuration structure.
 // It allows for passing custom settings and parameters to the qemu API.
 type Config struct {
@@ -1213,6 +1224,8 @@ type Config struct {
 	// fds is a list of open file descriptors to be passed to the spawned qemu process
 	fds []*os.File
 
+	IOThreads []IOThread
+
 	qemuParams []string
 }
 
@@ -1481,6 +1494,15 @@ func (config *Config) appendBios() {
 	}
 }
 
+func (config *Config) appendIOThreads() {
+	for _, t := range config.IOThreads {
+		if t.ID != "" {
+			config.qemuParams = append(config.qemuParams, "-object")
+			config.qemuParams = append(config.qemuParams, fmt.Sprintf("iothread,id=%s", t.ID))
+		}
+	}
+}
+
 // LaunchQemu can be used to launch a new qemu instance.
 //
 // The Config parameter contains a set of qemu parameters and settings.
@@ -1504,6 +1526,7 @@ func LaunchQemu(config Config, logger QMPLog) (string, error) {
 	config.appendKnobs()
 	config.appendKernel()
 	config.appendBios()
+	config.appendIOThreads()
 
 	if err := config.appendCPUs(); err != nil {
 		return "", err

virtcontainers/hypervisor.go

@@ -201,6 +201,10 @@ type HypervisorConfig struct {
 	// DisableBlockDeviceUse disallows a block device from being used.
 	DisableBlockDeviceUse bool
 
+	// EnableIOThreads enables IO to be processed in a separate thread.
+	// Supported currently for virtio-scsi driver.
+	EnableIOThreads bool
+
 	// Debug changes the default hypervisor and kernel parameters to
 	// enable debug output where available.
 	Debug bool

virtcontainers/qemu.go

@@ -379,8 +379,9 @@ func (q *qemu) createPod(podConfig PodConfig) error {
 		return err
 	}
 
+	var ioThread *govmmQemu.IOThread
 	if q.config.BlockDeviceDriver == VirtioSCSI {
-		devices = q.arch.appendSCSIController(devices)
+		devices, ioThread = q.arch.appendSCSIController(devices, q.config.EnableIOThreads)
 	}
 
 	cpuModel := q.arch.cpuModel()
@@ -414,6 +415,10 @@ func (q *qemu) createPod(podConfig PodConfig) error {
 		Bios:        firmwarePath,
 	}
 
+	if ioThread != nil {
+		qemuConfig.IOThreads = []govmmQemu.IOThread{*ioThread}
+	}
+
 	q.qemuConfig = qemuConfig
 
 	return nil

virtcontainers/qemu_arch_base.go

@@ -66,7 +66,7 @@ type qemuArch interface {
 	appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error)
 
 	// appendSCSIController appens a SCSI controller to devices
-	appendSCSIController(devices []govmmQemu.Device) []govmmQemu.Device
+	appendSCSIController(devices []govmmQemu.Device, enableIOThreads bool) ([]govmmQemu.Device, *govmmQemu.IOThread)
 
 	// appendBridges appends bridges to devices
 	appendBridges(devices []govmmQemu.Device, bridges []Bridge) []govmmQemu.Device
@@ -300,15 +300,27 @@ func (q *qemuArchBase) appendImage(devices []govmmQemu.Device, path string) ([]g
 	return q.appendBlockDevice(devices, drive), nil
 }
 
-func (q *qemuArchBase) appendSCSIController(devices []govmmQemu.Device) []govmmQemu.Device {
+func (q *qemuArchBase) appendSCSIController(devices []govmmQemu.Device, enableIOThreads bool) ([]govmmQemu.Device, *govmmQemu.IOThread) {
 	scsiController := govmmQemu.SCSIController{
 		ID:            scsiControllerID,
 		DisableModern: q.nestedRun,
 	}
 
+	var t *govmmQemu.IOThread
+
+	if enableIOThreads {
+		randBytes, _ := generateRandomBytes(8)
+
+		t = &govmmQemu.IOThread{
+			ID: fmt.Sprintf("%s-%s", "iothread", hex.EncodeToString(randBytes)),
+		}
+
+		scsiController.IOThread = t.ID
+	}
+
 	devices = append(devices, scsiController)
 
-	return devices
+	return devices, t
 }
 
 // appendBridges appends to devices the given bridges

virtcontainers/qemu_arch_base_test.go

@@ -470,6 +470,10 @@ func TestQemuArchBaseAppendSCSIController(t *testing.T) {
 		},
 	}
 
-	devices = qemuArchBase.appendSCSIController(devices)
+	devices, ioThread := qemuArchBase.appendSCSIController(devices, false)
 	assert.Equal(expectedOut, devices)
+	assert.Nil(ioThread)
+
+	_, ioThread = qemuArchBase.appendSCSIController(devices, true)
+	assert.NotNil(ioThread)
 }