DM: Virtio-Blk Rescan

This patch adds support to trigger rescan of virtio-blk device by the
guest VM. This is an alternate to hot-plugging virtio-blk device.
This feature stems from the kata requirement, which hot-plugs container
rootfs after the VM is launched.

As part of virtio-blk rescan,
1. Update the backing file for the virtio-blk device with valid file.
   Basically update the empty file (with dummy bctxt) that was passed
   during VM launch.
2. Update virtio-blk device configurations for udpated backing file.
3. Update size associated with valid backing file in the config space.
4. Notify guest OS, of the new config change.
5. On this notification, guest will do the following.
	(i). Update virtio-blk capacity.
 	(ii). Revalidate the disk.
 	(iii). Identify the newly plugged block device.

v5 -> v6:
- Removed use of dummy file and added a new parameter "nodisk"
  to virtio-blk which indicates user wants to create a virtio-blk
  device with dummy backend.
- Moved vm_monitor_rescan from pci core to virtio-blk as it currently
  applies to only virtio-blk.

v4 -> v5:
- Reverted back logic, so that blkrescan is only supported when
  VM is launched with empty backend file.

v3 -> v4:
- Close block context before allocating a new one
- Allow backend filepath  with additional options to be more generic
- Remove blank lines introduced as part of previous patches.

v2 -> v3:
- Renamed vdev ops vdev_blk rescan to vdev_rescan
- Renamed montior ops virtio_blkrescan_ops to virtio_rescan_ops
- Consolidated virtio-blk configuration specific part into
  a separate function
- Removed size requirement in acrnctl command.

v1 -> v2:
- Added more comments in the code.
- Renamed APIs from displug to blkrescan, inline with acrnctl cmd.
- Split the patch into two. This corresponds to changes in acrn-dm.

Tracked-On: #3051
Signed-off-by: Vijay Dhanraj <vijay.dhanraj@intel.com>
Acked-by: Yu Wang <yu1.wang@intel.com>

devicemodel/core/monitor.c

@@ -391,6 +391,35 @@ static void handle_query(struct mngr_msg *msg, int client_fd, void *param)
 	mngr_send_msg(client_fd, &ack, NULL, ACK_TIMEOUT);
 }
 
+static void handle_blkrescan(struct mngr_msg *msg, int client_fd, void *param)
+{
+	struct mngr_msg ack;
+	struct vm_ops *ops;
+	int ret = 0;
+	int count = 0;
+
+	ack.magic = MNGR_MSG_MAGIC;
+	ack.msgid = msg->msgid;
+	ack.timestamp = msg->timestamp;
+
+	wakeup_reason = msg->data.reason;
+
+	LIST_FOREACH(ops, &vm_ops_head, list) {
+		if (ops->ops->rescan) {
+			ret += ops->ops->rescan(ops->arg, msg->data.devargs);
+			count++;
+		}
+	}
+
+	if (!count) {
+		ack.data.err = -1;
+		fprintf(stderr, "No handler for id:%u\r\n", msg->msgid);
+	} else
+		ack.data.err = ret;
+
+	mngr_send_msg(client_fd, &ack, NULL, ACK_TIMEOUT);
+}
+
 static struct monitor_vm_ops pmc_ops = {
 	.stop       = NULL,
 	.resume     = vm_monitor_resume,
@@ -432,6 +461,7 @@ int monitor_init(struct vmctx *ctx)
 	ret += mngr_add_handler(monitor_fd, DM_PAUSE, handle_pause, NULL);
 	ret += mngr_add_handler(monitor_fd, DM_CONTINUE, handle_continue, NULL);
 	ret += mngr_add_handler(monitor_fd, DM_QUERY, handle_query, NULL);
+	ret += mngr_add_handler(monitor_fd, DM_BLKRESCAN, handle_blkrescan, NULL);
 
 	if (ret) {
 		pr_err("%s %d\r\n", __func__, __LINE__);

devicemodel/hw/pci/core.c

@@ -2397,6 +2397,25 @@ pci_emul_dior(struct vmctx *ctx, int vcpu, struct pci_vdev *dev, int baridx,
 	return value;
 }
 
+struct pci_vdev*
+pci_get_vdev_info(int slot)
+{
+	struct businfo *bi;
+	struct slotinfo *si;
+	struct pci_vdev *dev = NULL;
+
+	bi = pci_businfo[0];
+	assert(bi != NULL);
+
+	si = &bi->slotinfo[slot];
+	if (si != NULL)
+		dev = si->si_funcs[0].fi_devi;
+	else
+		fprintf(stderr, "slot=%d is empty!\n", slot);
+
+	return dev;
+}
+
 struct pci_vdev_ops pci_dummy = {
 	.class_name	= "dummy",
 	.vdev_init	= pci_emul_dinit,

devicemodel/hw/pci/virtio/virtio_block.c

@@ -39,6 +39,7 @@
 #include "pci_core.h"
 #include "virtio.h"
 #include "block_if.h"
+#include "monitor.h"
 
 #define VIRTIO_BLK_RINGSZ	64
 #define VIRTIO_BLK_MAX_OPTS_LEN	256
@@ -131,6 +132,15 @@ static int virtio_blk_debug;
 #define DPRINTF(params) do { if (virtio_blk_debug) printf params; } while (0)
 #define WPRINTF(params) (printf params)
 
+/*
+ * Flag to indicate VM Monitor Rescan registration.
+ */
+static bool register_vm_monitor_blkrescan = false;
+
+static struct monitor_vm_ops virtio_blk_rescan_ops = {
+	.rescan	= vm_monitor_blkrescan,
+};
+
 struct virtio_blk_ioreq {
 	struct blockif_req req;
 	struct virtio_blk *blk;
@@ -146,6 +156,7 @@ struct virtio_blk {
 	pthread_mutex_t mtx;
 	struct virtio_vq_info vq;
 	struct virtio_blk_config cfg;
+	bool dummy_bctxt; /* Used in blockrescan. Indicate if the bctxt can be used */
 	struct blockif_ctxt *bc;
 	char ident[VIRTIO_BLK_BLK_ID_BYTES + 1];
 	struct virtio_blk_ioreq ios[VIRTIO_BLK_RINGSZ];
@@ -176,7 +187,9 @@ virtio_blk_reset(void *vdev)
 
 	DPRINTF(("virtio_blk: device reset requested !\n"));
 	virtio_reset_dev(&blk->base);
-	blockif_set_wce(blk->bc, blk->original_wce);
+	/* Reset virtio-blk device only on valid bctxt*/
+	if (!blk->dummy_bctxt)
+		blockif_set_wce(blk->bc, blk->original_wce);
 }
 
 static void
@@ -250,6 +263,12 @@ virtio_blk_proc(struct virtio_blk *blk, struct virtio_vq_info *vq)
 	writeop = ((type == VBH_OP_WRITE) ||
 			(type == VBH_OP_DISCARD));
 
+	if (blk->dummy_bctxt) {
+		WPRINTF(("Block context invalid: Operation cannot be permitted!\n"));
+		virtio_blk_done(&io->req, EPERM);
+		return;
+	}
+
 	if (writeop && blockif_is_ro(blk->bc)) {
 		WPRINTF(("Cannot write to a read-only storage!\n"));
 		virtio_blk_done(&io->req, EROFS);
@@ -348,19 +367,56 @@ virtio_blk_get_caps(struct virtio_blk *blk, bool wb)
 	return caps;
 }
 
+static void
+virtio_blk_update_config_space(struct virtio_blk *blk)
+{
+	off_t size = 0;
+	int sectsz = 0, sts = 0, sto = 0;
+
+	size = blockif_size(blk->bc);
+	sectsz = blockif_sectsz(blk->bc);
+	blockif_psectsz(blk->bc, &sts, &sto);
+
+	/* setup virtio block config space */
+	blk->cfg.capacity = size / DEV_BSIZE; /* 512-byte units */
+	blk->cfg.size_max = 0;	/* not negotiated */
+	blk->cfg.seg_max = BLOCKIF_IOV_MAX;
+	blk->cfg.geometry.cylinders = 0;	/* no geometry */
+	blk->cfg.geometry.heads = 0;
+	blk->cfg.geometry.sectors = 0;
+	blk->cfg.blk_size = sectsz;
+	blk->cfg.topology.physical_block_exp =
+	    (sts > sectsz) ? (ffsll(sts / sectsz) - 1) : 0;
+	blk->cfg.topology.alignment_offset =
+	    (sto != 0) ? ((sts - sto) / sectsz) : 0;
+	blk->cfg.topology.min_io_size = 0;
+	blk->cfg.writeback = blockif_get_wce(blk->bc);
+	blk->original_wce = blk->cfg.writeback; /* save for reset */
+	if (blockif_candiscard(blk->bc)) {
+		blk->cfg.max_discard_sectors = blockif_max_discard_sectors(blk->bc);
+		blk->cfg.max_discard_seg = blockif_max_discard_seg(blk->bc);
+		blk->cfg.discard_sector_alignment = blockif_discard_sector_alignment(blk->bc);
+	}
+	blk->base.device_caps =
+		virtio_blk_get_caps(blk, !!blk->cfg.writeback);
+}
 static int
 virtio_blk_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 {
+	bool dummy_bctxt;
 	char bident[16];
 	struct blockif_ctxt *bctxt;
 	MD5_CTX mdctx;
 	u_char digest[16];
 	struct virtio_blk *blk;
-	off_t size;
-	int i, sectsz, sts, sto;
+	int i;
 	pthread_mutexattr_t attr;
 	int rc;
 
+	bctxt = NULL;
+	/* Assume the bctxt is valid, until identified otherwise */
+	dummy_bctxt = false;
+
 	if (opts == NULL) {
 		printf("virtio_blk: backing device required\n");
 		return -1;
@@ -373,15 +429,21 @@ virtio_blk_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 				dev->slot, dev->func) >= sizeof(bident)) {
 		WPRINTF(("bident error, please check slot and func\n"));
 	}
-	bctxt = blockif_open(opts, bident);
-	if (bctxt == NULL) {
-		perror("Could not open backing file");
-		return -1;
+
+	/*
+	 * If "nodisk" keyword is found in opts, this is not a valid backend
+	 * file. Skip blockif_open and set dummy bctxt in virtio_blk struct
+	 */
+	if (strstr(opts, "nodisk") != NULL) {
+		dummy_bctxt = true;
+	} else {
+		bctxt = blockif_open(opts, bident);
+		if (bctxt == NULL) {
+			perror("Could not open backing file");
+			return -1;
+		}
 	}
 
-	size = blockif_size(bctxt);
-	sectsz = blockif_sectsz(bctxt);
-	blockif_psectsz(bctxt, &sts, &sto);
 
 	blk = calloc(1, sizeof(struct virtio_blk));
 	if (!blk) {
@@ -390,6 +452,9 @@ virtio_blk_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 	}
 
 	blk->bc = bctxt;
+	/* Update virtio-blk device struct of dummy ctxt*/
+	blk->dummy_bctxt = dummy_bctxt;
+
 	for (i = 0; i < VIRTIO_BLK_RINGSZ; i++) {
 		struct virtio_blk_ioreq *io = &blk->ios[i];
 
@@ -434,29 +499,9 @@ virtio_blk_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 		WPRINTF(("virtio_blk: block ident too long\n"));
 	}
 
-	/* setup virtio block config space */
-	blk->cfg.capacity = size / DEV_BSIZE; /* 512-byte units */
-	blk->cfg.size_max = 0;	/* not negotiated */
-	blk->cfg.seg_max = BLOCKIF_IOV_MAX;
-	blk->cfg.geometry.cylinders = 0;	/* no geometry */
-	blk->cfg.geometry.heads = 0;
-	blk->cfg.geometry.sectors = 0;
-	blk->cfg.blk_size = sectsz;
-	blk->cfg.topology.physical_block_exp =
-	    (sts > sectsz) ? (ffsll(sts / sectsz) - 1) : 0;
-	blk->cfg.topology.alignment_offset =
-	    (sto != 0) ? ((sts - sto) / sectsz) : 0;
-	blk->cfg.topology.min_io_size = 0;
-	blk->cfg.topology.opt_io_size = 0;
-	blk->cfg.writeback = blockif_get_wce(blk->bc);
-	blk->original_wce = blk->cfg.writeback; /* save for reset */
-	if (blockif_candiscard(blk->bc)) {
-		blk->cfg.max_discard_sectors = blockif_max_discard_sectors(blk->bc);
-		blk->cfg.max_discard_seg = blockif_max_discard_seg(blk->bc);
-		blk->cfg.discard_sector_alignment = blockif_discard_sector_alignment(blk->bc);
-	}
-	blk->base.device_caps =
-		virtio_blk_get_caps(blk, !!blk->cfg.writeback);
+	/* Setup virtio block config space only for valid backend file*/
+	if (!blk->dummy_bctxt)
+		virtio_blk_update_config_space(blk);
 
 	/*
 	 * Should we move some of this into virtio.c?  Could
@@ -470,11 +515,25 @@ virtio_blk_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 	pci_set_cfgdata16(dev, PCIR_SUBVEND_0, VIRTIO_VENDOR);
 
 	if (virtio_interrupt_init(&blk->base, virtio_uses_msix())) {
-		blockif_close(blk->bc);
+		/* call close only for valid bctxt */
+		if (!blk->dummy_bctxt)
+			blockif_close(blk->bc);
 		free(blk);
 		return -1;
 	}
 	virtio_set_io_bar(&blk->base, 0);
+
+	/*
+	 * Register ops for virtio-blk Rescan
+	 */
+	if (register_vm_monitor_blkrescan == false) {
+
+		register_vm_monitor_blkrescan = true;
+		if (monitor_register_vm_ops(&virtio_blk_rescan_ops, ctx,
+						"virtio_blk_rescan") < 0)
+			fprintf(stderr, "Rescan registration to VM monitor failed\n");
+	}
+
 	return 0;
 }
 
@@ -487,11 +546,13 @@ virtio_blk_deinit(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 	if (dev->arg) {
 		DPRINTF(("virtio_blk: deinit\n"));
 		blk = (struct virtio_blk *) dev->arg;
-		bctxt = blk->bc;
-		if (blockif_flush_all(bctxt))
-			WPRINTF(("vrito_blk:"
-				"Failed to flush before close\n"));
-		blockif_close(bctxt);
+		/* De-init virtio-blk device only on valid bctxt*/
+		if (!blk->dummy_bctxt) {
+			bctxt = blk->bc;
+			if (blockif_flush_all(bctxt))
+				WPRINTF(("vrito_blk: Failed to flush before close\n"));
+			blockif_close(bctxt);
+		}
 		free(blk);
 	}
 }
@@ -508,7 +569,9 @@ virtio_blk_cfgwrite(void *vdev, int offset, int size, uint32_t value)
 	if ((offset == offsetof(struct virtio_blk_config, writeback))
 		&& (size == 1)) {
 		memcpy(ptr, &value, size);
-		blockif_set_wce(blk->bc, blkcfg->writeback);
+		/* Update write cache enable only on valid bctxt*/
+		if (!blk->dummy_bctxt)
+			blockif_set_wce(blk->bc, blkcfg->writeback);
 		if (blkcfg->writeback)
 			blk->base.device_caps |= VIRTIO_BLK_F_FLUSH;
 		else
@@ -532,6 +595,134 @@ virtio_blk_cfgread(void *vdev, int offset, int size, uint32_t *retval)
 	return 0;
 }
 
+/*
+ * The following operations are done as part of blk rescan,
+ * 1. Update the backing file for the virtio-blk device,
+ *    with valid file. Basically update the empty file (with dummy bctxt)
+ *    that was passed during VM launch.
+ * 2. Update virtio-blk device configurations that were ignored for dummy
+ *    backing file (dummy bctxt).
+ * 3. Update size associated with valid backing file in the config space.
+ * 4. Notify guest OS, of the new config change.
+ * 5. On this notification, guest will do the following.
+ *	(i). Update virtio-blk capacity.
+ *	(ii).Revalidate the disk.
+ *	(iii). Identify the newly plugged block device.
+ */
+
+/* Initiate Rescan of virtio-blk device */
+int
+virtio_blk_rescan(struct vmctx *ctx, struct pci_vdev *dev, char *newpath)
+{
+	int error = -1;
+	char bident[16];
+	struct blockif_ctxt *bctxt;
+	struct virtio_blk *blk = (struct virtio_blk *) dev->arg;
+
+	if (!blk) {
+		fprintf(stderr, "Invalid virtio_blk device!\n");
+		goto end;
+	}
+
+	/* validate inputs for virtio-blk blockrescan */
+	if (newpath == NULL) {
+		fprintf(stderr, "no path info available\n");
+		goto end;
+	}
+
+	if (strstr(newpath, "nodisk") != NULL) {
+		fprintf(stderr, "no valid backend file found\n");
+		goto end;
+	}
+
+	if (snprintf(bident, sizeof(bident), "%d:%d",
+				dev->slot, dev->func) >= sizeof(bident)) {
+		fprintf(stderr, "bident error, please check slot and func\n");
+		goto end;
+	}
+
+	/* If bctxt is valid, then return error. Current support is only when
+	 * user has passed empty file during VM launch and wants to update it.
+	 * If this is the case, blk->bc would be null.
+	 */
+	if (blk->bc) {
+		fprintf(stderr, "Replacing valid backend file not supported!\n");
+		goto end;
+	}
+
+	fprintf(stderr, "name=%s, Path=%s, ident=%s\n", dev->name, newpath, bident);
+	/* update the bctxt for the virtio-blk device */
+	bctxt = blockif_open(newpath, bident);
+	if (bctxt == NULL) {
+		fprintf(stderr, "Error opening backing file\n");
+		goto end;
+	}
+
+	blk->bc = bctxt;
+	blk->dummy_bctxt = false;
+
+	/* Update virtio-blk device configuration on valid file*/
+	virtio_blk_update_config_space(blk);
+
+	/* Notify guest of config change */
+	virtio_config_changed(dev->arg);
+
+	error = 0;
+end:
+	return error;
+}
+
+int
+vm_monitor_blkrescan(void *arg, char *devargs)
+{
+	char *str;
+	char *str_slot, *str_newpath;
+	int slot;
+	int error = 0;
+	struct pci_vdev *dev;
+	struct vmctx *ctx = (struct vmctx *)arg;
+
+	/*Extract slot,path and additional params from args*/
+	str = strdup(devargs);
+
+	str_slot = strsep(&str, ",");
+	str_newpath = strsep(&str, "");
+
+	if ((str_slot != NULL) && (str_newpath != NULL)) {
+		error = dm_strtoi(str_slot, &str_slot, 10, &slot);
+		if (error) {
+			fprintf(stderr, "Incorrect slot, error=0x%x!\n", error);
+			goto end;
+		}
+
+	} else {
+		fprintf(stderr, "Slot info or path not available!");
+		error = -1;
+		goto end;
+	}
+
+	dev = pci_get_vdev_info(slot);
+	if (dev == NULL) {
+		fprintf(stderr, "vdev info failed for Slot %d\n!", slot);
+		error = -1;
+		goto end;
+	}
+
+	if (strstr(dev->name, "virtio-blk") == NULL) {
+		error = -1;
+		fprintf(stderr, "virtio-blk only supports rescan: found %s at slot %d\n", dev->name, slot);
+	} else {
+		error = virtio_blk_rescan(ctx, dev, str_newpath);
+		if (error) {
+			fprintf(stderr, "virtio-blk rescan failed!");
+		}
+	}
+end:
+	if (str)
+		free(str);
+	return error;
+}
+
 struct pci_vdev_ops pci_ops_virtio_blk = {
 	.class_name	= "virtio-blk",
 	.vdev_init	= virtio_blk_init,

devicemodel/include/monitor.h

@@ -25,6 +25,7 @@ struct monitor_vm_ops {
 	int (*pause) (void *arg);
 	int (*unpause) (void *arg);
 	int (*query) (void *arg);
+	int (*rescan)(void *arg, char *devargs);
 };
 
 int monitor_register_vm_ops(struct monitor_vm_ops *ops, void *arg,
@@ -34,4 +35,5 @@ int monitor_register_vm_ops(struct monitor_vm_ops *ops, void *arg,
 unsigned get_wakeup_reason(void);
 int set_wakeup_timer(time_t t);
 int acrn_parse_intr_monitor(const char *opt);
+int vm_monitor_blkrescan(void *arg, char *devargs);
 #endif

devicemodel/include/pci_core.h

@@ -321,6 +321,7 @@ int	check_gsi_sharing_violation(void);
 int	pciaccess_init(void);
 void	pciaccess_cleanup(void);
 int	parse_bdf(char *s, int *bus, int *dev, int *func, int base);
+struct pci_vdev *pci_get_vdev_info(int slot);
 
 
 /**