dm: virtio poll mode support for RT

Device trap has great impact on latency of real time (RT) tasks.
This patch provide a virtio poll mode to avoid trap.

According to the virtio spec, backend devices can declare the
notification is not needed so that frontend will never trap.
This means the backends make commitment to the frontends they have a
poll mechanism which don’t need any frontends notification.

This patch uses a periodic timer to give backends pseudo notifications
so that drive them processing data in their virtqueues. People should
choose a appropriate notification peroid interval to use this poll
mode. Too big interval may cause virtqueue processing latency while
too small interval may cause high SOS CPU usage. The suggested interval
is between 100us to 1ms.

The poll mode is not enabled by default and traditional trap
notification mode will be used. To use poll mode for RT with interval
1ms. You can add following acrn-dm parameter.

	--virtio_poll 1000000

Tracked-On: #1956
Signed-off-by: Jie Deng <jie.deng@intel.com>
Acked-by: Yu Wang <yu1.wang@intel.com>

devicemodel/hw/pci/virtio/virtio.c

@@ -28,10 +28,13 @@
 #include <stdio.h>
 #include <stddef.h>
 #include <pthread.h>
+#include <string.h>
+#include <stdlib.h>
 
 #include "dm.h"
 #include "pci_core.h"
 #include "virtio.h"
+#include "timer.h"
 
 /*
  * Functions for dealing with generalized "virtual devices" as
@@ -45,6 +48,61 @@
  */
 #define DEV_STRUCT(vs) ((void *)(vs))
 
+static uint8_t virtio_poll_enabled;
+static size_t virtio_poll_interval;
+
+static void
+virtio_start_timer(struct acrn_timer *timer, time_t sec, time_t nsec)
+{
+	struct itimerspec ts;
+
+	/* setting the interval time */
+	ts.it_interval.tv_sec = 0;
+	ts.it_interval.tv_nsec = 0;
+	/* set the delay time it will be started when timer_setting */
+	ts.it_value.tv_sec = sec;
+	ts.it_value.tv_nsec = nsec;
+	assert(acrn_timer_settime(timer, &ts) == 0);
+}
+
+static void
+virtio_poll_timer(void *arg)
+{
+	struct virtio_base *base;
+	struct virtio_ops *vops;
+	struct virtio_vq_info *vq;
+	const char *name;
+	int i;
+
+	base = arg;
+	vops = base->vops;
+	name = vops->name;
+
+	if (base->mtx)
+		pthread_mutex_lock(base->mtx);
+
+	base->polling_in_progress = 1;
+
+	for (i = 0; i < base->vops->nvq; i++) {
+		vq = &base->queues[i];
+		vq->used->flags |= ACRN_VRING_USED_F_NO_NOTIFY;
+		/* TODO: call notify when necessary */
+		if (vq->notify)
+			(*vq->notify)(DEV_STRUCT(base), vq);
+		else if (vops->qnotify)
+			(*vops->qnotify)(DEV_STRUCT(base), vq);
+		else
+			fprintf(stderr,
+				"%s: qnotify queue %d: missing vq/vops notify\r\n",
+				name, i);
+	}
+
+	if (base->mtx)
+		pthread_mutex_unlock(base->mtx);
+
+	virtio_start_timer(&base->polling_timer, 0, virtio_poll_interval);
+}
+
 /**
  * @brief Link a virtio_base to its constants, the virtio device,
  * and the PCI emulation.
@@ -60,7 +118,8 @@
 void
 virtio_linkup(struct virtio_base *base, struct virtio_ops *vops,
 	      void *pci_virtio_dev, struct pci_vdev *dev,
-	      struct virtio_vq_info *queues)
+	      struct virtio_vq_info *queues,
+	      int backend_type)
 {
 	int i;
 
@@ -69,6 +128,7 @@ virtio_linkup(struct virtio_base *base, struct virtio_ops *vops,
 	base->vops = vops;
 	base->dev = dev;
 	dev->arg = base;
+	base->backend_type = backend_type;
 
 	base->queues = queues;
 	for (i = 0; i < vops->nvq; i++) {
@@ -100,6 +160,9 @@ virtio_reset_dev(struct virtio_base *base)
 /* if (base->mtx) */
 /* assert(pthread_mutex_isowned_np(base->mtx)); */
 
+	acrn_timer_deinit(&base->polling_timer);
+	base->polling_in_progress = 0;
+
 	nvq = base->vops->nvq;
 	for (vq = base->queues, i = 0; i < nvq; vq++, i++) {
 		vq->flags = 0;
@@ -570,6 +633,30 @@ vq_endchains(struct virtio_vq_info *vq, int used_all_avail)
 		vq_interrupt(base, vq);
 }
 
+/**
+ * @brief Helper function for clearing used ring flags.
+ *
+ * Driver should always use this helper function to clear used ring flags.
+ * For virtio poll mode, in order to avoid trap, we should never really
+ * clear used ring flags.
+ *
+ * @param base Pointer to struct virtio_base.
+ * @param vq Pointer to struct virtio_vq_info.
+ *
+ * @return None
+ */
+void vq_clear_used_ring_flags(struct virtio_base *base, struct virtio_vq_info *vq)
+{
+	int backend_type = base->backend_type;
+	int polling_in_progress = base->polling_in_progress;
+
+	/* we should never unmask notification in polling mode */
+	if (virtio_poll_enabled && backend_type == BACKEND_VBSU && polling_in_progress == 1)
+		return;
+
+	vq->used->flags &= ~ACRN_VRING_USED_F_NO_NOTIFY;
+}
+
 struct config_reg {
 	uint16_t	offset;	/* register offset */
 	uint8_t		size;	/* size (bytes) */
@@ -877,6 +964,17 @@ bad:
 			(*vops->set_status)(DEV_STRUCT(base), value);
 		if (value == 0)
 			(*vops->reset)(DEV_STRUCT(base));
+		if ((value & VIRTIO_CR_STATUS_DRIVER_OK) &&
+		     base->backend_type == BACKEND_VBSU &&
+		     virtio_poll_enabled) {
+			base->polling_timer.clockid = CLOCK_MONOTONIC;
+			acrn_timer_init(&base->polling_timer, virtio_poll_timer, base);
+			/* wait 5s to start virtio poll mode
+			 * skip vsbl and make sure device initialization completed
+			 * FIXME: Need optimization in the future
+			 */
+			virtio_start_timer(&base->polling_timer, 5, 0);
+		}
 		break;
 	case VIRTIO_CR_CFGVEC:
 		base->msix_cfg_idx = value;
@@ -1302,6 +1400,7 @@ virtio_common_cfg_write(struct pci_vdev *dev, uint64_t offset, int size,
 			(*vops->set_status)(DEV_STRUCT(base), value);
 		if (base->status == 0)
 			(*vops->reset)(DEV_STRUCT(base));
+		/* TODO: virtio poll mode for modern devices */
 		break;
 	case VIRTIO_COMMON_Q_SELECT:
 		/*
@@ -1878,3 +1977,26 @@ virtio_pci_modern_cfgwrite(struct vmctx *ctx, int vcpu, struct pci_vdev *dev,
 
 	return -1;
 }
+
+/**
+ * @brief Get the virtio poll parameters
+ *
+ * @param optarg Pointer to parameters string.
+ *
+ * @return fail -1 success 0
+ */
+int
+acrn_parse_virtio_poll_interval(const char *optarg)
+{
+	char *ptr;
+
+	virtio_poll_interval = strtoul(optarg, &ptr, 0);
+
+	/* poll interval is limited from 1us to 10ms */
+	if (virtio_poll_interval < 1 || virtio_poll_interval > 10000000)
+		return -1;
+
+	virtio_poll_enabled = 1;
+
+	return 0;
+}

devicemodel/hw/pci/virtio/virtio_audio.c

@@ -316,7 +316,8 @@ virtio_audio_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 		      &virtio_audio_ops_k,
 		      virt_audio,
 		      dev,
-		      virt_audio->vq);
+		      virt_audio->vq,
+		      BACKEND_VBSK);
 
 	rc = virtio_audio_kernel_init(virt_audio);
 	if (rc < 0) {

devicemodel/hw/pci/virtio/virtio_block.c

@@ -384,7 +384,7 @@ virtio_blk_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 					"error %d!\n", rc));
 
 	/* init virtio struct and virtqueues */
-	virtio_linkup(&blk->base, &virtio_blk_ops, blk, dev, &blk->vq);
+	virtio_linkup(&blk->base, &virtio_blk_ops, blk, dev, &blk->vq, BACKEND_VBSU);
 	blk->base.mtx = &blk->mtx;
 
 	blk->vq.qsize = VIRTIO_BLK_RINGSZ;

devicemodel/hw/pci/virtio/virtio_console.c

@@ -828,7 +828,7 @@ virtio_console_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 					"error %d!\n", rc));
 
 	virtio_linkup(&console->base, &virtio_console_ops, console, dev,
-		console->queues);
+		console->queues, BACKEND_VBSU);
 	console->base.mtx = &console->mtx;
 	console->base.device_caps = VIRTIO_CONSOLE_S_HOSTCAPS;
 

devicemodel/hw/pci/virtio/virtio_coreu.c

@@ -281,7 +281,7 @@ virtio_coreu_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 
 	DPRINTF(("vcoreu init: using VBS-U...\n"));
 	virtio_linkup(&vcoreu->base, &virtio_coreu_ops,
-			vcoreu, dev, vcoreu->queues);
+			vcoreu, dev, vcoreu->queues, BACKEND_VBSU);
 	vcoreu->base.mtx = &vcoreu->mtx;
 
 	vcoreu->queues[0].qsize  = VIRTIO_COREU_RINGSZ;

devicemodel/hw/pci/virtio/virtio_hdcp.c

@@ -413,7 +413,7 @@ virtio_hdcp_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 
 	DPRINTF(("vhdcp init: using VBS-U...\n"));
 	virtio_linkup(&vhdcp->base, &virtio_hdcp_ops,
-			vhdcp, dev, vhdcp->queues);
+			vhdcp, dev, vhdcp->queues, BACKEND_VBSU);
 	vhdcp->base.mtx = &vhdcp->mtx;
 
 	vhdcp->queues[0].qsize  = VIRTIO_HDCP_RINGSZ;

devicemodel/hw/pci/virtio/virtio_hyper_dmabuf.c

@@ -299,7 +299,8 @@ virtio_hyper_dmabuf_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 		      &virtio_hyper_dmabuf_ops_k,
 		      hyper_dmabuf,
 		      dev,
-		      hyper_dmabuf->vq);
+		      hyper_dmabuf->vq,
+		      BACKEND_VBSK);
 
 	rc = virtio_hyper_dmabuf_k_init();
 	if (rc < 0) {

devicemodel/hw/pci/virtio/virtio_input.c

@@ -640,7 +640,7 @@ virtio_input_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 		goto fail;
 	}
 
-	virtio_linkup(&vi->base, &virtio_input_ops, vi, dev, vi->queues);
+	virtio_linkup(&vi->base, &virtio_input_ops, vi, dev, vi->queues, BACKEND_VBSU);
 	vi->base.mtx = &vi->mtx;
 	vi->base.device_caps = VIRTIO_INPUT_S_HOSTCAPS;
 

devicemodel/hw/pci/virtio/virtio_ipu.c

@@ -320,7 +320,8 @@ virtio_ipu_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 		      &virtio_ipu_ops_k,
 		      ipu,
 		      dev,
-		      ipu->vq);
+		      ipu->vq,
+		      BACKEND_VBSK);
 
 	rc = virtio_ipu_k_init(ipu);
 	if (rc < 0) {

devicemodel/hw/pci/virtio/virtio_mei.c

@@ -1577,7 +1577,7 @@ vmei_notify_tx(void *data, struct virtio_vq_info *vq)
 
 	pthread_mutex_lock(&vmei->tx_mutex);
 	DPRINTF("TX: New OUT buffer available!\n");
-	vq->used->flags &= ~ACRN_VRING_USED_F_NO_NOTIFY;
+	vq_clear_used_ring_flags(&vmei->base, vq);
 	pthread_mutex_unlock(&vmei->tx_mutex);
 }
 
@@ -1870,7 +1870,7 @@ static void *vmei_rx_thread(void *param)
 	while (vmei->status != VMEI_STST_DEINIT) {
 		/* note - rx mutex is locked here */
 		while (vq_ring_ready(vq)) {
-			vq->used->flags &= ~ACRN_VRING_USED_F_NO_NOTIFY;
+			vq_clear_used_ring_flags(&vmei->base, vq);
 			mb();
 			if (vq_has_descs(vq) &&
 			    vmei->rx_need_sched &&
@@ -2255,7 +2255,7 @@ init:
 		goto fail;
 	}
 
-	virtio_linkup(&vmei->base, &virtio_mei_ops, vmei, dev, vmei->vqs);
+	virtio_linkup(&vmei->base, &virtio_mei_ops, vmei, dev, vmei->vqs, BACKEND_VBSU);
 	vmei->base.mtx = &vmei->mutex;
 
 	for (i = 0; i < VMEI_VQ_NUM; i++)

devicemodel/hw/pci/virtio/virtio_net.c

@@ -555,7 +555,7 @@ virtio_net_tx_thread(void *param)
 	for (;;) {
 		/* note - tx mutex is locked here */
 		while (net->resetting || !vq_has_descs(vq)) {
-			vq->used->flags &= ~ACRN_VRING_USED_F_NO_NOTIFY;
+			vq_clear_used_ring_flags(&net->base, vq);
 			/* memory barrier */
 			mb();
 			if (!net->resetting && vq_has_descs(vq))
@@ -724,7 +724,7 @@ virtio_net_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 	char nstr[80];
 	char tname[MAXCOMLEN + 1];
 	struct virtio_net *net;
-	char *devname;
+	char *devname = NULL;
 	char *vtopts;
 	char *opt;
 	int mac_provided;
@@ -751,25 +751,10 @@ virtio_net_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 		DPRINTF(("virtio_net: pthread_mutex_init failed with "
 			"error %d!\n", rc));
 
-	virtio_linkup(&net->base, &virtio_net_ops, net, dev, net->queues);
-	net->base.mtx = &net->mtx;
-	net->base.device_caps = VIRTIO_NET_S_HOSTCAPS;
-
-	net->queues[VIRTIO_NET_RXQ].qsize = VIRTIO_NET_RINGSZ;
-	net->queues[VIRTIO_NET_RXQ].notify = virtio_net_ping_rxq;
-	net->queues[VIRTIO_NET_TXQ].qsize = VIRTIO_NET_RINGSZ;
-	net->queues[VIRTIO_NET_TXQ].notify = virtio_net_ping_txq;
-#ifdef notyet
-	net->queues[VIRTIO_NET_CTLQ].qsize = VIRTIO_NET_RINGSZ;
-	net->queues[VIRTIO_NET_CTLQ].notify = virtio_net_ping_ctlq;
-#endif
-
 	/*
-	 * Attempt to open the tap device and read the MAC address
-	 * if specified
+	 * Read the MAC address if specified
 	 */
 	mac_provided = 0;
-	net->tapfd = -1;
 	net->vhost_net = NULL;
 	if (opts != NULL) {
 		int err;
@@ -795,14 +780,38 @@ virtio_net_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 				mac_provided = 1;
 			}
 		}
-
-		if (strncmp(devname, "tap", 3) == 0 ||
-		    strncmp(devname, "vmnet", 5) == 0)
-			virtio_net_tap_setup(net, devname);
-
-		free(devname);
 	}
 
+	virtio_linkup(&net->base, &virtio_net_ops, net, dev, net->queues,
+		      net->use_vhost ? BACKEND_VHOST : BACKEND_VBSU);
+	net->base.mtx = &net->mtx;
+	net->base.device_caps = VIRTIO_NET_S_HOSTCAPS;
+
+	net->queues[VIRTIO_NET_RXQ].qsize = VIRTIO_NET_RINGSZ;
+	net->queues[VIRTIO_NET_RXQ].notify = virtio_net_ping_rxq;
+	net->queues[VIRTIO_NET_TXQ].qsize = VIRTIO_NET_RINGSZ;
+	net->queues[VIRTIO_NET_TXQ].notify = virtio_net_ping_txq;
+#ifdef notyet
+	net->queues[VIRTIO_NET_CTLQ].qsize = VIRTIO_NET_RINGSZ;
+	net->queues[VIRTIO_NET_CTLQ].notify = virtio_net_ping_ctlq;
+#endif
+
+	/*
+	 * Attempt to open the tap device
+	 */
+	net->tapfd = -1;
+
+	if (!devname) {
+		WPRINTF(("virtio_net: devname NULL\n"));
+		return -1;
+	}
+
+	if (strncmp(devname, "tap", 3) == 0 ||
+	    strncmp(devname, "vmnet", 5) == 0)
+		virtio_net_tap_setup(net, devname);
+
+	free(devname);
+
 	/*
 	 * The default MAC address is the standard NetApp OUI of 00-a0-98,
 	 * followed by an MD5 of the PCI slot/func number and dev name

devicemodel/hw/pci/virtio/virtio_rnd.c

@@ -404,7 +404,7 @@ virtio_rnd_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 	if (rnd->vbs_k.status == VIRTIO_DEV_PRE_INIT) {
 		DPRINTF(("%s: VBS-K option detected!\n", __func__));
 		virtio_linkup(&rnd->base, &virtio_rnd_ops_k,
-			      rnd, dev, &rnd->vq);
+			      rnd, dev, &rnd->vq, BACKEND_VBSK);
 		rc = virtio_rnd_kernel_init(rnd);
 		if (rc < 0) {
 			WPRINTF(("virtio_rnd: VBS-K init failed,error %d!\n",
@@ -417,7 +417,7 @@ virtio_rnd_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 	if (rnd->vbs_k.status == VIRTIO_DEV_INITIAL ||
 	    rnd->vbs_k.status != VIRTIO_DEV_INIT_SUCCESS) {
 		DPRINTF(("%s: fallback to VBS-U...\n", __func__));
-		virtio_linkup(&rnd->base, &virtio_rnd_ops, rnd, dev, &rnd->vq);
+		virtio_linkup(&rnd->base, &virtio_rnd_ops, rnd, dev, &rnd->vq, BACKEND_VBSU);
 	}
 
 	rnd->base.mtx = &rnd->mtx;

devicemodel/hw/pci/virtio/virtio_rpmb.c

@@ -512,7 +512,7 @@ virtio_rpmb_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 		goto out;
 	}
 
-	virtio_linkup(&rpmb->base, &virtio_rpmb_ops, rpmb, dev, &rpmb->vq);
+	virtio_linkup(&rpmb->base, &virtio_rpmb_ops, rpmb, dev, &rpmb->vq, BACKEND_VBSU);
 
 	rpmb->base.mtx = &rpmb->mtx;
 	rpmb->vq.qsize = VIRTIO_RPMB_RINGSZ;