dm: fix the issue when guest tries to disable memory range access

According to PCI spec 3.0 section 6.2.2 "Device Control", guest
could write the command register to control device response to
io/mem access.

The origial code register/unregister the memory range which is
not suitable because it can't handle the sequence:
  1. disble the device response to specific memory range
  2. reboot guest (DM will try to free the memory range which
     was freed in step 1 already)

Tracked-On: #1277
Signed-off-by: Yin Fengwei <fengwei.yin@intel.com>
Acked-by: Yu Wang <yu1.wang@intel.com>

devicemodel/core/inout.c

@@ -26,6 +26,7 @@
  * $FreeBSD$
  */
 
+#include <stdbool.h>
 #include <stdio.h>
 #include <string.h>
 #include <assert.h>
@@ -44,6 +45,7 @@ static struct {
 	int		flags;
 	inout_func_t	handler;
 	void		*arg;
+	bool		enabled;
 } inout_handlers[MAX_IOPORTS];
 
 static int
@@ -113,6 +115,11 @@ emulate_inout(struct vmctx *ctx, int *pvcpu, struct pio_request *pio_request)
 		if (!(flags & IOPORT_F_OUT))
 			return -1;
 	}
+
+	if (inout_handlers[port].enabled == false) {
+		return -1;
+	}
+
 	retval = handler(ctx, *pvcpu, in, port, bytes,
 		(uint32_t *)&(pio_request->value), arg);
 	return retval;
@@ -141,6 +148,42 @@ init_inout(void)
 	}
 }
 
+int
+disable_inout(struct inout_port *iop)
+{
+	int i;
+
+	if (!VERIFY_IOPORT(iop->port, iop->size)) {
+		printf("invalid input: port:0x%x, size:%d",
+				iop->port, iop->size);
+		return -1;
+	}
+
+	for (i = iop->port; i < iop->port + iop->size; i++) {
+		inout_handlers[i].enabled = false;
+	}
+
+	return 0;
+}
+
+int
+enable_inout(struct inout_port *iop)
+{
+	int i;
+
+	if (!VERIFY_IOPORT(iop->port, iop->size)) {
+		printf("invalid input: port:0x%x, size:%d",
+				iop->port, iop->size);
+		return -1;
+	}
+
+	for (i = iop->port; i < iop->port + iop->size; i++) {
+		inout_handlers[i].enabled = true;
+	}
+
+	return 0;
+}
+
 int
 register_inout(struct inout_port *iop)
 {
@@ -168,6 +211,7 @@ register_inout(struct inout_port *iop)
 		inout_handlers[i].flags = iop->flags;
 		inout_handlers[i].handler = iop->handler;
 		inout_handlers[i].arg = iop->arg;
+		inout_handlers[i].enabled = true;
 	}
 
 	return 0;

devicemodel/core/mem.c

@@ -33,6 +33,7 @@
  */
 
 #include <errno.h>
+#include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <assert.h>
@@ -47,6 +48,7 @@ struct mmio_rb_range {
 	struct mem_range	mr_param;
 	uint64_t                mr_base;
 	uint64_t                mr_end;
+	bool			enabled;
 };
 
 struct mmio_rb_tree;
@@ -181,6 +183,10 @@ emulate_mem(struct vmctx *ctx, struct mmio_request *mmio_req)
 
 	assert(entry != NULL);
 
+	if (entry->enabled == false) {
+		return -1;
+	}
+
 	if (mmio_req->direction == REQUEST_READ)
 		err = mem_read(ctx, 0, paddr, (uint64_t *)&mmio_req->value,
 				size, &entry->mr_param);
@@ -207,6 +213,7 @@ register_mem_int(struct mmio_rb_tree *rbt, struct mem_range *memp)
 		mrp->mr_param = *memp;
 		mrp->mr_base = memp->base;
 		mrp->mr_end = memp->base + memp->size - 1;
+		mrp->enabled = true;
 		pthread_rwlock_wrlock(&mmio_rwlock);
 		if (mmio_rb_lookup(rbt, memp->base, &entry) != 0)
 			err = mmio_rb_add(rbt, mrp);
@@ -219,6 +226,68 @@ register_mem_int(struct mmio_rb_tree *rbt, struct mem_range *memp)
 	return err;
 }
 
+int
+disable_mem(struct mem_range *memp)
+{
+	uint64_t paddr = memp->base;
+	struct mmio_rb_range *entry = NULL;
+
+	pthread_rwlock_rdlock(&mmio_rwlock);
+	/*
+	 * First check the per-VM cache
+	 */
+	if (mmio_hint && paddr >= mmio_hint->mr_base &&
+			paddr <= mmio_hint->mr_end)
+		entry = mmio_hint;
+
+	if (entry == NULL) {
+		if (mmio_rb_lookup(&mmio_rb_root, paddr, &entry) == 0)
+			/* Update the per-VMU cache */
+			mmio_hint = entry;
+		else if (mmio_rb_lookup(&mmio_rb_fallback, paddr, &entry)) {
+			pthread_rwlock_unlock(&mmio_rwlock);
+			return -ESRCH;
+		}
+	}
+
+	assert(entry != NULL);
+	entry->enabled = false;
+	pthread_rwlock_unlock(&mmio_rwlock);
+
+	return 0;
+}
+
+int
+enable_mem(struct mem_range *memp)
+{
+	uint64_t paddr = memp->base;
+	struct mmio_rb_range *entry = NULL;
+
+	pthread_rwlock_rdlock(&mmio_rwlock);
+	/*
+	 * First check the per-VM cache
+	 */
+	if (mmio_hint && paddr >= mmio_hint->mr_base &&
+			paddr <= mmio_hint->mr_end)
+		entry = mmio_hint;
+
+	if (entry == NULL) {
+		if (mmio_rb_lookup(&mmio_rb_root, paddr, &entry) == 0)
+			/* Update the per-VMU cache */
+			mmio_hint = entry;
+		else if (mmio_rb_lookup(&mmio_rb_fallback, paddr, &entry)) {
+			pthread_rwlock_unlock(&mmio_rwlock);
+			return -ESRCH;
+		}
+	}
+
+	assert(entry != NULL);
+	entry->enabled = true;
+	pthread_rwlock_unlock(&mmio_rwlock);
+
+	return 0;
+}
+
 int
 register_mem(struct mem_range *memp)
 {

devicemodel/hw/pci/core.c

@@ -508,6 +508,66 @@ register_bar(struct pci_vdev *dev, int idx)
 	modify_bar_registration(dev, idx, 1);
 }
 
+static void
+enable_bar(struct pci_vdev *dev, int idx)
+{
+	int error = 0;
+	struct inout_port iop;
+	struct mem_range mr;
+
+	switch (dev->bar[idx].type) {
+	case PCIBAR_IO:
+		bzero(&iop, sizeof(struct inout_port));
+		iop.name = dev->name;
+		iop.port = dev->bar[idx].addr;
+		iop.size = dev->bar[idx].size;
+		error = enable_inout(&iop);
+		break;
+	case PCIBAR_MEM32:
+	case PCIBAR_MEM64:
+		bzero(&mr, sizeof(struct mem_range));
+		mr.name = dev->name;
+		mr.base = dev->bar[idx].addr;
+		mr.size = dev->bar[idx].size;
+		error = enable_mem(&mr);
+		break;
+	default:
+		error = EINVAL;
+		break;
+	}
+	assert(error == 0);
+}
+
+static void
+disable_bar(struct pci_vdev *dev, int idx)
+{
+	int error = 0;
+	struct inout_port iop;
+	struct mem_range mr;
+
+	switch (dev->bar[idx].type) {
+	case PCIBAR_IO:
+		bzero(&iop, sizeof(struct inout_port));
+		iop.name = dev->name;
+		iop.port = dev->bar[idx].addr;
+		iop.size = dev->bar[idx].size;
+		error = disable_inout(&iop);
+		break;
+	case PCIBAR_MEM32:
+	case PCIBAR_MEM64:
+		bzero(&mr, sizeof(struct mem_range));
+		mr.name = dev->name;
+		mr.base = dev->bar[idx].addr;
+		mr.size = dev->bar[idx].size;
+		error = disable_mem(&mr);
+		break;
+	default:
+		error = EINVAL;
+		break;
+	}
+	assert(error == 0);
+}
+
 /* Are we decoding i/o port accesses for the emulated pci device? */
 static int
 porten(struct pci_vdev *dev)
@@ -1875,9 +1935,9 @@ pci_emul_cmdsts_write(struct pci_vdev *dev, int coff, uint32_t new, int bytes)
 		/* I/O address space decoding changed? */
 			if (changed & PCIM_CMD_PORTEN) {
 				if (porten(dev))
-					register_bar(dev, i);
+					enable_bar(dev, i);
 				else
-					unregister_bar(dev, i);
+					disable_bar(dev, i);
 			}
 			break;
 		case PCIBAR_MEM32:
@@ -1885,9 +1945,9 @@ pci_emul_cmdsts_write(struct pci_vdev *dev, int coff, uint32_t new, int bytes)
 		/* MMIO address space decoding changed? */
 			if (changed & PCIM_CMD_MEMEN) {
 				if (memen(dev))
-					register_bar(dev, i);
+					enable_bar(dev, i);
 				else
-					unregister_bar(dev, i);
+					disable_bar(dev, i);
 				}
 			break;
 		default:

devicemodel/include/inout.h

@@ -74,6 +74,8 @@ void	init_inout(void);
 int	emulate_inout(struct vmctx *ctx, int *pvcpu, struct pio_request *req);
 int	register_inout(struct inout_port *iop);
 int	unregister_inout(struct inout_port *iop);
+int	enable_inout(struct inout_port *iop);
+int	disable_inout(struct inout_port *iop);
 int	init_bvmcons(void);
 void	deinit_bvmcons(void);
 void	enable_bvmcons(void);

devicemodel/include/mem.h

@@ -42,6 +42,7 @@ struct mem_range {
 	long		arg2;
 	uint64_t	base;
 	uint64_t	size;
+	bool		enabled;
 };
 #define	MEM_F_READ		0x1
 #define	MEM_F_WRITE		0x2
@@ -54,5 +55,7 @@ int	register_mem(struct mem_range *memp);
 int	register_mem_fallback(struct mem_range *memp);
 int	unregister_mem(struct mem_range *memp);
 int	unregister_mem_fallback(struct mem_range *memp);
+int	disable_mem(struct mem_range *memp);
+int	enable_mem(struct mem_range *memp);
 
 #endif	/* _MEM_H_ */