dm: enhence the mevent API

There is one race issue between mevent callback (which is called
in mevent_dispatch thread) and mevent_delete (which could be called
in dev thread). And the callback is called after mevent_delete.

libevent have the exactly same issue. The issue is decripted here:
https://github.com/libevent/libevent/blob/master/whatsnew-2.1.txt

The fixing is:
We introduce a teardown callback to mevent and make sure there is
no race issue between callback and teardown call.

This patch updates the mevent API and the caller as well.

Tracked-On: #1877
Signed-off-by: Yin Fengwei <fengwei.yin@intel.com>
Acked-by: Yu Wang <yu1.wang@intel.com>

devicemodel/arch/x86/pm.c

@@ -296,7 +296,7 @@ smi_cmd_handler(struct vmctx *ctx, int vcpu, int in, int port, int bytes,
 		pm1_control |= PM1_SCI_EN;
 		if (power_button == NULL) {
 			power_button = mevent_add(SIGTERM, EVF_SIGNAL,
-			    power_button_handler, ctx);
+				power_button_handler, ctx, NULL, NULL);
 			old_power_handler = signal(SIGTERM, SIG_IGN);
 		}
 		break;

devicemodel/core/mevent.c

@@ -56,10 +56,13 @@ static int mevent_pipefd[2];
 static pthread_mutex_t mevent_lmutex = PTHREAD_MUTEX_INITIALIZER;
 
 struct mevent {
-	void			(*me_func)(int, enum ev_type, void *);
+	void			(*run)(int, enum ev_type, void *);
+	void			*run_param;
+	void			(*teardown)(void *);
+	void			*teardown_param;
+
 	int			me_fd;
 	enum			ev_type me_type;
-	void			*me_param;
 	int			me_cq;
 	int			me_state;
 
@@ -114,7 +117,7 @@ mevent_notify(void)
 	 * If calling from outside the i/o thread, write a byte on the
 	 * pipe to force the i/o thread to exit the blocking epoll call.
 	 */
-	if (mevent_pipefd[1] != 0 && pthread_self() != mevent_tid)
+	if (mevent_pipefd[1] != 0 && !is_dispatch_thread())
 		if (write(mevent_pipefd[1], &c, 1) <= 0)
 			return -1;
 	return 0;
@@ -159,6 +162,9 @@ mevent_destroy(void)
                     mevp->me_fd != STDIN_FILENO)
                        close(mevp->me_fd);
 
+		if (mevp->teardown)
+			mevp->teardown(mevp->teardown_param);
+
 		free(mevp);
 	}
 
@@ -175,6 +181,9 @@ mevent_destroy(void)
                     mevp->me_fd != STDIN_FILENO)
                        close(mevp->me_fd);
 
+		if (mevp->teardown)
+			mevp->teardown(mevp->teardown_param);
+
 		free(mevp);
 	}
 	mevent_qunlock();
@@ -190,19 +199,20 @@ mevent_handle(struct epoll_event *kev, int numev)
 		mevp = kev[i].data.ptr;
 
 		if (mevp->me_state)
-			(*mevp->me_func)(mevp->me_fd, mevp->me_type, mevp->me_param);
+			(*mevp->run)(mevp->me_fd, mevp->me_type, mevp->run_param);
 	}
 }
 
 struct mevent *
 mevent_add(int tfd, enum ev_type type,
-	   void (*func)(int, enum ev_type, void *), void *param)
+	   void (*run)(int, enum ev_type, void *), void *run_param,
+	   void (*teardown)(void *), void *teardown_param)
 {
 	int ret;
 	struct epoll_event ee;
 	struct mevent *lp, *mevp;
 
-	if (tfd < 0 || func == NULL)
+	if (tfd < 0 || run == NULL)
 		return NULL;
 
 	if (type == EVF_TIMER)
@@ -227,10 +237,13 @@ mevent_add(int tfd, enum ev_type type,
 
 	mevp->me_fd = tfd;
 	mevp->me_type = type;
-	mevp->me_func = func;
-	mevp->me_param = param;
 	mevp->me_state = 1;
 
+	mevp->run = run;
+	mevp->run_param = run_param;
+	mevp->teardown = teardown;
+	mevp->teardown_param = teardown_param;
+
 	ee.events = mevent_kq_filter(mevp);
 	ee.data.ptr = mevp;
 	ret = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, mevp->me_fd, &ee);
@@ -241,6 +254,8 @@ mevent_add(int tfd, enum ev_type type,
 
 		return mevp;
 	} else {
+		if (mevp->teardown)
+			mevp->teardown(mevp->teardown_param);
 		free(mevp);
 		return NULL;
 	}
@@ -308,6 +323,9 @@ mevent_drain_del_list(void)
 		if (evp->closefd) {
 			close(evp->me_fd);
 		}
+
+		if (evp->teardown)
+			evp->teardown(evp->teardown_param);
 		free(evp);
 	}
 	mevent_qunlock();
@@ -323,7 +341,7 @@ mevent_delete_event(struct mevent *evp, int closefd)
 	evp->closefd = closefd;
 
 	epoll_ctl(epoll_fd, EPOLL_CTL_DEL, evp->me_fd, NULL);
-	if (!is_dispatch_thread()) {
+	if (!is_dispatch_thread() && evp->teardown != NULL) {
 		mevent_add_to_del_list(evp, closefd);
 	} else {
 		if (evp->closefd) {
@@ -398,7 +416,7 @@ mevent_dispatch(void)
 	/*
 	 * Add internal event handler for the pipe write fd
 	 */
-	pipev = mevent_add(mevent_pipefd[0], EVF_READ, mevent_pipe_read, NULL);
+	pipev = mevent_add(mevent_pipefd[0], EVF_READ, mevent_pipe_read, NULL, NULL, NULL);
 	assert(pipev != NULL);
 
 	for (;;) {

devicemodel/core/timer.c

@@ -71,7 +71,7 @@ acrn_timer_init(struct acrn_timer *timer, void (*cb)(void *), void *param)
 		return -1;
 	}
 
-	timer->mevp = mevent_add(timer->fd, EVF_READ, timer_handler, timer);
+	timer->mevp = mevent_add(timer->fd, EVF_READ, timer_handler, timer, NULL, NULL);
 	if (timer->mevp == NULL) {
 		close(timer->fd);
 		perror("acrn_timer mevent add failed.\n");

devicemodel/hw/pci/virtio/vhost.c

@@ -283,7 +283,7 @@ vhost_vq_register_eventfd(struct vhost_dev *vdev,
 		 */
 		if (is_register) {
 			vq->mevp = mevent_add(vq->call_fd, EVF_READ,
-				vhost_vq_notify, vq);
+				vhost_vq_notify, vq, NULL, NULL);
 			if (!vq->mevp) {
 				WPRINTF("mevent_add failed\n");
 				rc = -1;

devicemodel/hw/pci/virtio/virtio_console.c

@@ -679,7 +679,7 @@ virtio_console_add_backend(struct virtio_console *console,
 	if (virtio_console_backend_can_read(be_type)) {
 		if (isatty(fd)) {
 			be->evp = mevent_add(fd, EVF_READ,
-					virtio_console_backend_read, be);
+				virtio_console_backend_read, be, NULL, NULL);
 			if (be->evp == NULL) {
 				WPRINTF(("vtcon: mevent_add failed\n"));
 				error = -1;

devicemodel/hw/pci/virtio/virtio_input.c

@@ -634,7 +634,7 @@ virtio_input_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts)
 		goto fail;
 	}
 
-	vi->mevp = mevent_add(vi->fd, EVF_READ, virtio_input_read_event, vi);
+	vi->mevp = mevent_add(vi->fd, EVF_READ, virtio_input_read_event, vi, NULL, NULL);
 	if (vi->mevp == NULL) {
 		WPRINTF(("vtinput: could not register event\n"));
 		goto fail;

devicemodel/hw/pci/virtio/virtio_mei.c

@@ -1017,7 +1017,7 @@ vmei_host_client_native_connect(struct vmei_host_client *hclient)
 
 	/* add READ event into mevent */
 	hclient->rx_mevp = mevent_add(hclient->client_fd, EVF_READ,
-				      vmei_rx_callback, hclient);
+				      vmei_rx_callback, hclient, NULL, NULL);
 	if (!hclient->rx_mevp)
 		return MEI_HBM_REJECTED;
 
@@ -1951,7 +1951,7 @@ vmei_start(struct virtio_mei *vmei, bool do_rescan)
 
 	hclient->client_fd = pipefd[0];
 	hclient->rx_mevp = mevent_add(hclient->client_fd, EVF_READ,
-				      vmei_rx_callback, hclient);
+				      vmei_rx_callback, hclient, NULL, NULL);
 	vmei->hbm_fd = pipefd[1];
 
 	if (do_rescan) {
@@ -2051,7 +2051,7 @@ static int vmei_add_reset_event(struct virtio_mei *vmei)
 
 	vmei->dev_state_first = true;
 	vmei->reset_mevp = mevent_add(dev_state_fd, EVF_READ_ET,
-				      vmei_reset_callback, vmei);
+		vmei_reset_callback, vmei, NULL, NULL);
 	if (!vmei->reset_mevp) {
 		close(dev_state_fd);
 		return -ENOMEM;

devicemodel/hw/pci/virtio/virtio_net.c

@@ -707,7 +707,7 @@ virtio_net_tap_setup(struct virtio_net *net, char *devname)
 
 	if (vhost_fd < 0) {
 		net->mevp = mevent_add(net->tapfd, EVF_READ,
-				       virtio_net_rx_callback, net);
+				       virtio_net_rx_callback, net, NULL, NULL);
 		if (net->mevp == NULL) {
 			WPRINTF(("Could not register event\n"));
 			close(net->tapfd);

devicemodel/hw/uart_core.c

@@ -270,8 +270,7 @@ uart_opentty(struct uart_vdev *uart)
 {
 	ttyopen(&uart->tty);
 	if (isatty(uart->tty.fd_in)) {
-		uart->mev = mevent_add(uart->tty.fd_in, EVF_READ,
+		uart->mev = mevent_add(uart->tty.fd_in, EVF_READ, uart_drain, uart, NULL, NULL);
-			uart_drain, uart);
 		assert(uart->mev != NULL);
 	}
 }

devicemodel/include/mevent.h

@@ -41,8 +41,8 @@ enum ev_type {
 struct mevent;
 
 struct mevent *mevent_add(int fd, enum ev_type type,
-			  void (*func)(int, enum ev_type, void *),
+			  void (*run)(int, enum ev_type, void *), void *param,
-			  void *param);
+			  void (*teardown)(void *), void *teardown_param);
 int	mevent_enable(struct mevent *evp);
 int	mevent_disable(struct mevent *evp);
 int	mevent_delete(struct mevent *evp);