Attachment #210713 for bug #236922

Lines 82-87 Link Here

(-)sys/dev/virtio/network/if_vtnetvar.h (+6 lines)
82	struct taskqueue *vtnrx_tq;	82	struct taskqueue *vtnrx_tq;
83	struct task vtnrx_intrtask;	83	struct task vtnrx_intrtask;
84	struct lro_ctrl vtnrx_lro;	84	struct lro_ctrl vtnrx_lro;
		85	#ifdef DEV_NETMAP
		86	struct virtio_net_hdr_mrg_rxbuf vtnrx_shrhdr;
		87	#endif /* DEV_NETMAP */
85	char vtnrx_name[16];	88	char vtnrx_name[16];
86	} __aligned(CACHE_LINE_SIZE);	89	} __aligned(CACHE_LINE_SIZE);
87		90
Lines 118-123 Link Here
118	#ifndef VTNET_LEGACY_TX	121	#ifndef VTNET_LEGACY_TX
119	struct task vtntx_defrtask;	122	struct task vtntx_defrtask;
120	#endif	123	#endif
		124	#ifdef DEV_NETMAP
		125	struct virtio_net_hdr_mrg_rxbuf vtntx_shrhdr;
		126	#endif /* DEV_NETMAP */
121	char vtntx_name[16];	127	char vtntx_name[16];
122	} __aligned(CACHE_LINE_SIZE);	128	} __aligned(CACHE_LINE_SIZE);
123		129




		na->tx_rings[idx]->nr_mode == NKR_NETMAP_ON);
}

/* Free all the unused buffer in all the RX virtqueues.
 * This function is called when entering and exiting netmap mode.
 * - buffers queued by the virtio driver return skbuf/mbuf pointer
 *   and need to be freed;
 * - buffers queued by netmap return the txq/rxq, and do not need work
 */
static void
vtnet_netmap_free_bufs(struct vtnet_softc *sc)
{
	int i, nmb = 0, n = 0, last;

	for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
		struct vtnet_rxq *rxq = &sc->vtnet_rxqs[i];
		struct vtnet_txq *txq = &sc->vtnet_txqs[i];
		struct vtnet_tx_header *txhdr;
		struct virtqueue *vq;
		struct mbuf *m;

		last = 0;
		vq = rxq->vtnrx_vq;
		while ((m = virtqueue_drain(vq, &last)) != NULL) {
			n++;
			if (m != (void *)rxq)
				m_freem(m);
			else
				nmb++;
		}

		last = 0;
		vq = txq->vtntx_vq;
		while ((txhdr = virtqueue_drain(vq, &last)) != NULL) {
			n++;
			if (txhdr != (void *)txq) {
				m_freem(txhdr->vth_mbuf);
				uma_zfree(vtnet_tx_header_zone, txhdr);
			} else
				nmb++;
		}
	}
	D("freed %d mbufs, %d netmap bufs on %d queues",
		n - nmb, nmb, i);
}

static void
vtnet_free_used(struct virtqueue *vq, int netmap_bufs, enum txrx t, int idx)
{
	void *cookie;

{
	struct ifnet *ifp = na->ifp;
	struct vtnet_softc *sc = ifp->if_softc;
	int success;
	enum txrx t;
	int i;


	vtnet_drain_taskqueues(sc);

	VTNET_CORE_LOCK(sc);

	/* We need nm_netmap_on() to return true when called by
	 * vtnet_init_locked() below. */
	/* enable or disable flags and callbacks in na and ifp */
	if (state) {
		nm_set_native_flags(na);
	} else {
		nm_clear_native_flags(na);
	}
	/* We need to trigger a device reset in order to unexpose guest buffers
	 * published to the host. */
	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);

	/* drain queues so netmap and native drivers
	 * do not interfere with each other
	 */
//	vtnet_netmap_free_bufs(sc);
	/* Get pending used buffers. The way they are freed depends on whether
	 * they are netmap buffer or they are mbufs. We can tell apart the two
	 * cases by looking at kring->nr_mode, before this is possibly updated

				kring->nr_mode == NKR_NETMAP_ON, NR_RX, i);
		VTNET_RXQ_UNLOCK(rxq);
	}
	vtnet_init_locked(sc);       /* also enable intr */
	success = (ifp->if_drv_flags & IFF_DRV_RUNNING) ? 0 : ENXIO;

	if (state) {
		for_rx_tx(t) {
			/* Hardware rings. */

	}

	VTNET_CORE_UNLOCK(sc);
	return (ifp->if_drv_flags & IFF_DRV_RUNNING) ? 0 : ENXIO;
	return success;
}



	struct netmap_ring *ring = kring->ring;
	u_int ring_nr = kring->ring_id;
	u_int nm_i;	/* index into the netmap ring */
//	u_int nic_i;	/* index into the NIC ring */
	u_int const lim = kring->nkr_num_slots - 1;
	u_int const head = kring->rhead;


	if (nm_i != head) {	/* we have new packets to send */
		struct sglist *sg = txq->vtntx_sg;

//		nic_i = netmap_idx_k2n(kring, nm_i);
		for (; nm_i != head; nm_i = nm_next(nm_i, lim)) {
			/* we use an empty header here */
			struct netmap_slot *slot = &ring->slot[nm_i];

			 * and kick the hypervisor (if necessary).
			 */
			sglist_reset(sg); // cheap
			// if vtnet_hdr_size > 0 ...
			err = sglist_append(sg, &txq->vtntx_shrhdr, sc->vtnet_hdr_size);
			// XXX later, support multi segment
//			err = sglist_append_phys(sg, paddr, len);
			err |= sglist_append_phys(sg, paddr, len);
			KASSERT(err == 0, ("%s: cannot append to sglist %d",
						__func__, err));
			err = virtqueue_enqueue(vq, /*cookie=*/txq, sg,
						/*readable=*/sg->sg_nseg,
						/*writeable=*/0);
			/* use na as the cookie */
			err = virtqueue_enqueue(vq, txq, sg, sg->sg_nseg, 0);
			if (unlikely(err)) {
				if (err != ENOSPC)
					nm_prerr("virtqueue_enqueue(%s) failed: %d\n",
							kring->name, err);
				break;
			}
//			nic_i = nm_next(nic_i, lim);
		}

		/* No more free TX slots? Ask the hypervisor for notifications,
		 * possibly only when a considerable amount of work has been
		 * done.
		 */
		ND(3,"sent %d packets, hwcur %d", n, nm_i);
//		virtqueue_disable_intr(vq); // ??
		virtqueue_notify(vq);

		/* Update hwcur depending on where we stopped. */
		kring->nr_hwcur = nm_i; /* note we migth break early */

	} else {
		if (ring->head != ring->tail)
		    ND(5, "pure notify ? head %d tail %d nused %d %d",
			ring->head, ring->tail, virtqueue_nused(vq),
			(virtqueue_dump(vq), 1));
		virtqueue_notify(vq);
		if (interrupts) {
			virtqueue_enable_intr(vq); // like postpone with 0
		}
	}

    /* Free used slots. We only consider our own used buffers, recognized
	 * by the token we passed to virtqueue_add_outbuf.
	 */
	n = 0;
	for (;;) {
		struct vtnet_tx_header *txhdr = virtqueue_dequeue(vq, NULL);
		if (txhdr == NULL)
			break;
		if (likely(txhdr == (void *)txq)) {
			nm_prerr("BUG: TX token mismatch\n");
		else
			n++;
			if (virtqueue_nused(vq) < 32) { // XXX slow release
				break;
			}
		} else { /* leftover from previous transmission */
			nm_prerr("BUG: TX token mismatch\n");
			m_freem(txhdr->vth_mbuf);
			uma_zfree(vtnet_tx_header_zone, txhdr);
		}
	}
	if (n) {
		kring->nr_hwtail += n;
		if (kring->nr_hwtail > lim)
			kring->nr_hwtail -= lim + 1;
	}
	if (nm_i != kring->nr_hwtail /* && vtnet_txq_below_threshold(txq) == 0*/) {
		ND(3, "disable intr, hwcur %d", nm_i);
		virtqueue_disable_intr(vq);
	} else if (interrupts && virtqueue_nfree(vq) < 32) {
		ND(3, "enable intr, hwcur %d", nm_i);
		virtqueue_postpone_intr(vq, VQ_POSTPONE_SHORT);
	}

	if (interrupts && virtqueue_nfree(vq) < 32)
		virtqueue_postpone_intr(vq, VQ_POSTPONE_LONG);

	return 0;
}


		struct netmap_slot *slot = &ring->slot[nm_i];
		uint64_t paddr;
		void *addr = PNMB(na, slot, &paddr);
		int err = 0;

		if (addr == NETMAP_BUF_BASE(na)) { /* bad buf */
			if (netmap_ring_reinit(kring))

		}

		slot->flags &= ~NS_BUF_CHANGED;
		sglist_reset(&sg); // cheap
		err = sglist_append(&sg, &rxq->vtnrx_shrhdr, sc->vtnet_hdr_size);
//		err = sglist_append_phys(&sg, paddr, NETMAP_BUF_SIZE(na));
		err |= sglist_append_phys(&sg, paddr, NETMAP_BUF_SIZE(na));
		KASSERT(err == 0, ("%s: cannot append to sglist %d",
					__func__, err));
		/* writable for the host */
		err = virtqueue_enqueue(vq, /*cookie=*/rxq, &sg,
				/*readable=*/0, /*writeable=*/sg.sg_nseg);
		KASSERT(err == 0, ("%s: cannot append to sglist %d",
					__func__, err));

		if (unlikely(err)) {
			if (err != ENOSPC)
				nm_prerr("virtqueue_enqueue(%s) failed: %d\n",
					kring->name, err);
			break;
		}
		nm_i = nm_next(nm_i, lim);
	}

	return nm_i;
}


	u_int nm_i;	/* index into the netmap ring */
	u_int const lim = kring->nkr_num_slots - 1;
	u_int const head = kring->rhead;
	int force_update = (flags & NAF_FORCE_READ) || (kring->nr_kflags & NKR_PENDINTR);
				(kring->nr_kflags & NKR_PENDINTR);
	int interrupts = !(kring->nr_kflags & NKR_NOINTR);

	/* device-specific */

	struct vtnet_rxq *rxq = &sc->vtnet_rxqs[ring_nr];
	struct virtqueue *vq = rxq->vtnrx_vq;

	/* XXX netif_carrier_ok ? */

//	if (head > lim)
//		return netmap_ring_reinit(kring);

	rmb();
	/*
	 * First part: import newly received packets.
	 * Only accept our own buffers (matching the token). We should only get
	 * matching buffers, because of vtnet_netmap_free_rx_unused_bufs()
	 * and vtnet_netmap_init_buffers().  We may need to stop early to avoid 
	 * hwtail to overrun hwcur.
	 */
	if (netmap_no_pendintr || force_update) {
		uint32_t hwtail_lim = nm_prev(kring->nr_hwcur, lim);
		struct netmap_adapter *token;

		vtnet_rxq_disable_intr(rxq);


		kring->nr_hwtail = nm_i;
		kring->nr_kflags &= ~NKR_PENDINTR;
	}
	ND("[B] h %d c %d hwcur %d hwtail %d",
	ring->head, ring->cur, kring->nr_hwcur,
			  kring->nr_hwtail);

	/*
	 * Second part: skip past packets that userspace has released.
	 */
	nm_i = kring->nr_hwcur; /* netmap ring index */
	if (nm_i != head) {
		int err = vtnet_netmap_kring_refill(kring, nm_i, head);
		if (err < 0)
			return 1;
		kring->nr_hwcur = err;
		virtqueue_notify(vq);
		/* After draining the queue may need an intr from the hypervisor */
		if (interrupts) {
			vtnet_rxq_enable_intr(rxq);
		}
	}
	ND("[C] h %d c %d t %d hwcur %d hwtail %d",
	ring->head, ring->cur, ring->tail,
	kring->nr_hwcur, kring->nr_hwtail);

	ND("[C] h %d c %d t %d hwcur %d hwtail %d", ring->head, ring->cur,
		ring->tail, kring->nr_hwcur, kring->nr_hwtail);

	return 0;
}


	}
}

/* Make RX virtqueues buffers pointing to netmap buffers. */
static int
vtnet_netmap_init_rx_buffers(struct vtnet_softc *sc)
{
	struct ifnet *ifp = sc->vtnet_ifp;
	struct netmap_adapter* na = NA(ifp);
	unsigned int r;

	if (!nm_native_on(na))
		return 0;
	for (r = 0; r < na->num_rx_rings; r++) {
                struct netmap_kring *kring = na->rx_rings[r];
		struct vtnet_rxq *rxq = &sc->vtnet_rxqs[r];
		struct virtqueue *vq = rxq->vtnrx_vq;
	        struct netmap_slot* slot;
		int err = 0;

		slot = netmap_reset(na, NR_RX, r, 0);
		if (!slot) {
			D("strange, null netmap ring %d", r);
			return 0;
		}
		/* Add up to na>-num_rx_desc-1 buffers to this RX virtqueue.
		 * It's important to leave one virtqueue slot free, otherwise
		 * we can run into ring->cur/ring->tail wraparounds.
		 */
		err = vtnet_netmap_kring_refill(kring, 0, na->num_rx_desc-1);
		if (err < 0)
			return 0;
		virtqueue_notify(vq);
	}

	return 1;
}

static int
vtnet_netmap_tx_slots(struct vtnet_softc *sc)
{
	int div;


	na.ifp = sc->vtnet_ifp;
	na.na_flags = 0;
//	na.num_tx_desc =  1024;// sc->vtnet_rx_nmbufs;
	na.num_tx_desc = vtnet_netmap_tx_slots(sc);
//	na.num_rx_desc =  1024; // sc->vtnet_rx_nmbufs;
	na.num_rx_desc = vtnet_netmap_rx_slots(sc);
	na.num_tx_rings = na.num_rx_rings = sc->vtnet_max_vq_pairs;
	na.rx_buf_maxsize = 0;
	na.nm_register = vtnet_netmap_reg;
	na.nm_txsync = vtnet_netmap_txsync;
	na.nm_rxsync = vtnet_netmap_rxsync;
	na.nm_intr = vtnet_netmap_intr;
	na.num_tx_rings = na.num_rx_rings = sc->vtnet_max_vq_pairs;
	na.nm_config = vtnet_netmap_config;
	D("max rings %d", sc->vtnet_max_vq_pairs);

	netmap_attach(&na);

	nm_prinf("vtnet attached txq=%d, txd=%d rxq=%d, rxd=%d\n",
			na.num_tx_rings, na.num_tx_desc,
			na.num_tx_rings, na.num_rx_desc);

	netmap_attach(&na);
}
/* end of file */

Return to bug 236922

Lines 52-58 Link Here

(-)sys/dev/netmap/if_vtnet_netmap.h (-42 / +172 lines)
52	na->tx_rings[idx]->nr_mode == NKR_NETMAP_ON);	52	na->tx_rings[idx]->nr_mode == NKR_NETMAP_ON);
53	}	53	}
54		54
		55	/* Free all the unused buffer in all the RX virtqueues.
		56	* This function is called when entering and exiting netmap mode.
		57	* - buffers queued by the virtio driver return skbuf/mbuf pointer
		58	* and need to be freed;
		59	* - buffers queued by netmap return the txq/rxq, and do not need work
		60	*/
55	static void	61	static void
		62	vtnet_netmap_free_bufs(struct vtnet_softc *sc)
		63	{
		64	int i, nmb = 0, n = 0, last;
		65
		66	for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
		67	struct vtnet_rxq *rxq = &sc->vtnet_rxqs[i];
		68	struct vtnet_txq *txq = &sc->vtnet_txqs[i];
		69	struct vtnet_tx_header *txhdr;
		70	struct virtqueue *vq;
		71	struct mbuf *m;
		72
		73	last = 0;
		74	vq = rxq->vtnrx_vq;
		75	while ((m = virtqueue_drain(vq, &last)) != NULL) {
		76	n++;
		77	if (m != (void *)rxq)
		78	m_freem(m);
		79	else
		80	nmb++;
		81	}
		82
		83	last = 0;
		84	vq = txq->vtntx_vq;
		85	while ((txhdr = virtqueue_drain(vq, &last)) != NULL) {
		86	n++;
		87	if (txhdr != (void *)txq) {
		88	m_freem(txhdr->vth_mbuf);
		89	uma_zfree(vtnet_tx_header_zone, txhdr);
		90	} else
		91	nmb++;
		92	}
		93	}
		94	D("freed %d mbufs, %d netmap bufs on %d queues",
		95	n - nmb, nmb, i);
		96	}
		97
		98	static void
56	vtnet_free_used(struct virtqueue *vq, int netmap_bufs, enum txrx t, int idx)	99	vtnet_free_used(struct virtqueue *vq, int netmap_bufs, enum txrx t, int idx)
57	{	100	{
58	void *cookie;	101	void *cookie;
Lines 89-95 Link Here
89	{	132	{
90	struct ifnet *ifp = na->ifp;	133	struct ifnet *ifp = na->ifp;
91	struct vtnet_softc *sc = ifp->if_softc;	134	struct vtnet_softc *sc = ifp->if_softc;
92	int success;
93	enum txrx t;	135	enum txrx t;
94	int i;	136	int i;
95		137
Lines 98-112 Link Here
98	vtnet_drain_taskqueues(sc);	140	vtnet_drain_taskqueues(sc);
99		141
100	VTNET_CORE_LOCK(sc);	142	VTNET_CORE_LOCK(sc);
101
102	/* We need nm_netmap_on() to return true when called by	143	/* We need nm_netmap_on() to return true when called by
103	* vtnet_init_locked() below. */	144	* vtnet_init_locked() below. */
104	if (state)	145	/* enable or disable flags and callbacks in na and ifp */
		146	if (state) {
105	nm_set_native_flags(na);	147	nm_set_native_flags(na);
106		148	} else {
		149	nm_clear_native_flags(na);
		150	}
107	/* We need to trigger a device reset in order to unexpose guest buffers	151	/* We need to trigger a device reset in order to unexpose guest buffers
108	* published to the host. */	152	* published to the host. */
109	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING \| IFF_DRV_OACTIVE);	153	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING \| IFF_DRV_OACTIVE);
		154
		155	/* drain queues so netmap and native drivers
		156	* do not interfere with each other
		157	*/
		158	// vtnet_netmap_free_bufs(sc);
110	/* Get pending used buffers. The way they are freed depends on whether	159	/* Get pending used buffers. The way they are freed depends on whether
111	* they are netmap buffer or they are mbufs. We can tell apart the two	160	* they are netmap buffer or they are mbufs. We can tell apart the two
112	* cases by looking at kring->nr_mode, before this is possibly updated	161	* cases by looking at kring->nr_mode, before this is possibly updated
Lines 128-136 Link Here
128	kring->nr_mode == NKR_NETMAP_ON, NR_RX, i);	177	kring->nr_mode == NKR_NETMAP_ON, NR_RX, i);
129	VTNET_RXQ_UNLOCK(rxq);	178	VTNET_RXQ_UNLOCK(rxq);
130	}	179	}
131	vtnet_init_locked(sc);	180	vtnet_init_locked(sc); /* also enable intr */
132	success = (ifp->if_drv_flags & IFF_DRV_RUNNING) ? 0 : ENXIO;
133
134	if (state) {	181	if (state) {
135	for_rx_tx(t) {	182	for_rx_tx(t) {
136	/* Hardware rings. */	183	/* Hardware rings. */
Lines 173-180 Link Here
173	}	220	}
174		221
175	VTNET_CORE_UNLOCK(sc);	222	VTNET_CORE_UNLOCK(sc);
176		223	return (ifp->if_drv_flags & IFF_DRV_RUNNING) ? 0 : ENXIO;
177	return success;
178	}	224	}
179		225
180		226
Lines 187-192 Link Here
187	struct netmap_ring *ring = kring->ring;	233	struct netmap_ring *ring = kring->ring;
188	u_int ring_nr = kring->ring_id;	234	u_int ring_nr = kring->ring_id;
189	u_int nm_i; /* index into the netmap ring */	235	u_int nm_i; /* index into the netmap ring */
		236	// u_int nic_i; /* index into the NIC ring */
190	u_int const lim = kring->nkr_num_slots - 1;	237	u_int const lim = kring->nkr_num_slots - 1;
191	u_int const head = kring->rhead;	238	u_int const head = kring->rhead;
192		239
Lines 206-211 Link Here
206	if (nm_i != head) { /* we have new packets to send */	253	if (nm_i != head) { /* we have new packets to send */
207	struct sglist *sg = txq->vtntx_sg;	254	struct sglist *sg = txq->vtntx_sg;
208		255
		256	// nic_i = netmap_idx_k2n(kring, nm_i);
209	for (; nm_i != head; nm_i = nm_next(nm_i, lim)) {	257	for (; nm_i != head; nm_i = nm_next(nm_i, lim)) {
210	/* we use an empty header here */	258	/* we use an empty header here */
211	struct netmap_slot *slot = &ring->slot[nm_i];	259	struct netmap_slot *slot = &ring->slot[nm_i];
Lines 221-269 Link Here
221	* and kick the hypervisor (if necessary).	269	* and kick the hypervisor (if necessary).
222	*/	270	*/
223	sglist_reset(sg); // cheap	271	sglist_reset(sg); // cheap
		272	// if vtnet_hdr_size > 0 ...
224	err = sglist_append(sg, &txq->vtntx_shrhdr, sc->vtnet_hdr_size);	273	err = sglist_append(sg, &txq->vtntx_shrhdr, sc->vtnet_hdr_size);
		274	// XXX later, support multi segment
		275	// err = sglist_append_phys(sg, paddr, len);
225	err \|= sglist_append_phys(sg, paddr, len);	276	err \|= sglist_append_phys(sg, paddr, len);
226	KASSERT(err == 0, ("%s: cannot append to sglist %d",	277	KASSERT(err == 0, ("%s: cannot append to sglist %d",
227	__func__, err));	278	__func__, err));
228	err = virtqueue_enqueue(vq, /cookie=/txq, sg,	279	err = virtqueue_enqueue(vq, /cookie=/txq, sg,
229	/readable=/sg->sg_nseg,	280	/readable=/sg->sg_nseg,
230	/writeable=/0);	281	/writeable=/0);
		282	/* use na as the cookie */
		283	err = virtqueue_enqueue(vq, txq, sg, sg->sg_nseg, 0);
231	if (unlikely(err)) {	284	if (unlikely(err)) {
232	if (err != ENOSPC)	285	if (err != ENOSPC)
233	nm_prerr("virtqueue_enqueue(%s) failed: %d\n",	286	nm_prerr("virtqueue_enqueue(%s) failed: %d\n",
234	kring->name, err);	287	kring->name, err);
235	break;	288	break;
236	}	289	}
		290	// nic_i = nm_next(nic_i, lim);
237	}	291	}
238		292
		293	/* No more free TX slots? Ask the hypervisor for notifications,
		294	* possibly only when a considerable amount of work has been
		295	* done.
		296	*/
		297	ND(3,"sent %d packets, hwcur %d", n, nm_i);
		298	// virtqueue_disable_intr(vq); // ??
239	virtqueue_notify(vq);	299	virtqueue_notify(vq);
240		300
241	/* Update hwcur depending on where we stopped. */	301	/* Update hwcur depending on where we stopped. */
242	kring->nr_hwcur = nm_i; /* note we migth break early */	302	kring->nr_hwcur = nm_i; /* note we migth break early */
		303
		304	} else {
		305	if (ring->head != ring->tail)
		306	ND(5, "pure notify ? head %d tail %d nused %d %d",
		307	ring->head, ring->tail, virtqueue_nused(vq),
		308	(virtqueue_dump(vq), 1));
		309	virtqueue_notify(vq);
		310	if (interrupts) {
		311	virtqueue_enable_intr(vq); // like postpone with 0
		312	}
243	}	313	}
244		314
245	/* Free used slots. We only consider our own used buffers, recognized	315	/* Free used slots. We only consider our own used buffers, recognized
246	* by the token we passed to virtqueue_enqueue.	316	* by the token we passed to virtqueue_add_outbuf.
247	*/	317	*/
248	n = 0;	318	n = 0;
249	for (;;) {	319	for (;;) {
250	void *token = virtqueue_dequeue(vq, NULL);	320	struct vtnet_tx_header *txhdr = virtqueue_dequeue(vq, NULL);
251	if (token == NULL)	321	if (txhdr == NULL)
252	break;	322	break;
253	if (unlikely(token != (void *)txq))	323	if (likely(txhdr == (void *)txq)) {
254	nm_prerr("BUG: TX token mismatch\n");
255	else
256	n++;	324	n++;
		325	if (virtqueue_nused(vq) < 32) { // XXX slow release
		326	break;
		327	}
		328	} else { /* leftover from previous transmission */
		329	nm_prerr("BUG: TX token mismatch\n");
		330	m_freem(txhdr->vth_mbuf);
		331	uma_zfree(vtnet_tx_header_zone, txhdr);
		332	}
257	}	333	}
258	if (n > 0) {	334	if (n) {
259	kring->nr_hwtail += n;	335	kring->nr_hwtail += n;
260	if (kring->nr_hwtail > lim)	336	if (kring->nr_hwtail > lim)
261	kring->nr_hwtail -= lim + 1;	337	kring->nr_hwtail -= lim + 1;
262	}	338	}
		339	if (nm_i != kring->nr_hwtail /* && vtnet_txq_below_threshold(txq) == 0*/) {
		340	ND(3, "disable intr, hwcur %d", nm_i);
		341	virtqueue_disable_intr(vq);
		342	} else if (interrupts && virtqueue_nfree(vq) < 32) {
		343	ND(3, "enable intr, hwcur %d", nm_i);
		344	virtqueue_postpone_intr(vq, VQ_POSTPONE_SHORT);
		345	}
263		346
264	if (interrupts && virtqueue_nfree(vq) < 32)
265	virtqueue_postpone_intr(vq, VQ_POSTPONE_LONG);
266
267	return 0;	347	return 0;
268	}	348	}
269		349
Lines 289-295 Link Here
289	struct netmap_slot *slot = &ring->slot[nm_i];	369	struct netmap_slot *slot = &ring->slot[nm_i];
290	uint64_t paddr;	370	uint64_t paddr;
291	void *addr = PNMB(na, slot, &paddr);	371	void *addr = PNMB(na, slot, &paddr);
292	int err;	372	int err = 0;
293		373
294	if (addr == NETMAP_BUF_BASE(na)) { /* bad buf */	374	if (addr == NETMAP_BUF_BASE(na)) { /* bad buf */
295	if (netmap_ring_reinit(kring))	375	if (netmap_ring_reinit(kring))
Lines 297-318 Link Here
297	}	377	}
298		378
299	slot->flags &= ~NS_BUF_CHANGED;	379	slot->flags &= ~NS_BUF_CHANGED;
300	sglist_reset(&sg);	380	sglist_reset(&sg); // cheap
301	err = sglist_append(&sg, &rxq->vtnrx_shrhdr, sc->vtnet_hdr_size);	381	err = sglist_append(&sg, &rxq->vtnrx_shrhdr, sc->vtnet_hdr_size);
		382	// err = sglist_append_phys(&sg, paddr, NETMAP_BUF_SIZE(na));
302	err \|= sglist_append_phys(&sg, paddr, NETMAP_BUF_SIZE(na));	383	err \|= sglist_append_phys(&sg, paddr, NETMAP_BUF_SIZE(na));
303	KASSERT(err == 0, ("%s: cannot append to sglist %d",
304	__func__, err));
305	/* writable for the host */	384	/* writable for the host */
306	err = virtqueue_enqueue(vq, /cookie=/rxq, &sg,	385	err = virtqueue_enqueue(vq, /cookie=/rxq, &sg,
307	/readable=/0, /writeable=/sg.sg_nseg);	386	/readable=/0, /writeable=/sg.sg_nseg);
		387	KASSERT(err == 0, ("%s: cannot append to sglist %d",
		388	__func__, err));
		389
308	if (unlikely(err)) {	390	if (unlikely(err)) {
309	if (err != ENOSPC)	391	if (err != ENOSPC)
310	nm_prerr("virtqueue_enqueue(%s) failed: %d\n",	392	nm_prerr("virtqueue_enqueue(%s) failed: %d\n",
311	kring->name, err);	393	kring->name, err);
312	break;	394	break;
313	}	395	}
		396	nm_i = nm_next(nm_i, lim);
314	}	397	}
315
316	return nm_i;	398	return nm_i;
317	}	399	}
318		400
Lines 357-364 Link Here
357	u_int nm_i; /* index into the netmap ring */	439	u_int nm_i; /* index into the netmap ring */
358	u_int const lim = kring->nkr_num_slots - 1;	440	u_int const lim = kring->nkr_num_slots - 1;
359	u_int const head = kring->rhead;	441	u_int const head = kring->rhead;
360	int force_update = (flags & NAF_FORCE_READ) \|\|	442	int force_update = (flags & NAF_FORCE_READ) \|\| (kring->nr_kflags & NKR_PENDINTR);
361	(kring->nr_kflags & NKR_PENDINTR);
362	int interrupts = !(kring->nr_kflags & NKR_NOINTR);	443	int interrupts = !(kring->nr_kflags & NKR_NOINTR);
363		444
364	/* device-specific */	445	/* device-specific */
Lines 366-381 Link Here
366	struct vtnet_rxq *rxq = &sc->vtnet_rxqs[ring_nr];	447	struct vtnet_rxq *rxq = &sc->vtnet_rxqs[ring_nr];
367	struct virtqueue *vq = rxq->vtnrx_vq;	448	struct virtqueue *vq = rxq->vtnrx_vq;
368		449
		450	/* XXX netif_carrier_ok ? */
		451
		452	// if (head > lim)
		453	// return netmap_ring_reinit(kring);
		454
369	rmb();	455	rmb();
370	/*	456	/*
371	* First part: import newly received packets.	457	* First part: import newly received packets.
372	* Only accept our own buffers (matching the token). We should only get	458	* Only accept our own buffers (matching the token). We should only get
373	* matching buffers. We may need to stop early to avoid hwtail to overrun	459	* matching buffers, because of vtnet_netmap_free_rx_unused_bufs()
374	* hwcur.	460	* and vtnet_netmap_init_buffers(). We may need to stop early to avoid
		461	* hwtail to overrun hwcur.
375	*/	462	*/
376	if (netmap_no_pendintr \|\| force_update) {	463	if (netmap_no_pendintr \|\| force_update) {
377	uint32_t hwtail_lim = nm_prev(kring->nr_hwcur, lim);	464	uint32_t hwtail_lim = nm_prev(kring->nr_hwcur, lim);
378	void *token;	465	struct netmap_adapter *token;
379		466
380	vtnet_rxq_disable_intr(rxq);	467	vtnet_rxq_disable_intr(rxq);
381		468
Lines 408-431 Link Here
408	kring->nr_hwtail = nm_i;	495	kring->nr_hwtail = nm_i;
409	kring->nr_kflags &= ~NKR_PENDINTR;	496	kring->nr_kflags &= ~NKR_PENDINTR;
410	}	497	}
411	ND("[B] h %d c %d hwcur %d hwtail %d", ring->head, ring->cur,	498	ND("[B] h %d c %d hwcur %d hwtail %d",
412	kring->nr_hwcur, kring->nr_hwtail);	499	ring->head, ring->cur, kring->nr_hwcur,
		500	kring->nr_hwtail);
413		501
414	/*	502	/*
415	* Second part: skip past packets that userspace has released.	503	* Second part: skip past packets that userspace has released.
416	*/	504	*/
417	nm_i = kring->nr_hwcur; /* netmap ring index */	505	nm_i = kring->nr_hwcur; /* netmap ring index */
418	if (nm_i != head) {	506	if (nm_i != head) {
419	int nm_j = vtnet_netmap_kring_refill(kring, nm_i, head);	507	int err = vtnet_netmap_kring_refill(kring, nm_i, head);
420	if (nm_j < 0)	508	if (err < 0)
421	return nm_j;	509	return 1;
422	kring->nr_hwcur = nm_j;	510	kring->nr_hwcur = err;
423	virtqueue_notify(vq);	511	virtqueue_notify(vq);
		512	/* After draining the queue may need an intr from the hypervisor */
		513	if (interrupts) {
		514	vtnet_rxq_enable_intr(rxq);
		515	}
424	}	516	}
		517	ND("[C] h %d c %d t %d hwcur %d hwtail %d",
		518	ring->head, ring->cur, ring->tail,
		519	kring->nr_hwcur, kring->nr_hwtail);
425		520
426	ND("[C] h %d c %d t %d hwcur %d hwtail %d", ring->head, ring->cur,
427	ring->tail, kring->nr_hwcur, kring->nr_hwtail);
428
429	return 0;	521	return 0;
430	}	522	}
431		523
Lines 452-458 Link Here
452	}	544	}
453	}	545	}
454		546
		547	/* Make RX virtqueues buffers pointing to netmap buffers. */
455	static int	548	static int
		549	vtnet_netmap_init_rx_buffers(struct vtnet_softc *sc)
		550	{
		551	struct ifnet *ifp = sc->vtnet_ifp;
		552	struct netmap_adapter* na = NA(ifp);
		553	unsigned int r;
		554
		555	if (!nm_native_on(na))
		556	return 0;
		557	for (r = 0; r < na->num_rx_rings; r++) {
		558	struct netmap_kring *kring = na->rx_rings[r];
		559	struct vtnet_rxq *rxq = &sc->vtnet_rxqs[r];
		560	struct virtqueue *vq = rxq->vtnrx_vq;
		561	struct netmap_slot* slot;
		562	int err = 0;
		563
		564	slot = netmap_reset(na, NR_RX, r, 0);
		565	if (!slot) {
		566	D("strange, null netmap ring %d", r);
		567	return 0;
		568	}
		569	/* Add up to na>-num_rx_desc-1 buffers to this RX virtqueue.
		570	* It's important to leave one virtqueue slot free, otherwise
		571	* we can run into ring->cur/ring->tail wraparounds.
		572	*/
		573	err = vtnet_netmap_kring_refill(kring, 0, na->num_rx_desc-1);
		574	if (err < 0)
		575	return 0;
		576	virtqueue_notify(vq);
		577	}
		578
		579	return 1;
		580	}
		581
		582	static int
456	vtnet_netmap_tx_slots(struct vtnet_softc *sc)	583	vtnet_netmap_tx_slots(struct vtnet_softc *sc)
457	{	584	{
458	int div;	585	int div;
Lines 521-540 Link Here
521		648
522	na.ifp = sc->vtnet_ifp;	649	na.ifp = sc->vtnet_ifp;
523	na.na_flags = 0;	650	na.na_flags = 0;
		651	// na.num_tx_desc = 1024;// sc->vtnet_rx_nmbufs;
524	na.num_tx_desc = vtnet_netmap_tx_slots(sc);	652	na.num_tx_desc = vtnet_netmap_tx_slots(sc);
		653	// na.num_rx_desc = 1024; // sc->vtnet_rx_nmbufs;
525	na.num_rx_desc = vtnet_netmap_rx_slots(sc);	654	na.num_rx_desc = vtnet_netmap_rx_slots(sc);
526	na.num_tx_rings = na.num_rx_rings = sc->vtnet_max_vq_pairs;
527	na.rx_buf_maxsize = 0;	655	na.rx_buf_maxsize = 0;
528	na.nm_register = vtnet_netmap_reg;	656	na.nm_register = vtnet_netmap_reg;
529	na.nm_txsync = vtnet_netmap_txsync;	657	na.nm_txsync = vtnet_netmap_txsync;
530	na.nm_rxsync = vtnet_netmap_rxsync;	658	na.nm_rxsync = vtnet_netmap_rxsync;
531	na.nm_intr = vtnet_netmap_intr;	659	na.nm_intr = vtnet_netmap_intr;
		660	na.num_tx_rings = na.num_rx_rings = sc->vtnet_max_vq_pairs;
532	na.nm_config = vtnet_netmap_config;	661	na.nm_config = vtnet_netmap_config;
		662	D("max rings %d", sc->vtnet_max_vq_pairs);
533		663
534	netmap_attach(&na);
535
536	nm_prinf("vtnet attached txq=%d, txd=%d rxq=%d, rxd=%d\n",	664	nm_prinf("vtnet attached txq=%d, txd=%d rxq=%d, rxd=%d\n",
537	na.num_tx_rings, na.num_tx_desc,	665	na.num_tx_rings, na.num_tx_desc,
538	na.num_tx_rings, na.num_rx_desc);	666	na.num_tx_rings, na.num_rx_desc);
		667
		668	netmap_attach(&na);
539	}	669	}
540	/* end of file */	670	/* end of file */