|
Lines 346-351
struct iflib_txq {
Link Here
|
| 346 |
qidx_t ift_size; |
346 |
qidx_t ift_size; |
| 347 |
uint16_t ift_id; |
347 |
uint16_t ift_id; |
| 348 |
struct callout ift_timer; |
348 |
struct callout ift_timer; |
|
|
349 |
#ifdef DEV_NETMAP |
| 350 |
struct callout ift_netmap_timer; |
| 351 |
#endif /* DEV_NETMAP */ |
| 349 |
|
352 |
|
| 350 |
if_txsd_vec_t ift_sds; |
353 |
if_txsd_vec_t ift_sds; |
| 351 |
uint8_t ift_qstatus; |
354 |
uint8_t ift_qstatus; |
|
Lines 753-758
iflib_num_tx_descs(if_ctx_t ctx)
Link Here
|
| 753 |
MODULE_DEPEND(iflib, netmap, 1, 1, 1); |
756 |
MODULE_DEPEND(iflib, netmap, 1, 1, 1); |
| 754 |
|
757 |
|
| 755 |
static int netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, bool init); |
758 |
static int netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, bool init); |
|
|
759 |
static void iflib_netmap_timer(void *arg); |
| 756 |
|
760 |
|
| 757 |
/* |
761 |
/* |
| 758 |
* device-specific sysctl variables: |
762 |
* device-specific sysctl variables: |
|
Lines 918-923
netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, bool init)
Link Here
|
| 918 |
return (0); |
922 |
return (0); |
| 919 |
} |
923 |
} |
| 920 |
|
924 |
|
|
|
925 |
#define NETMAP_TX_TIMER_US 90 |
| 926 |
|
| 921 |
/* |
927 |
/* |
| 922 |
* Reconcile kernel and user view of the transmit ring. |
928 |
* Reconcile kernel and user view of the transmit ring. |
| 923 |
* |
929 |
* |
|
Lines 1047-1055
iflib_netmap_txsync(struct netmap_kring *kring, int flags)
Link Here
|
| 1047 |
* Second part: reclaim buffers for completed transmissions. |
1053 |
* Second part: reclaim buffers for completed transmissions. |
| 1048 |
* |
1054 |
* |
| 1049 |
* If there are unclaimed buffers, attempt to reclaim them. |
1055 |
* If there are unclaimed buffers, attempt to reclaim them. |
| 1050 |
* If none are reclaimed, and TX IRQs are not in use, do an initial |
1056 |
* If we don't manage to reclaim them all, and TX IRQs are not in use, |
| 1051 |
* minimal delay, then trigger the tx handler which will spin in the |
1057 |
* trigger a per-tx-queue timer to try again later. |
| 1052 |
* group task queue. |
|
|
| 1053 |
*/ |
1058 |
*/ |
| 1054 |
if (kring->nr_hwtail != nm_prev(kring->nr_hwcur, lim)) { |
1059 |
if (kring->nr_hwtail != nm_prev(kring->nr_hwcur, lim)) { |
| 1055 |
if (iflib_tx_credits_update(ctx, txq)) { |
1060 |
if (iflib_tx_credits_update(ctx, txq)) { |
|
Lines 1058-1068
iflib_netmap_txsync(struct netmap_kring *kring, int flags)
Link Here
|
| 1058 |
kring->nr_hwtail = nm_prev(netmap_idx_n2k(kring, nic_i), lim); |
1063 |
kring->nr_hwtail = nm_prev(netmap_idx_n2k(kring, nic_i), lim); |
| 1059 |
} |
1064 |
} |
| 1060 |
} |
1065 |
} |
|
|
1066 |
|
| 1061 |
if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ)) |
1067 |
if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ)) |
| 1062 |
if (kring->nr_hwtail != nm_prev(kring->nr_hwcur, lim)) { |
1068 |
if (kring->nr_hwtail != nm_prev(kring->nr_hwcur, lim)) { |
| 1063 |
callout_reset_on(&txq->ift_timer, hz < 2000 ? 1 : hz / 1000, |
1069 |
callout_reset_sbt_on(&txq->ift_netmap_timer, |
| 1064 |
iflib_timer, txq, txq->ift_timer.c_cpu); |
1070 |
NETMAP_TX_TIMER_US * SBT_1US, SBT_1US, |
| 1065 |
} |
1071 |
iflib_netmap_timer, txq, |
|
|
1072 |
txq->ift_netmap_timer.c_cpu, 0); |
| 1073 |
} |
| 1066 |
return (0); |
1074 |
return (0); |
| 1067 |
} |
1075 |
} |
| 1068 |
|
1076 |
|
|
Lines 1263-1290
iflib_netmap_rxq_init(if_ctx_t ctx, iflib_rxq_t rxq)
Link Here
|
| 1263 |
} |
1271 |
} |
| 1264 |
|
1272 |
|
| 1265 |
static void |
1273 |
static void |
| 1266 |
iflib_netmap_timer_adjust(if_ctx_t ctx, iflib_txq_t txq, uint32_t *reset_on) |
1274 |
iflib_netmap_timer(void *arg) |
| 1267 |
{ |
1275 |
{ |
| 1268 |
struct netmap_kring *kring; |
1276 |
iflib_txq_t txq = arg; |
| 1269 |
uint16_t txqid; |
1277 |
if_ctx_t ctx = txq->ift_ctx; |
| 1270 |
|
|
|
| 1271 |
txqid = txq->ift_id; |
| 1272 |
kring = netmap_kring_on(NA(ctx->ifc_ifp), txqid, NR_TX); |
| 1273 |
if (kring == NULL) |
| 1274 |
return; |
| 1275 |
|
1278 |
|
| 1276 |
if (kring->nr_hwcur != nm_next(kring->nr_hwtail, kring->nkr_num_slots - 1)) { |
1279 |
/* |
| 1277 |
bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, |
1280 |
* Wake up the netmap application, to give it a chance to |
| 1278 |
BUS_DMASYNC_POSTREAD); |
1281 |
* call txsync and reclaim more completed TX buffers. |
| 1279 |
if (ctx->isc_txd_credits_update(ctx->ifc_softc, txqid, false)) |
1282 |
*/ |
| 1280 |
netmap_tx_irq(ctx->ifc_ifp, txqid); |
1283 |
netmap_tx_irq(ctx->ifc_ifp, txq->ift_id); |
| 1281 |
if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ)) { |
|
|
| 1282 |
if (hz < 2000) |
| 1283 |
*reset_on = 1; |
| 1284 |
else |
| 1285 |
*reset_on = hz / 1000; |
| 1286 |
} |
| 1287 |
} |
| 1288 |
} |
1284 |
} |
| 1289 |
|
1285 |
|
| 1290 |
#define iflib_netmap_detach(ifp) netmap_detach(ifp) |
1286 |
#define iflib_netmap_detach(ifp) netmap_detach(ifp) |
|
Lines 1296-1303
iflib_netmap_timer_adjust(if_ctx_t ctx, iflib_txq_t txq, uint32_t *reset_on)
Link Here
|
| 1296 |
|
1292 |
|
| 1297 |
#define iflib_netmap_attach(ctx) (0) |
1293 |
#define iflib_netmap_attach(ctx) (0) |
| 1298 |
#define netmap_rx_irq(ifp, qid, budget) (0) |
1294 |
#define netmap_rx_irq(ifp, qid, budget) (0) |
| 1299 |
#define netmap_tx_irq(ifp, qid) do {} while (0) |
|
|
| 1300 |
#define iflib_netmap_timer_adjust(ctx, txq, reset_on) |
| 1301 |
#endif |
1295 |
#endif |
| 1302 |
|
1296 |
|
| 1303 |
#if defined(__i386__) || defined(__amd64__) |
1297 |
#if defined(__i386__) || defined(__amd64__) |
|
Lines 2287-2293
iflib_timer(void *arg)
Link Here
|
| 2287 |
if_ctx_t ctx = txq->ift_ctx; |
2281 |
if_ctx_t ctx = txq->ift_ctx; |
| 2288 |
if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; |
2282 |
if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; |
| 2289 |
uint64_t this_tick = ticks; |
2283 |
uint64_t this_tick = ticks; |
| 2290 |
uint32_t reset_on = hz / 2; |
|
|
| 2291 |
|
2284 |
|
| 2292 |
if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) |
2285 |
if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) |
| 2293 |
return; |
2286 |
return; |
|
Lines 2312-2328
iflib_timer(void *arg)
Link Here
|
| 2312 |
} |
2305 |
} |
| 2313 |
txq->ift_cleaned_prev = txq->ift_cleaned; |
2306 |
txq->ift_cleaned_prev = txq->ift_cleaned; |
| 2314 |
} |
2307 |
} |
| 2315 |
#ifdef DEV_NETMAP |
|
|
| 2316 |
if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) |
| 2317 |
iflib_netmap_timer_adjust(ctx, txq, &reset_on); |
| 2318 |
#endif |
| 2319 |
/* handle any laggards */ |
2308 |
/* handle any laggards */ |
| 2320 |
if (txq->ift_db_pending) |
2309 |
if (txq->ift_db_pending) |
| 2321 |
GROUPTASK_ENQUEUE(&txq->ift_task); |
2310 |
GROUPTASK_ENQUEUE(&txq->ift_task); |
| 2322 |
|
2311 |
|
| 2323 |
sctx->isc_pause_frames = 0; |
2312 |
sctx->isc_pause_frames = 0; |
| 2324 |
if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) |
2313 |
if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) |
| 2325 |
callout_reset_on(&txq->ift_timer, reset_on, iflib_timer, txq, txq->ift_timer.c_cpu); |
2314 |
callout_reset_on(&txq->ift_timer, hz / 2, iflib_timer, txq, txq->ift_timer.c_cpu); |
| 2326 |
return; |
2315 |
return; |
| 2327 |
|
2316 |
|
| 2328 |
hung: |
2317 |
hung: |
|
Lines 2396-2401
iflib_init_locked(if_ctx_t ctx)
Link Here
|
| 2396 |
for (i = 0, txq = ctx->ifc_txqs; i < sctx->isc_ntxqsets; i++, txq++) { |
2385 |
for (i = 0, txq = ctx->ifc_txqs; i < sctx->isc_ntxqsets; i++, txq++) { |
| 2397 |
CALLOUT_LOCK(txq); |
2386 |
CALLOUT_LOCK(txq); |
| 2398 |
callout_stop(&txq->ift_timer); |
2387 |
callout_stop(&txq->ift_timer); |
|
|
2388 |
#ifdef DEV_NETMAP |
| 2389 |
callout_stop(&txq->ift_netmap_timer); |
| 2390 |
#endif /* DEV_NETMAP */ |
| 2399 |
CALLOUT_UNLOCK(txq); |
2391 |
CALLOUT_UNLOCK(txq); |
| 2400 |
iflib_netmap_txq_init(ctx, txq); |
2392 |
iflib_netmap_txq_init(ctx, txq); |
| 2401 |
} |
2393 |
} |
|
Lines 2485-2490
iflib_stop(if_ctx_t ctx)
Link Here
|
| 2485 |
|
2477 |
|
| 2486 |
CALLOUT_LOCK(txq); |
2478 |
CALLOUT_LOCK(txq); |
| 2487 |
callout_stop(&txq->ift_timer); |
2479 |
callout_stop(&txq->ift_timer); |
|
|
2480 |
#ifdef DEV_NETMAP |
| 2481 |
callout_stop(&txq->ift_netmap_timer); |
| 2482 |
#endif /* DEV_NETMAP */ |
| 2488 |
CALLOUT_UNLOCK(txq); |
2483 |
CALLOUT_UNLOCK(txq); |
| 2489 |
|
2484 |
|
| 2490 |
/* clean any enqueued buffers */ |
2485 |
/* clean any enqueued buffers */ |
|
Lines 3882-3888
_task_fn_admin(void *context)
Link Here
|
| 3882 |
iflib_txq_t txq; |
3877 |
iflib_txq_t txq; |
| 3883 |
int i; |
3878 |
int i; |
| 3884 |
bool oactive, running, do_reset, do_watchdog, in_detach; |
3879 |
bool oactive, running, do_reset, do_watchdog, in_detach; |
| 3885 |
uint32_t reset_on = hz / 2; |
|
|
| 3886 |
|
3880 |
|
| 3887 |
STATE_LOCK(ctx); |
3881 |
STATE_LOCK(ctx); |
| 3888 |
running = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING); |
3882 |
running = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING); |
|
Lines 3910-3921
_task_fn_admin(void *context)
Link Here
|
| 3910 |
} |
3904 |
} |
| 3911 |
IFDI_UPDATE_ADMIN_STATUS(ctx); |
3905 |
IFDI_UPDATE_ADMIN_STATUS(ctx); |
| 3912 |
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { |
3906 |
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { |
| 3913 |
#ifdef DEV_NETMAP |
3907 |
callout_reset_on(&txq->ift_timer, hz / 2, iflib_timer, txq, |
| 3914 |
reset_on = hz / 2; |
3908 |
txq->ift_timer.c_cpu); |
| 3915 |
if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) |
|
|
| 3916 |
iflib_netmap_timer_adjust(ctx, txq, &reset_on); |
| 3917 |
#endif |
| 3918 |
callout_reset_on(&txq->ift_timer, reset_on, iflib_timer, txq, txq->ift_timer.c_cpu); |
| 3919 |
} |
3909 |
} |
| 3920 |
IFDI_LINK_INTR_ENABLE(ctx); |
3910 |
IFDI_LINK_INTR_ENABLE(ctx); |
| 3921 |
if (do_reset) |
3911 |
if (do_reset) |
|
Lines 5088-5093
iflib_pseudo_deregister(if_ctx_t ctx)
Link Here
|
| 5088 |
tqg = qgroup_if_io_tqg; |
5078 |
tqg = qgroup_if_io_tqg; |
| 5089 |
for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { |
5079 |
for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { |
| 5090 |
callout_drain(&txq->ift_timer); |
5080 |
callout_drain(&txq->ift_timer); |
|
|
5081 |
#ifdef DEV_NETMAP |
| 5082 |
callout_drain(&txq->ift_netmap_timer); |
| 5083 |
#endif /* DEV_NETMAP */ |
| 5091 |
if (txq->ift_task.gt_uniq != NULL) |
5084 |
if (txq->ift_task.gt_uniq != NULL) |
| 5092 |
taskqgroup_detach(tqg, &txq->ift_task); |
5085 |
taskqgroup_detach(tqg, &txq->ift_task); |
| 5093 |
} |
5086 |
} |
|
Lines 5174-5179
iflib_device_deregister(if_ctx_t ctx)
Link Here
|
| 5174 |
tqg = qgroup_if_io_tqg; |
5167 |
tqg = qgroup_if_io_tqg; |
| 5175 |
for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { |
5168 |
for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { |
| 5176 |
callout_drain(&txq->ift_timer); |
5169 |
callout_drain(&txq->ift_timer); |
|
|
5170 |
#ifdef DEV_NETMAP |
| 5171 |
callout_drain(&txq->ift_netmap_timer); |
| 5172 |
#endif /* DEV_NETMAP */ |
| 5177 |
if (txq->ift_task.gt_uniq != NULL) |
5173 |
if (txq->ift_task.gt_uniq != NULL) |
| 5178 |
taskqgroup_detach(tqg, &txq->ift_task); |
5174 |
taskqgroup_detach(tqg, &txq->ift_task); |
| 5179 |
} |
5175 |
} |
|
Lines 5583-5590
iflib_queues_alloc(if_ctx_t ctx)
Link Here
|
| 5583 |
} else { |
5579 |
} else { |
| 5584 |
txq->ift_br_offset = 0; |
5580 |
txq->ift_br_offset = 0; |
| 5585 |
} |
5581 |
} |
| 5586 |
/* XXX fix this */ |
|
|
| 5587 |
txq->ift_timer.c_cpu = cpu; |
| 5588 |
|
5582 |
|
| 5589 |
if (iflib_txsd_alloc(txq)) { |
5583 |
if (iflib_txsd_alloc(txq)) { |
| 5590 |
device_printf(dev, "Critical Failure setting up TX buffers\n"); |
5584 |
device_printf(dev, "Critical Failure setting up TX buffers\n"); |
|
Lines 5597-5602
iflib_queues_alloc(if_ctx_t ctx)
Link Here
|
| 5597 |
device_get_nameunit(dev), txq->ift_id); |
5591 |
device_get_nameunit(dev), txq->ift_id); |
| 5598 |
mtx_init(&txq->ift_mtx, txq->ift_mtx_name, NULL, MTX_DEF); |
5592 |
mtx_init(&txq->ift_mtx, txq->ift_mtx_name, NULL, MTX_DEF); |
| 5599 |
callout_init_mtx(&txq->ift_timer, &txq->ift_mtx, 0); |
5593 |
callout_init_mtx(&txq->ift_timer, &txq->ift_mtx, 0); |
|
|
5594 |
txq->ift_timer.c_cpu = cpu; |
| 5595 |
#ifdef DEV_NETMAP |
| 5596 |
callout_init_mtx(&txq->ift_netmap_timer, &txq->ift_mtx, 0); |
| 5597 |
txq->ift_netmap_timer.c_cpu = cpu; |
| 5598 |
#endif /* DEV_NETMAP */ |
| 5600 |
|
5599 |
|
| 5601 |
err = ifmp_ring_alloc(&txq->ift_br, 2048, txq, iflib_txq_drain, |
5600 |
err = ifmp_ring_alloc(&txq->ift_br, 2048, txq, iflib_txq_drain, |
| 5602 |
iflib_txq_can_drain, M_IFLIB, M_WAITOK); |
5601 |
iflib_txq_can_drain, M_IFLIB, M_WAITOK); |