|
Lines 346-351
struct iflib_txq {
Link Here
|
| 346 |
qidx_t ift_size; |
346 |
qidx_t ift_size; |
| 347 |
uint16_t ift_id; |
347 |
uint16_t ift_id; |
| 348 |
struct callout ift_timer; |
348 |
struct callout ift_timer; |
|
|
349 |
#ifdef DEV_NETMAP |
| 350 |
struct callout ift_netmap_timer; |
| 351 |
#endif /* DEV_NETMAP */ |
| 349 |
|
352 |
|
| 350 |
if_txsd_vec_t ift_sds; |
353 |
if_txsd_vec_t ift_sds; |
| 351 |
uint8_t ift_qstatus; |
354 |
uint8_t ift_qstatus; |
|
Lines 753-758
iflib_num_tx_descs(if_ctx_t ctx)
Link Here
|
| 753 |
MODULE_DEPEND(iflib, netmap, 1, 1, 1); |
756 |
MODULE_DEPEND(iflib, netmap, 1, 1, 1); |
| 754 |
|
757 |
|
| 755 |
static int netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, bool init); |
758 |
static int netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, bool init); |
|
|
759 |
static void iflib_netmap_timer(void *arg); |
| 756 |
|
760 |
|
| 757 |
/* |
761 |
/* |
| 758 |
* device-specific sysctl variables: |
762 |
* device-specific sysctl variables: |
|
Lines 918-923
netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, bool init)
Link Here
|
| 918 |
return (0); |
922 |
return (0); |
| 919 |
} |
923 |
} |
| 920 |
|
924 |
|
|
|
925 |
#define NETMAP_TX_TIMER_US 90 |
| 926 |
|
| 921 |
/* |
927 |
/* |
| 922 |
* Reconcile kernel and user view of the transmit ring. |
928 |
* Reconcile kernel and user view of the transmit ring. |
| 923 |
* |
929 |
* |
|
Lines 1058-1068
iflib_netmap_txsync(struct netmap_kring *kring, int flags)
Link Here
|
| 1058 |
kring->nr_hwtail = nm_prev(netmap_idx_n2k(kring, nic_i), lim); |
1064 |
kring->nr_hwtail = nm_prev(netmap_idx_n2k(kring, nic_i), lim); |
| 1059 |
} |
1065 |
} |
| 1060 |
} |
1066 |
} |
|
|
1067 |
|
| 1061 |
if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ)) |
1068 |
if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ)) |
| 1062 |
if (kring->nr_hwtail != nm_prev(kring->nr_hwcur, lim)) { |
1069 |
if (kring->nr_hwtail != nm_prev(kring->nr_hwcur, lim)) { |
| 1063 |
callout_reset_on(&txq->ift_timer, hz < 2000 ? 1 : hz / 1000, |
1070 |
callout_reset_sbt(&txq->ift_netmap_timer, |
| 1064 |
iflib_timer, txq, txq->ift_timer.c_cpu); |
1071 |
NETMAP_TX_TIMER_US * SBT_1US, SBT_1US, |
| 1065 |
} |
1072 |
iflib_netmap_timer, txq, txq->ift_netmap_timer.c_cpu); |
|
|
1073 |
} |
| 1066 |
return (0); |
1074 |
return (0); |
| 1067 |
} |
1075 |
} |
| 1068 |
|
1076 |
|
|
Lines 1263-1290
iflib_netmap_rxq_init(if_ctx_t ctx, iflib_rxq_t rxq)
Link Here
|
| 1263 |
} |
1271 |
} |
| 1264 |
|
1272 |
|
| 1265 |
static void |
1273 |
static void |
| 1266 |
iflib_netmap_timer_adjust(if_ctx_t ctx, iflib_txq_t txq, uint32_t *reset_on) |
1274 |
iflib_netmap_timer(void *arg) |
| 1267 |
{ |
1275 |
{ |
| 1268 |
struct netmap_kring *kring; |
1276 |
iflib_txq_t txq = arg; |
| 1269 |
uint16_t txqid; |
1277 |
if_ctx_t ctx = txq->ift_ctx; |
| 1270 |
|
|
|
| 1271 |
txqid = txq->ift_id; |
| 1272 |
kring = netmap_kring_on(NA(ctx->ifc_ifp), txqid, NR_TX); |
| 1273 |
if (kring == NULL) |
| 1274 |
return; |
| 1275 |
|
1278 |
|
| 1276 |
if (kring->nr_hwcur != nm_next(kring->nr_hwtail, kring->nkr_num_slots - 1)) { |
1279 |
netmap_tx_irq(ctx->ifc_ifp, txq->ift_id); |
| 1277 |
bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, |
|
|
| 1278 |
BUS_DMASYNC_POSTREAD); |
| 1279 |
if (ctx->isc_txd_credits_update(ctx->ifc_softc, txqid, false)) |
| 1280 |
netmap_tx_irq(ctx->ifc_ifp, txqid); |
| 1281 |
if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ)) { |
| 1282 |
if (hz < 2000) |
| 1283 |
*reset_on = 1; |
| 1284 |
else |
| 1285 |
*reset_on = hz / 1000; |
| 1286 |
} |
| 1287 |
} |
| 1288 |
} |
1280 |
} |
| 1289 |
|
1281 |
|
| 1290 |
#define iflib_netmap_detach(ifp) netmap_detach(ifp) |
1282 |
#define iflib_netmap_detach(ifp) netmap_detach(ifp) |
|
Lines 1296-1303
iflib_netmap_timer_adjust(if_ctx_t ctx, iflib_txq_t txq, uint32_t *reset_on)
Link Here
|
| 1296 |
|
1288 |
|
| 1297 |
#define iflib_netmap_attach(ctx) (0) |
1289 |
#define iflib_netmap_attach(ctx) (0) |
| 1298 |
#define netmap_rx_irq(ifp, qid, budget) (0) |
1290 |
#define netmap_rx_irq(ifp, qid, budget) (0) |
| 1299 |
#define netmap_tx_irq(ifp, qid) do {} while (0) |
|
|
| 1300 |
#define iflib_netmap_timer_adjust(ctx, txq, reset_on) |
| 1301 |
#endif |
1291 |
#endif |
| 1302 |
|
1292 |
|
| 1303 |
#if defined(__i386__) || defined(__amd64__) |
1293 |
#if defined(__i386__) || defined(__amd64__) |
|
Lines 2287-2293
iflib_timer(void *arg)
Link Here
|
| 2287 |
if_ctx_t ctx = txq->ift_ctx; |
2277 |
if_ctx_t ctx = txq->ift_ctx; |
| 2288 |
if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; |
2278 |
if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; |
| 2289 |
uint64_t this_tick = ticks; |
2279 |
uint64_t this_tick = ticks; |
| 2290 |
uint32_t reset_on = hz / 2; |
|
|
| 2291 |
|
2280 |
|
| 2292 |
if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) |
2281 |
if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) |
| 2293 |
return; |
2282 |
return; |
|
Lines 2312-2328
iflib_timer(void *arg)
Link Here
|
| 2312 |
} |
2301 |
} |
| 2313 |
txq->ift_cleaned_prev = txq->ift_cleaned; |
2302 |
txq->ift_cleaned_prev = txq->ift_cleaned; |
| 2314 |
} |
2303 |
} |
| 2315 |
#ifdef DEV_NETMAP |
|
|
| 2316 |
if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) |
| 2317 |
iflib_netmap_timer_adjust(ctx, txq, &reset_on); |
| 2318 |
#endif |
| 2319 |
/* handle any laggards */ |
2304 |
/* handle any laggards */ |
| 2320 |
if (txq->ift_db_pending) |
2305 |
if (txq->ift_db_pending) |
| 2321 |
GROUPTASK_ENQUEUE(&txq->ift_task); |
2306 |
GROUPTASK_ENQUEUE(&txq->ift_task); |
| 2322 |
|
2307 |
|
| 2323 |
sctx->isc_pause_frames = 0; |
2308 |
sctx->isc_pause_frames = 0; |
| 2324 |
if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) |
2309 |
if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) |
| 2325 |
callout_reset_on(&txq->ift_timer, reset_on, iflib_timer, txq, txq->ift_timer.c_cpu); |
2310 |
callout_reset_on(&txq->ift_timer, hz / 2, iflib_timer, txq, txq->ift_timer.c_cpu); |
| 2326 |
return; |
2311 |
return; |
| 2327 |
|
2312 |
|
| 2328 |
hung: |
2313 |
hung: |
|
Lines 2396-2401
iflib_init_locked(if_ctx_t ctx)
Link Here
|
| 2396 |
for (i = 0, txq = ctx->ifc_txqs; i < sctx->isc_ntxqsets; i++, txq++) { |
2381 |
for (i = 0, txq = ctx->ifc_txqs; i < sctx->isc_ntxqsets; i++, txq++) { |
| 2397 |
CALLOUT_LOCK(txq); |
2382 |
CALLOUT_LOCK(txq); |
| 2398 |
callout_stop(&txq->ift_timer); |
2383 |
callout_stop(&txq->ift_timer); |
|
|
2384 |
#ifdef DEV_NETMAP |
| 2385 |
callout_stop(&txq->ift_netmap_timer); |
| 2386 |
#endif /* DEV_NETMAP */ |
| 2399 |
CALLOUT_UNLOCK(txq); |
2387 |
CALLOUT_UNLOCK(txq); |
| 2400 |
iflib_netmap_txq_init(ctx, txq); |
2388 |
iflib_netmap_txq_init(ctx, txq); |
| 2401 |
} |
2389 |
} |
|
Lines 2485-2490
iflib_stop(if_ctx_t ctx)
Link Here
|
| 2485 |
|
2473 |
|
| 2486 |
CALLOUT_LOCK(txq); |
2474 |
CALLOUT_LOCK(txq); |
| 2487 |
callout_stop(&txq->ift_timer); |
2475 |
callout_stop(&txq->ift_timer); |
|
|
2476 |
#ifdef DEV_NETMAP |
| 2477 |
callout_stop(&txq->ift_netmap_timer); |
| 2478 |
#endif /* DEV_NETMAP */ |
| 2488 |
CALLOUT_UNLOCK(txq); |
2479 |
CALLOUT_UNLOCK(txq); |
| 2489 |
|
2480 |
|
| 2490 |
/* clean any enqueued buffers */ |
2481 |
/* clean any enqueued buffers */ |
|
Lines 3799-3809
_task_fn_tx(void *context)
Link Here
|
| 3799 |
#endif |
3790 |
#endif |
| 3800 |
if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) |
3791 |
if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) |
| 3801 |
return; |
3792 |
return; |
| 3802 |
#ifdef DEV_NETMAP |
|
|
| 3803 |
if ((if_getcapenable(ifp) & IFCAP_NETMAP) && |
| 3804 |
netmap_tx_irq(ifp, txq->ift_id)) |
| 3805 |
goto skip_ifmp; |
| 3806 |
#endif |
| 3807 |
#ifdef ALTQ |
3793 |
#ifdef ALTQ |
| 3808 |
if (ALTQ_IS_ENABLED(&ifp->if_snd)) |
3794 |
if (ALTQ_IS_ENABLED(&ifp->if_snd)) |
| 3809 |
iflib_altq_if_start(ifp); |
3795 |
iflib_altq_if_start(ifp); |
|
Lines 3817-3825
_task_fn_tx(void *context)
Link Here
|
| 3817 |
*/ |
3803 |
*/ |
| 3818 |
if (abdicate) |
3804 |
if (abdicate) |
| 3819 |
ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); |
3805 |
ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); |
| 3820 |
#ifdef DEV_NETMAP |
|
|
| 3821 |
skip_ifmp: |
| 3822 |
#endif |
| 3823 |
if (ctx->ifc_flags & IFC_LEGACY) |
3806 |
if (ctx->ifc_flags & IFC_LEGACY) |
| 3824 |
IFDI_INTR_ENABLE(ctx); |
3807 |
IFDI_INTR_ENABLE(ctx); |
| 3825 |
else |
3808 |
else |
|
Lines 3882-3888
_task_fn_admin(void *context)
Link Here
|
| 3882 |
iflib_txq_t txq; |
3865 |
iflib_txq_t txq; |
| 3883 |
int i; |
3866 |
int i; |
| 3884 |
bool oactive, running, do_reset, do_watchdog, in_detach; |
3867 |
bool oactive, running, do_reset, do_watchdog, in_detach; |
| 3885 |
uint32_t reset_on = hz / 2; |
|
|
| 3886 |
|
3868 |
|
| 3887 |
STATE_LOCK(ctx); |
3869 |
STATE_LOCK(ctx); |
| 3888 |
running = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING); |
3870 |
running = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING); |
|
Lines 3910-3921
_task_fn_admin(void *context)
Link Here
|
| 3910 |
} |
3892 |
} |
| 3911 |
IFDI_UPDATE_ADMIN_STATUS(ctx); |
3893 |
IFDI_UPDATE_ADMIN_STATUS(ctx); |
| 3912 |
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { |
3894 |
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { |
| 3913 |
#ifdef DEV_NETMAP |
3895 |
callout_reset_on(&txq->ift_timer, hz / 2, iflib_timer, txq, |
| 3914 |
reset_on = hz / 2; |
3896 |
txq->ift_timer.c_cpu); |
| 3915 |
if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) |
|
|
| 3916 |
iflib_netmap_timer_adjust(ctx, txq, &reset_on); |
| 3917 |
#endif |
| 3918 |
callout_reset_on(&txq->ift_timer, reset_on, iflib_timer, txq, txq->ift_timer.c_cpu); |
| 3919 |
} |
3897 |
} |
| 3920 |
IFDI_LINK_INTR_ENABLE(ctx); |
3898 |
IFDI_LINK_INTR_ENABLE(ctx); |
| 3921 |
if (do_reset) |
3899 |
if (do_reset) |
|
Lines 5088-5093
iflib_pseudo_deregister(if_ctx_t ctx)
Link Here
|
| 5088 |
tqg = qgroup_if_io_tqg; |
5066 |
tqg = qgroup_if_io_tqg; |
| 5089 |
for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { |
5067 |
for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { |
| 5090 |
callout_drain(&txq->ift_timer); |
5068 |
callout_drain(&txq->ift_timer); |
|
|
5069 |
#ifdef DEV_NETMAP |
| 5070 |
callout_drain(&txq->ift_netmap_timer); |
| 5071 |
#endif /* DEV_NETMAP */ |
| 5091 |
if (txq->ift_task.gt_uniq != NULL) |
5072 |
if (txq->ift_task.gt_uniq != NULL) |
| 5092 |
taskqgroup_detach(tqg, &txq->ift_task); |
5073 |
taskqgroup_detach(tqg, &txq->ift_task); |
| 5093 |
} |
5074 |
} |
|
Lines 5174-5179
iflib_device_deregister(if_ctx_t ctx)
Link Here
|
| 5174 |
tqg = qgroup_if_io_tqg; |
5155 |
tqg = qgroup_if_io_tqg; |
| 5175 |
for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { |
5156 |
for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { |
| 5176 |
callout_drain(&txq->ift_timer); |
5157 |
callout_drain(&txq->ift_timer); |
|
|
5158 |
#ifdef DEV_NETMAP |
| 5159 |
callout_drain(&txq->ift_netmap_timer); |
| 5160 |
#endif /* DEV_NETMAP */ |
| 5177 |
if (txq->ift_task.gt_uniq != NULL) |
5161 |
if (txq->ift_task.gt_uniq != NULL) |
| 5178 |
taskqgroup_detach(tqg, &txq->ift_task); |
5162 |
taskqgroup_detach(tqg, &txq->ift_task); |
| 5179 |
} |
5163 |
} |
|
Lines 5583-5590
iflib_queues_alloc(if_ctx_t ctx)
Link Here
|
| 5583 |
} else { |
5567 |
} else { |
| 5584 |
txq->ift_br_offset = 0; |
5568 |
txq->ift_br_offset = 0; |
| 5585 |
} |
5569 |
} |
| 5586 |
/* XXX fix this */ |
|
|
| 5587 |
txq->ift_timer.c_cpu = cpu; |
| 5588 |
|
5570 |
|
| 5589 |
if (iflib_txsd_alloc(txq)) { |
5571 |
if (iflib_txsd_alloc(txq)) { |
| 5590 |
device_printf(dev, "Critical Failure setting up TX buffers\n"); |
5572 |
device_printf(dev, "Critical Failure setting up TX buffers\n"); |
|
Lines 5597-5602
iflib_queues_alloc(if_ctx_t ctx)
Link Here
|
| 5597 |
device_get_nameunit(dev), txq->ift_id); |
5579 |
device_get_nameunit(dev), txq->ift_id); |
| 5598 |
mtx_init(&txq->ift_mtx, txq->ift_mtx_name, NULL, MTX_DEF); |
5580 |
mtx_init(&txq->ift_mtx, txq->ift_mtx_name, NULL, MTX_DEF); |
| 5599 |
callout_init_mtx(&txq->ift_timer, &txq->ift_mtx, 0); |
5581 |
callout_init_mtx(&txq->ift_timer, &txq->ift_mtx, 0); |
|
|
5582 |
txq->ift_timer.c_cpu = cpu; |
| 5583 |
#ifdef DEV_NETMAP |
| 5584 |
callout_init_mtx(&txq->ift_netmap_timer, &txq->ift_mtx, 0); |
| 5585 |
txq->ift_netmap_timer.c_cpu = cpu; |
| 5586 |
#endif /* DEV_NETMAP */ |
| 5600 |
|
5587 |
|
| 5601 |
err = ifmp_ring_alloc(&txq->ift_br, 2048, txq, iflib_txq_drain, |
5588 |
err = ifmp_ring_alloc(&txq->ift_br, 2048, txq, iflib_txq_drain, |
| 5602 |
iflib_txq_can_drain, M_IFLIB, M_WAITOK); |
5589 |
iflib_txq_can_drain, M_IFLIB, M_WAITOK); |