|
Lines 190-195
struct iflib_ctx {
Link Here
|
| 190 |
uint16_t ifc_sysctl_nrxqs; |
190 |
uint16_t ifc_sysctl_nrxqs; |
| 191 |
uint16_t ifc_sysctl_qs_eq_override; |
191 |
uint16_t ifc_sysctl_qs_eq_override; |
| 192 |
uint16_t ifc_sysctl_rx_budget; |
192 |
uint16_t ifc_sysctl_rx_budget; |
|
|
193 |
uint16_t ifc_sysctl_nm_tx_tmr_us; |
| 193 |
uint16_t ifc_sysctl_tx_abdicate; |
194 |
uint16_t ifc_sysctl_tx_abdicate; |
| 194 |
uint16_t ifc_sysctl_core_offset; |
195 |
uint16_t ifc_sysctl_core_offset; |
| 195 |
#define CORE_OFFSET_UNSPECIFIED 0xffff |
196 |
#define CORE_OFFSET_UNSPECIFIED 0xffff |
|
Lines 346-351
struct iflib_txq {
Link Here
|
| 346 |
qidx_t ift_size; |
347 |
qidx_t ift_size; |
| 347 |
uint16_t ift_id; |
348 |
uint16_t ift_id; |
| 348 |
struct callout ift_timer; |
349 |
struct callout ift_timer; |
|
|
350 |
#ifdef DEV_NETMAP |
| 351 |
struct callout ift_netmap_timer; |
| 352 |
#endif /* DEV_NETMAP */ |
| 349 |
|
353 |
|
| 350 |
if_txsd_vec_t ift_sds; |
354 |
if_txsd_vec_t ift_sds; |
| 351 |
uint8_t ift_qstatus; |
355 |
uint8_t ift_qstatus; |
|
Lines 753-758
iflib_num_tx_descs(if_ctx_t ctx)
Link Here
|
| 753 |
MODULE_DEPEND(iflib, netmap, 1, 1, 1); |
757 |
MODULE_DEPEND(iflib, netmap, 1, 1, 1); |
| 754 |
|
758 |
|
| 755 |
static int netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, bool init); |
759 |
static int netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, bool init); |
|
|
760 |
static void iflib_netmap_timer(void *arg); |
| 756 |
|
761 |
|
| 757 |
/* |
762 |
/* |
| 758 |
* device-specific sysctl variables: |
763 |
* device-specific sysctl variables: |
|
Lines 1060-1068
iflib_netmap_txsync(struct netmap_kring *kring, int flags)
Link Here
|
| 1060 |
} |
1065 |
} |
| 1061 |
if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ)) |
1066 |
if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ)) |
| 1062 |
if (kring->nr_hwtail != nm_prev(kring->nr_hwcur, lim)) { |
1067 |
if (kring->nr_hwtail != nm_prev(kring->nr_hwcur, lim)) { |
| 1063 |
callout_reset_on(&txq->ift_timer, hz < 2000 ? 1 : hz / 1000, |
1068 |
uint16_t timer_us = ctx->ifc_sysctl_nm_tx_tmr_us; |
| 1064 |
iflib_timer, txq, txq->ift_timer.c_cpu); |
1069 |
|
| 1065 |
} |
1070 |
if (timer_us == 0) |
|
|
1071 |
timer_us = 500; |
| 1072 |
callout_reset_sbt(&txq->ift_netmap_timer, |
| 1073 |
timer_us * SBT_1US, SBT_1US, iflib_netmap_timer, |
| 1074 |
txq, 0); |
| 1075 |
} |
| 1066 |
return (0); |
1076 |
return (0); |
| 1067 |
} |
1077 |
} |
| 1068 |
|
1078 |
|
|
Lines 1263-1290
iflib_netmap_rxq_init(if_ctx_t ctx, iflib_rxq_t rxq)
Link Here
|
| 1263 |
} |
1273 |
} |
| 1264 |
|
1274 |
|
| 1265 |
static void |
1275 |
static void |
| 1266 |
iflib_netmap_timer_adjust(if_ctx_t ctx, iflib_txq_t txq, uint32_t *reset_on) |
1276 |
iflib_netmap_timer(void *arg) |
| 1267 |
{ |
1277 |
{ |
| 1268 |
struct netmap_kring *kring; |
1278 |
iflib_txq_t txq = arg; |
| 1269 |
uint16_t txqid; |
1279 |
if_ctx_t ctx = txq->ift_ctx; |
| 1270 |
|
|
|
| 1271 |
txqid = txq->ift_id; |
| 1272 |
kring = netmap_kring_on(NA(ctx->ifc_ifp), txqid, NR_TX); |
| 1273 |
if (kring == NULL) |
| 1274 |
return; |
| 1275 |
|
1280 |
|
| 1276 |
if (kring->nr_hwcur != nm_next(kring->nr_hwtail, kring->nkr_num_slots - 1)) { |
1281 |
netmap_tx_irq(ctx->ifc_ifp, txq->ift_id); |
| 1277 |
bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, |
|
|
| 1278 |
BUS_DMASYNC_POSTREAD); |
| 1279 |
if (ctx->isc_txd_credits_update(ctx->ifc_softc, txqid, false)) |
| 1280 |
netmap_tx_irq(ctx->ifc_ifp, txqid); |
| 1281 |
if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ)) { |
| 1282 |
if (hz < 2000) |
| 1283 |
*reset_on = 1; |
| 1284 |
else |
| 1285 |
*reset_on = hz / 1000; |
| 1286 |
} |
| 1287 |
} |
| 1288 |
} |
1282 |
} |
| 1289 |
|
1283 |
|
| 1290 |
#define iflib_netmap_detach(ifp) netmap_detach(ifp) |
1284 |
#define iflib_netmap_detach(ifp) netmap_detach(ifp) |
|
Lines 1296-1303
iflib_netmap_timer_adjust(if_ctx_t ctx, iflib_txq_t txq, uint32_t *reset_on)
Link Here
|
| 1296 |
|
1290 |
|
| 1297 |
#define iflib_netmap_attach(ctx) (0) |
1291 |
#define iflib_netmap_attach(ctx) (0) |
| 1298 |
#define netmap_rx_irq(ifp, qid, budget) (0) |
1292 |
#define netmap_rx_irq(ifp, qid, budget) (0) |
| 1299 |
#define netmap_tx_irq(ifp, qid) do {} while (0) |
|
|
| 1300 |
#define iflib_netmap_timer_adjust(ctx, txq, reset_on) |
| 1301 |
#endif |
1293 |
#endif |
| 1302 |
|
1294 |
|
| 1303 |
#if defined(__i386__) || defined(__amd64__) |
1295 |
#if defined(__i386__) || defined(__amd64__) |
|
Lines 2287-2293
iflib_timer(void *arg)
Link Here
|
| 2287 |
if_ctx_t ctx = txq->ift_ctx; |
2279 |
if_ctx_t ctx = txq->ift_ctx; |
| 2288 |
if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; |
2280 |
if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; |
| 2289 |
uint64_t this_tick = ticks; |
2281 |
uint64_t this_tick = ticks; |
| 2290 |
uint32_t reset_on = hz / 2; |
|
|
| 2291 |
|
2282 |
|
| 2292 |
if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) |
2283 |
if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) |
| 2293 |
return; |
2284 |
return; |
|
Lines 2312-2328
iflib_timer(void *arg)
Link Here
|
| 2312 |
} |
2303 |
} |
| 2313 |
txq->ift_cleaned_prev = txq->ift_cleaned; |
2304 |
txq->ift_cleaned_prev = txq->ift_cleaned; |
| 2314 |
} |
2305 |
} |
| 2315 |
#ifdef DEV_NETMAP |
|
|
| 2316 |
if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) |
| 2317 |
iflib_netmap_timer_adjust(ctx, txq, &reset_on); |
| 2318 |
#endif |
| 2319 |
/* handle any laggards */ |
2306 |
/* handle any laggards */ |
| 2320 |
if (txq->ift_db_pending) |
2307 |
if (txq->ift_db_pending) |
| 2321 |
GROUPTASK_ENQUEUE(&txq->ift_task); |
2308 |
GROUPTASK_ENQUEUE(&txq->ift_task); |
| 2322 |
|
2309 |
|
| 2323 |
sctx->isc_pause_frames = 0; |
2310 |
sctx->isc_pause_frames = 0; |
| 2324 |
if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) |
2311 |
if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) |
| 2325 |
callout_reset_on(&txq->ift_timer, reset_on, iflib_timer, txq, txq->ift_timer.c_cpu); |
2312 |
callout_reset_on(&txq->ift_timer, hz / 2, iflib_timer, txq, txq->ift_timer.c_cpu); |
| 2326 |
return; |
2313 |
return; |
| 2327 |
|
2314 |
|
| 2328 |
hung: |
2315 |
hung: |
|
Lines 2396-2401
iflib_init_locked(if_ctx_t ctx)
Link Here
|
| 2396 |
for (i = 0, txq = ctx->ifc_txqs; i < sctx->isc_ntxqsets; i++, txq++) { |
2383 |
for (i = 0, txq = ctx->ifc_txqs; i < sctx->isc_ntxqsets; i++, txq++) { |
| 2397 |
CALLOUT_LOCK(txq); |
2384 |
CALLOUT_LOCK(txq); |
| 2398 |
callout_stop(&txq->ift_timer); |
2385 |
callout_stop(&txq->ift_timer); |
|
|
2386 |
#ifdef DEV_NETMAP |
| 2387 |
callout_stop(&txq->ift_netmap_timer); |
| 2388 |
#endif /* DEV_NETMAP */ |
| 2399 |
CALLOUT_UNLOCK(txq); |
2389 |
CALLOUT_UNLOCK(txq); |
| 2400 |
iflib_netmap_txq_init(ctx, txq); |
2390 |
iflib_netmap_txq_init(ctx, txq); |
| 2401 |
} |
2391 |
} |
|
Lines 2485-2490
iflib_stop(if_ctx_t ctx)
Link Here
|
| 2485 |
|
2475 |
|
| 2486 |
CALLOUT_LOCK(txq); |
2476 |
CALLOUT_LOCK(txq); |
| 2487 |
callout_stop(&txq->ift_timer); |
2477 |
callout_stop(&txq->ift_timer); |
|
|
2478 |
#ifdef DEV_NETMAP |
| 2479 |
callout_stop(&txq->ift_netmap_timer); |
| 2480 |
#endif /* DEV_NETMAP */ |
| 2488 |
CALLOUT_UNLOCK(txq); |
2481 |
CALLOUT_UNLOCK(txq); |
| 2489 |
|
2482 |
|
| 2490 |
/* clean any enqueued buffers */ |
2483 |
/* clean any enqueued buffers */ |
|
Lines 3882-3888
_task_fn_admin(void *context)
Link Here
|
| 3882 |
iflib_txq_t txq; |
3875 |
iflib_txq_t txq; |
| 3883 |
int i; |
3876 |
int i; |
| 3884 |
bool oactive, running, do_reset, do_watchdog, in_detach; |
3877 |
bool oactive, running, do_reset, do_watchdog, in_detach; |
| 3885 |
uint32_t reset_on = hz / 2; |
|
|
| 3886 |
|
3878 |
|
| 3887 |
STATE_LOCK(ctx); |
3879 |
STATE_LOCK(ctx); |
| 3888 |
running = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING); |
3880 |
running = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING); |
|
Lines 3910-3921
_task_fn_admin(void *context)
Link Here
|
| 3910 |
} |
3902 |
} |
| 3911 |
IFDI_UPDATE_ADMIN_STATUS(ctx); |
3903 |
IFDI_UPDATE_ADMIN_STATUS(ctx); |
| 3912 |
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { |
3904 |
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { |
| 3913 |
#ifdef DEV_NETMAP |
3905 |
callout_reset_on(&txq->ift_timer, hz / 2, iflib_timer, txq, txq->ift_timer.c_cpu); |
| 3914 |
reset_on = hz / 2; |
|
|
| 3915 |
if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) |
| 3916 |
iflib_netmap_timer_adjust(ctx, txq, &reset_on); |
| 3917 |
#endif |
| 3918 |
callout_reset_on(&txq->ift_timer, reset_on, iflib_timer, txq, txq->ift_timer.c_cpu); |
| 3919 |
} |
3906 |
} |
| 3920 |
IFDI_LINK_INTR_ENABLE(ctx); |
3907 |
IFDI_LINK_INTR_ENABLE(ctx); |
| 3921 |
if (do_reset) |
3908 |
if (do_reset) |
|
Lines 5088-5093
iflib_pseudo_deregister(if_ctx_t ctx)
Link Here
|
| 5088 |
tqg = qgroup_if_io_tqg; |
5075 |
tqg = qgroup_if_io_tqg; |
| 5089 |
for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { |
5076 |
for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { |
| 5090 |
callout_drain(&txq->ift_timer); |
5077 |
callout_drain(&txq->ift_timer); |
|
|
5078 |
#ifdef DEV_NETMAP |
| 5079 |
callout_drain(&txq->ift_netmap_timer); |
| 5080 |
#endif /* DEV_NETMAP */ |
| 5091 |
if (txq->ift_task.gt_uniq != NULL) |
5081 |
if (txq->ift_task.gt_uniq != NULL) |
| 5092 |
taskqgroup_detach(tqg, &txq->ift_task); |
5082 |
taskqgroup_detach(tqg, &txq->ift_task); |
| 5093 |
} |
5083 |
} |
|
Lines 5174-5179
iflib_device_deregister(if_ctx_t ctx)
Link Here
|
| 5174 |
tqg = qgroup_if_io_tqg; |
5164 |
tqg = qgroup_if_io_tqg; |
| 5175 |
for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { |
5165 |
for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { |
| 5176 |
callout_drain(&txq->ift_timer); |
5166 |
callout_drain(&txq->ift_timer); |
|
|
5167 |
#ifdef DEV_NETMAP |
| 5168 |
callout_drain(&txq->ift_netmap_timer); |
| 5169 |
#endif /* DEV_NETMAP */ |
| 5177 |
if (txq->ift_task.gt_uniq != NULL) |
5170 |
if (txq->ift_task.gt_uniq != NULL) |
| 5178 |
taskqgroup_detach(tqg, &txq->ift_task); |
5171 |
taskqgroup_detach(tqg, &txq->ift_task); |
| 5179 |
} |
5172 |
} |
|
Lines 5583-5590
iflib_queues_alloc(if_ctx_t ctx)
Link Here
|
| 5583 |
} else { |
5576 |
} else { |
| 5584 |
txq->ift_br_offset = 0; |
5577 |
txq->ift_br_offset = 0; |
| 5585 |
} |
5578 |
} |
| 5586 |
/* XXX fix this */ |
|
|
| 5587 |
txq->ift_timer.c_cpu = cpu; |
| 5588 |
|
5579 |
|
| 5589 |
if (iflib_txsd_alloc(txq)) { |
5580 |
if (iflib_txsd_alloc(txq)) { |
| 5590 |
device_printf(dev, "Critical Failure setting up TX buffers\n"); |
5581 |
device_printf(dev, "Critical Failure setting up TX buffers\n"); |
|
Lines 5597-5602
iflib_queues_alloc(if_ctx_t ctx)
Link Here
|
| 5597 |
device_get_nameunit(dev), txq->ift_id); |
5588 |
device_get_nameunit(dev), txq->ift_id); |
| 5598 |
mtx_init(&txq->ift_mtx, txq->ift_mtx_name, NULL, MTX_DEF); |
5589 |
mtx_init(&txq->ift_mtx, txq->ift_mtx_name, NULL, MTX_DEF); |
| 5599 |
callout_init_mtx(&txq->ift_timer, &txq->ift_mtx, 0); |
5590 |
callout_init_mtx(&txq->ift_timer, &txq->ift_mtx, 0); |
|
|
5591 |
#ifdef DEV_NETMAP |
| 5592 |
callout_init_mtx(&txq->ift_netmap_timer, &txq->ift_mtx, 0); |
| 5593 |
#endif /* DEV_NETMAP */ |
| 5600 |
|
5594 |
|
| 5601 |
err = ifmp_ring_alloc(&txq->ift_br, 2048, txq, iflib_txq_drain, |
5595 |
err = ifmp_ring_alloc(&txq->ift_br, 2048, txq, iflib_txq_drain, |
| 5602 |
iflib_txq_can_drain, M_IFLIB, M_WAITOK); |
5596 |
iflib_txq_can_drain, M_IFLIB, M_WAITOK); |
|
Lines 6610-6615
iflib_add_device_sysctl_pre(if_ctx_t ctx)
Link Here
|
| 6610 |
SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "rx_budget", |
6604 |
SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "rx_budget", |
| 6611 |
CTLFLAG_RWTUN, &ctx->ifc_sysctl_rx_budget, 0, |
6605 |
CTLFLAG_RWTUN, &ctx->ifc_sysctl_rx_budget, 0, |
| 6612 |
"set the RX budget"); |
6606 |
"set the RX budget"); |
|
|
6607 |
SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "nm_tx_tmr_us", |
| 6608 |
CTLFLAG_RWTUN, &ctx->ifc_sysctl_nm_tx_tmr_us, 0, |
| 6609 |
"set the TX netmap timer period"); |
| 6613 |
SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "tx_abdicate", |
6610 |
SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "tx_abdicate", |
| 6614 |
CTLFLAG_RWTUN, &ctx->ifc_sysctl_tx_abdicate, 0, |
6611 |
CTLFLAG_RWTUN, &ctx->ifc_sysctl_tx_abdicate, 0, |
| 6615 |
"cause TX to abdicate instead of running to completion"); |
6612 |
"cause TX to abdicate instead of running to completion"); |