Lines 216-221
static int arc_dead;
Link Here
|
216 |
extern boolean_t zfs_prefetch_disable; |
216 |
extern boolean_t zfs_prefetch_disable; |
217 |
|
217 |
|
218 |
/* |
218 |
/* |
|
|
219 |
* KD 2015-02-10 |
220 |
* We have to be able to test for UIO use inside the arc allocator. |
221 |
* NOTE: DO NOT MODIFY HERE! |
222 |
*/ |
223 |
extern int zio_use_uma; |
224 |
extern int zfs_dynamic_write_buffer; |
225 |
|
226 |
|
227 |
/* |
219 |
* The arc has filled available memory and has now warmed up. |
228 |
* The arc has filled available memory and has now warmed up. |
220 |
*/ |
229 |
*/ |
221 |
static boolean_t arc_warm; |
230 |
static boolean_t arc_warm; |
Lines 233-239
int zfs_arc_p_min_shift = 0;
Link Here
|
233 |
int zfs_disable_dup_eviction = 0; |
242 |
int zfs_disable_dup_eviction = 0; |
234 |
uint64_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */ |
243 |
uint64_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */ |
235 |
u_int zfs_arc_free_target = 0; |
244 |
u_int zfs_arc_free_target = 0; |
|
|
245 |
u_int zfs_arc_wakeup_pager = 0; |
246 |
u_int zfs_arc_wakeup_delay = 500; |
236 |
|
247 |
|
|
|
248 |
#define WAKE_PAGER |
249 |
#ifdef WAKE_PAGER |
250 |
#define WAKE_PAGER_CONSTANT 10 / 9 /* Pager wakeup threshold */ |
251 |
static int arc_init_done = 0; /* We know arc_warm is valid */ |
252 |
#endif /* WAKE_PAGER */ |
253 |
|
237 |
static int sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS); |
254 |
static int sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS); |
238 |
static int sysctl_vfs_zfs_arc_meta_limit(SYSCTL_HANDLER_ARGS); |
255 |
static int sysctl_vfs_zfs_arc_meta_limit(SYSCTL_HANDLER_ARGS); |
239 |
|
256 |
|
Lines 242-248
static void
Link Here
|
242 |
arc_free_target_init(void *unused __unused) |
259 |
arc_free_target_init(void *unused __unused) |
243 |
{ |
260 |
{ |
244 |
|
261 |
|
245 |
zfs_arc_free_target = vm_pageout_wakeup_thresh; |
262 |
zfs_arc_free_target = vm_pageout_wakeup_thresh + ((cnt.v_free_target - vm_pageout_wakeup_thresh) / 2); |
|
|
263 |
#ifdef WAKE_PAGER |
264 |
zfs_arc_wakeup_pager = zfs_arc_free_target * WAKE_PAGER_CONSTANT; |
265 |
#endif /* WAKE_PAGER */ |
246 |
} |
266 |
} |
247 |
SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY, |
267 |
SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY, |
248 |
arc_free_target_init, NULL); |
268 |
arc_free_target_init, NULL); |
Lines 264-270
SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_average_block
Link Here
|
264 |
SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_shrink_shift, CTLFLAG_RW, |
284 |
SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_shrink_shift, CTLFLAG_RW, |
265 |
&arc_shrink_shift, 0, |
285 |
&arc_shrink_shift, 0, |
266 |
"log2(fraction of arc to reclaim)"); |
286 |
"log2(fraction of arc to reclaim)"); |
267 |
|
287 |
SYSCTL_INT(_vfs_zfs, OID_AUTO, dynamic_write_buffer, CTLFLAG_RWTUN, |
|
|
288 |
&zfs_dynamic_write_buffer, 0, |
289 |
"Dynamically restrict dirty data when memory is low"); |
290 |
#ifdef WAKE_PAGER |
291 |
SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_wakeup_pager, CTLFLAG_RWTUN, |
292 |
&zfs_arc_wakeup_pager, 0, "Wake VM below this number of pages"); |
293 |
SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_wakeup_delay, CTLFLAG_RWTUN, |
294 |
&zfs_arc_wakeup_delay, 0, "May wake up VM once this number of MS"); |
295 |
#endif /* WAKE_PAGER */ |
268 |
/* |
296 |
/* |
269 |
* We don't have a tunable for arc_free_target due to the dependency on |
297 |
* We don't have a tunable for arc_free_target due to the dependency on |
270 |
* pagedaemon initialisation. |
298 |
* pagedaemon initialisation. |
Lines 291-296
sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS
Link Here
|
291 |
return (EINVAL); |
319 |
return (EINVAL); |
292 |
|
320 |
|
293 |
zfs_arc_free_target = val; |
321 |
zfs_arc_free_target = val; |
|
|
322 |
#ifdef WAKE_PAGER |
323 |
zfs_arc_wakeup_pager = zfs_arc_free_target * WAKE_PAGER_CONSTANT; |
324 |
#endif /* WAKE_PAGER */ |
294 |
|
325 |
|
295 |
return (0); |
326 |
return (0); |
296 |
} |
327 |
} |
Lines 3367-3372
int64_t arc_pages_pp_reserve = 64;
Link Here
|
3367 |
int64_t arc_swapfs_reserve = 64; |
3398 |
int64_t arc_swapfs_reserve = 64; |
3368 |
|
3399 |
|
3369 |
/* |
3400 |
/* |
|
|
3401 |
* Declare file-local static for event processor bypass |
3402 |
*/ |
3403 |
static unsigned int arc_no_wake_event = 0; |
3404 |
|
3405 |
/* |
3370 |
* Return the amount of memory that can be consumed before reclaim will be |
3406 |
* Return the amount of memory that can be consumed before reclaim will be |
3371 |
* needed. Positive if there is sufficient free memory, negative indicates |
3407 |
* needed. Positive if there is sufficient free memory, negative indicates |
3372 |
* the amount of memory that needs to be freed up. |
3408 |
* the amount of memory that needs to be freed up. |
Lines 3379-3384
arc_available_memory(void)
Link Here
|
3379 |
free_memory_reason_t r = FMR_UNKNOWN; |
3415 |
free_memory_reason_t r = FMR_UNKNOWN; |
3380 |
|
3416 |
|
3381 |
#ifdef _KERNEL |
3417 |
#ifdef _KERNEL |
|
|
3418 |
#ifdef WAKE_PAGER |
3419 |
sbintime_t now; |
3420 |
static sbintime_t last_pagedaemon_wake = 0; |
3421 |
#endif /* WAKE_PAGER */ |
3382 |
if (needfree > 0) { |
3422 |
if (needfree > 0) { |
3383 |
n = PAGESIZE * (-needfree); |
3423 |
n = PAGESIZE * (-needfree); |
3384 |
if (n < lowest) { |
3424 |
if (n < lowest) { |
Lines 3397-3402
arc_available_memory(void)
Link Here
|
3397 |
r = FMR_LOTSFREE; |
3437 |
r = FMR_LOTSFREE; |
3398 |
} |
3438 |
} |
3399 |
|
3439 |
|
|
|
3440 |
#ifdef WAKE_PAGER |
3441 |
/* |
3442 |
* If memory is less than the ARC wakeup threshold and time has expired since |
3443 |
* the last time we woke the pager... Do not execute until the ARC warms up. |
3444 |
*/ |
3445 |
if ((arc_init_done) && |
3446 |
(((int64_t) freemem - zfs_arc_wakeup_pager) < 0) && |
3447 |
(arc_warm == B_TRUE) |
3448 |
) { |
3449 |
now = getsbinuptime(); |
3450 |
if ((now - last_pagedaemon_wake) / SBT_1MS > zfs_arc_wakeup_delay) { |
3451 |
last_pagedaemon_wake = now; |
3452 |
arc_no_wake_event++; /* Set bypass flag for ARC */ |
3453 |
DTRACE_PROBE(arc__wake_pagedaemon); |
3454 |
pagedaemon_wakeup(); /* Wake the pager */ |
3455 |
} |
3456 |
} |
3457 |
|
3458 |
#endif /* WAKE_PAGER */ |
3459 |
|
3400 |
#ifdef sun |
3460 |
#ifdef sun |
3401 |
/* |
3461 |
/* |
3402 |
* check that we're out of range of the pageout scanner. It starts to |
3462 |
* check that we're out of range of the pageout scanner. It starts to |
Lines 3505-3510
arc_available_memory(void)
Link Here
|
3505 |
last_free_memory = lowest; |
3565 |
last_free_memory = lowest; |
3506 |
last_free_reason = r; |
3566 |
last_free_reason = r; |
3507 |
DTRACE_PROBE2(arc__available_memory, int64_t, lowest, int, r); |
3567 |
DTRACE_PROBE2(arc__available_memory, int64_t, lowest, int, r); |
|
|
3568 |
|
3508 |
return (lowest); |
3569 |
return (lowest); |
3509 |
} |
3570 |
} |
3510 |
|
3571 |
|
Lines 3524-3529
extern kmem_cache_t *zio_buf_cache[];
Link Here
|
3524 |
extern kmem_cache_t *zio_data_buf_cache[]; |
3585 |
extern kmem_cache_t *zio_data_buf_cache[]; |
3525 |
extern kmem_cache_t *range_seg_cache; |
3586 |
extern kmem_cache_t *range_seg_cache; |
3526 |
|
3587 |
|
|
|
3588 |
/* |
3589 |
* Used by arc_kmem_reap_now() and consider_reaping_arc_caches() |
3590 |
* to limit the time spent reaping. |
3591 |
* |
3592 |
* The arc_reaping_in_progress is a (somewhat racy) left over from a |
3593 |
* previous version of this code which could trigger multiple ARC cache |
3594 |
* reapings in parallel which should be avoided to reduce lock |
3595 |
* contention. It's hasn't been removed yet to encourage further |
3596 |
* experimenting. |
3597 |
*/ |
3598 |
static unsigned int arc_reaping_in_progress = 0; |
3599 |
static unsigned int arc_pagedaemon_ignore = 0; |
3600 |
static sbintime_t last_reaping = 0; |
3601 |
|
3602 |
static void __noinline |
3603 |
reap_arc_caches(void) |
3604 |
{ |
3605 |
size_t i; |
3606 |
kmem_cache_t *prev_cache = NULL; |
3607 |
kmem_cache_t *prev_data_cache = NULL; |
3608 |
|
3609 |
for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { |
3610 |
if (zio_buf_cache[i] != prev_cache) { |
3611 |
prev_cache = zio_buf_cache[i]; |
3612 |
kmem_cache_reap_now(zio_buf_cache[i]); |
3613 |
} |
3614 |
if (zio_data_buf_cache[i] != prev_data_cache) { |
3615 |
prev_data_cache = zio_data_buf_cache[i]; |
3616 |
kmem_cache_reap_now(zio_data_buf_cache[i]); |
3617 |
} |
3618 |
} |
3619 |
kmem_cache_reap_now(buf_cache); |
3620 |
kmem_cache_reap_now(hdr_full_cache); |
3621 |
kmem_cache_reap_now(hdr_l2only_cache); |
3622 |
kmem_cache_reap_now(range_seg_cache); |
3623 |
} |
3624 |
|
3527 |
static __noinline void |
3625 |
static __noinline void |
3528 |
arc_kmem_reap_now(void) |
3626 |
arc_kmem_reap_now(void) |
3529 |
{ |
3627 |
{ |
Lines 3532-3537
arc_kmem_reap_now(void)
Link Here
|
3532 |
kmem_cache_t *prev_data_cache = NULL; |
3630 |
kmem_cache_t *prev_data_cache = NULL; |
3533 |
|
3631 |
|
3534 |
DTRACE_PROBE(arc__kmem_reap_start); |
3632 |
DTRACE_PROBE(arc__kmem_reap_start); |
|
|
3633 |
arc_reaping_in_progress++; |
3634 |
|
3535 |
#ifdef _KERNEL |
3635 |
#ifdef _KERNEL |
3536 |
if (arc_meta_used >= arc_meta_limit) { |
3636 |
if (arc_meta_used >= arc_meta_limit) { |
3537 |
/* |
3637 |
/* |
Lines 3548-3567
arc_kmem_reap_now(void)
Link Here
|
3548 |
#endif |
3648 |
#endif |
3549 |
#endif |
3649 |
#endif |
3550 |
|
3650 |
|
3551 |
for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { |
3651 |
reap_arc_caches(); |
3552 |
if (zio_buf_cache[i] != prev_cache) { |
|
|
3553 |
prev_cache = zio_buf_cache[i]; |
3554 |
kmem_cache_reap_now(zio_buf_cache[i]); |
3555 |
} |
3556 |
if (zio_data_buf_cache[i] != prev_data_cache) { |
3557 |
prev_data_cache = zio_data_buf_cache[i]; |
3558 |
kmem_cache_reap_now(zio_data_buf_cache[i]); |
3559 |
} |
3560 |
} |
3561 |
kmem_cache_reap_now(buf_cache); |
3562 |
kmem_cache_reap_now(hdr_full_cache); |
3563 |
kmem_cache_reap_now(hdr_l2only_cache); |
3564 |
kmem_cache_reap_now(range_seg_cache); |
3565 |
|
3652 |
|
3566 |
#ifdef sun |
3653 |
#ifdef sun |
3567 |
if (zio_arena != NULL) { |
3654 |
if (zio_arena != NULL) { |
Lines 3572-3581
arc_kmem_reap_now(void)
Link Here
|
3572 |
vmem_qcache_reap(zio_arena); |
3659 |
vmem_qcache_reap(zio_arena); |
3573 |
} |
3660 |
} |
3574 |
#endif |
3661 |
#endif |
|
|
3662 |
#ifdef _KERNEL |
3663 |
last_reaping = getsbinuptime(); |
3664 |
#endif |
3665 |
arc_reaping_in_progress = 0; |
3575 |
DTRACE_PROBE(arc__kmem_reap_end); |
3666 |
DTRACE_PROBE(arc__kmem_reap_end); |
3576 |
} |
3667 |
} |
3577 |
|
3668 |
|
|
|
3669 |
|
3578 |
/* |
3670 |
/* |
|
|
3671 |
* Declared writable to allow resetting it. |
3672 |
* XXX: Should probably be a uint64 and integrated with kstat. |
3673 |
*/ |
3674 |
static unsigned int arc_cache_reapings_skipped = 0; |
3675 |
SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_cache_reapings_skipped, CTLFLAG_RW, |
3676 |
&arc_cache_reapings_skipped, 0, "Number of times the ARC caches have not been reaped due to the reap delay"); |
3677 |
|
3678 |
static unsigned int min_arc_reap_delay = 200; |
3679 |
SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_reap_delay_min, CTLFLAG_RW, |
3680 |
&min_arc_reap_delay, 200, "Minimum delay between ARC cache reapings (milliseconds)"); |
3681 |
|
3682 |
static void __noinline |
3683 |
consider_reaping_arc_caches(void) |
3684 |
{ |
3685 |
#ifdef _KERNEL |
3686 |
sbintime_t now; |
3687 |
|
3688 |
if (arc_reaping_in_progress) |
3689 |
{ |
3690 |
/* Already reaping in another thread. */ |
3691 |
arc_cache_reapings_skipped++; |
3692 |
return; |
3693 |
} |
3694 |
|
3695 |
now = getsbinuptime(); |
3696 |
if ((now - last_reaping) / SBT_1MS < min_arc_reap_delay) |
3697 |
{ |
3698 |
/* Too soon to reap again. */ |
3699 |
arc_cache_reapings_skipped++; |
3700 |
return; |
3701 |
} |
3702 |
#endif |
3703 |
arc_kmem_reap_now(); |
3704 |
} |
3705 |
|
3706 |
/* |
3579 |
* Threads can block in arc_get_data_buf() waiting for this thread to evict |
3707 |
* Threads can block in arc_get_data_buf() waiting for this thread to evict |
3580 |
* enough data and signal them to proceed. When this happens, the threads in |
3708 |
* enough data and signal them to proceed. When this happens, the threads in |
3581 |
* arc_get_data_buf() are sleeping while holding the hash lock for their |
3709 |
* arc_get_data_buf() are sleeping while holding the hash lock for their |
Lines 3617-3624
arc_reclaim_thread(void *dummy __unused)
Link Here
|
3617 |
*/ |
3745 |
*/ |
3618 |
growtime = ddi_get_lbolt() + (arc_grow_retry * hz); |
3746 |
growtime = ddi_get_lbolt() + (arc_grow_retry * hz); |
3619 |
|
3747 |
|
3620 |
arc_kmem_reap_now(); |
|
|
3621 |
|
3622 |
/* |
3748 |
/* |
3623 |
* If we are still low on memory, shrink the ARC |
3749 |
* If we are still low on memory, shrink the ARC |
3624 |
* so that we have arc_shrink_min free space. |
3750 |
* so that we have arc_shrink_min free space. |
Lines 3692-3697
arc_user_evicts_thread(void *dummy __unused)
Link Here
|
3692 |
while (!arc_user_evicts_thread_exit) { |
3818 |
while (!arc_user_evicts_thread_exit) { |
3693 |
mutex_exit(&arc_user_evicts_lock); |
3819 |
mutex_exit(&arc_user_evicts_lock); |
3694 |
|
3820 |
|
|
|
3821 |
/* |
3822 |
* Consider reaping the ARC caches at least once per |
3823 |
* second, but more often when signalled under pressure. |
3824 |
*/ |
3825 |
consider_reaping_arc_caches(); |
3826 |
|
3695 |
arc_do_user_evicts(); |
3827 |
arc_do_user_evicts(); |
3696 |
|
3828 |
|
3697 |
/* |
3829 |
/* |
Lines 5243-5249
static eventhandler_tag arc_event_lowmem = NULL;
Link Here
|
5243 |
static void |
5375 |
static void |
5244 |
arc_lowmem(void *arg __unused, int howto __unused) |
5376 |
arc_lowmem(void *arg __unused, int howto __unused) |
5245 |
{ |
5377 |
{ |
5246 |
|
5378 |
if (arc_no_wake_event) { /* Don't do it if we woke the pager */ |
|
|
5379 |
arc_no_wake_event = 0; /* Just clear the flag */ |
5380 |
return; |
5381 |
} |
5247 |
mutex_enter(&arc_reclaim_lock); |
5382 |
mutex_enter(&arc_reclaim_lock); |
5248 |
/* XXX: Memory deficit should be passed as argument. */ |
5383 |
/* XXX: Memory deficit should be passed as argument. */ |
5249 |
needfree = btoc(arc_c >> arc_shrink_shift); |
5384 |
needfree = btoc(arc_c >> arc_shrink_shift); |
Lines 5491-5496
arc_init(void)
Link Here
|
5491 |
printf(" in /boot/loader.conf.\n"); |
5626 |
printf(" in /boot/loader.conf.\n"); |
5492 |
} |
5627 |
} |
5493 |
#endif |
5628 |
#endif |
|
|
5629 |
#ifdef WAKE_PAGER |
5630 |
arc_init_done++; /* For anyone who wants to know */ |
5631 |
#endif /* WAKE_PAGER */ |
5494 |
} |
5632 |
} |
5495 |
|
5633 |
|
5496 |
void |
5634 |
void |