Lines 138-143
Link Here
|
138 |
#include <sys/sdt.h> |
138 |
#include <sys/sdt.h> |
139 |
|
139 |
|
140 |
#include <vm/vm_pageout.h> |
140 |
#include <vm/vm_pageout.h> |
|
|
141 |
#include <machine/vmparam.h> |
141 |
|
142 |
|
142 |
#ifdef illumos |
143 |
#ifdef illumos |
143 |
#ifndef _KERNEL |
144 |
#ifndef _KERNEL |
Lines 159-164
typedef enum arc_reclaim_strategy {
Link Here
|
159 |
ARC_RECLAIM_CONS /* Conservative reclaim strategy */ |
160 |
ARC_RECLAIM_CONS /* Conservative reclaim strategy */ |
160 |
} arc_reclaim_strategy_t; |
161 |
} arc_reclaim_strategy_t; |
161 |
|
162 |
|
|
|
163 |
typedef enum arc_cache_reclaim_stragegy { |
164 |
ARC_CACHE_RECLAIM_NOW, /* Immediate reclaim strategy */ |
165 |
ARC_CACHE_RECLAIM_SIZE, /* Free size reclaim strategy */ |
166 |
ARC_CACHE_RECLAIM_FORCE, /* Forced immediate reclaim strategy */ |
167 |
} arc_cache_reclaim_strategy_t; |
168 |
|
169 |
/* When the last cache reclaim was processed. */ |
170 |
static clock_t arc_cache_reclaim_last = 0; |
171 |
|
162 |
/* |
172 |
/* |
163 |
* The number of iterations through arc_evict_*() before we |
173 |
* The number of iterations through arc_evict_*() before we |
164 |
* drop & reacquire the lock. |
174 |
* drop & reacquire the lock. |
Lines 193-201
extern int zfs_prefetch_disable;
Link Here
|
193 |
*/ |
203 |
*/ |
194 |
static boolean_t arc_warm; |
204 |
static boolean_t arc_warm; |
195 |
|
205 |
|
196 |
/* |
|
|
197 |
* These tunables are for performance analysis. |
198 |
*/ |
199 |
uint64_t zfs_arc_max; |
206 |
uint64_t zfs_arc_max; |
200 |
uint64_t zfs_arc_min; |
207 |
uint64_t zfs_arc_min; |
201 |
uint64_t zfs_arc_meta_limit = 0; |
208 |
uint64_t zfs_arc_meta_limit = 0; |
Lines 204-210
int zfs_arc_shrink_shift = 0;
Link Here
|
204 |
int zfs_arc_p_min_shift = 0; |
211 |
int zfs_arc_p_min_shift = 0; |
205 |
int zfs_disable_dup_eviction = 0; |
212 |
int zfs_disable_dup_eviction = 0; |
206 |
uint64_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */ |
213 |
uint64_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */ |
|
|
214 |
u_int zfs_arc_free_target = 0; |
215 |
u_int zfs_arc_cache_target = 0; |
216 |
int zfs_arc_cache_period = 10; |
217 |
int zfs_arc_cache_partial = 0; |
218 |
int zfs_arc_cache_free_period = 300; |
219 |
uint64_t zfs_arc_cache_free_max = (1 <<24); /* 16MB */ |
207 |
|
220 |
|
|
|
221 |
#ifdef _KERNEL |
222 |
static int sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS); |
223 |
static int sysctl_vfs_zfs_arc_cache_target(SYSCTL_HANDLER_ARGS); |
224 |
|
225 |
static void |
226 |
arc_target_init(void *unused __unused) |
227 |
{ |
228 |
|
229 |
zfs_arc_free_target = vm_pageout_wakeup_thresh; |
230 |
zfs_arc_cache_target = (vm_pageout_wakeup_thresh / 2) * 3; |
231 |
} |
232 |
SYSINIT(arc_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY, |
233 |
arc_target_init, NULL); |
234 |
|
208 |
TUNABLE_QUAD("vfs.zfs.arc_max", &zfs_arc_max); |
235 |
TUNABLE_QUAD("vfs.zfs.arc_max", &zfs_arc_max); |
209 |
TUNABLE_QUAD("vfs.zfs.arc_min", &zfs_arc_min); |
236 |
TUNABLE_QUAD("vfs.zfs.arc_min", &zfs_arc_min); |
210 |
TUNABLE_QUAD("vfs.zfs.arc_meta_limit", &zfs_arc_meta_limit); |
237 |
TUNABLE_QUAD("vfs.zfs.arc_meta_limit", &zfs_arc_meta_limit); |
Lines 217-223
SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_min, CTLFLAG_
Link Here
|
217 |
SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_average_blocksize, CTLFLAG_RDTUN, |
244 |
SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_average_blocksize, CTLFLAG_RDTUN, |
218 |
&zfs_arc_average_blocksize, 0, |
245 |
&zfs_arc_average_blocksize, 0, |
219 |
"ARC average blocksize"); |
246 |
"ARC average blocksize"); |
|
|
247 |
SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_cache_reclaim_period, CTLFLAG_RWTUN, |
248 |
&zfs_arc_cache_period, 0, |
249 |
"Min number of seconds between ARC cache reclaims"); |
250 |
SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_cache_reclaim_partial, CTLFLAG_RWTUN, |
251 |
&zfs_arc_cache_partial, 0, |
252 |
"Enable ARC to perform partial cache reclaims"); |
253 |
SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_cache_free_max, CTLFLAG_RWTUN, |
254 |
&zfs_arc_cache_free_max, 0, |
255 |
"Maximum free bytes in an ARC cache zone before reclaim will be triggered"); |
256 |
SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_cache_free_period, CTLFLAG_RWTUN, |
257 |
&zfs_arc_cache_free_period, 0, |
258 |
"Min number of seconds between ARC free size based cache reclaims"); |
259 |
/* |
260 |
* We don't have a tunable for these sysctls due to their dependency on |
261 |
* pagedaemon initialisation. |
262 |
*/ |
263 |
SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_free_target, |
264 |
CTLTYPE_UINT | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, sizeof(u_int), |
265 |
sysctl_vfs_zfs_arc_free_target, "IU", |
266 |
"Desired number of free pages below which ARC triggers reclaim"); |
267 |
SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_cache_target, |
268 |
CTLTYPE_UINT | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, sizeof(u_int), |
269 |
sysctl_vfs_zfs_arc_cache_target, "IU", |
270 |
"Desired number of free pages below which ARC triggers cache reclaim"); |
220 |
|
271 |
|
|
|
272 |
|
273 |
static int |
274 |
sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS) |
275 |
{ |
276 |
u_int val; |
277 |
int err; |
278 |
|
279 |
val = zfs_arc_free_target; |
280 |
err = sysctl_handle_int(oidp, &val, 0, req); |
281 |
if (err != 0 || req->newptr == NULL) |
282 |
return (err); |
283 |
|
284 |
if (val < minfree) |
285 |
return (EINVAL); |
286 |
if (val > cnt.v_page_count) |
287 |
return (EINVAL); |
288 |
|
289 |
zfs_arc_free_target = val; |
290 |
|
291 |
return (0); |
292 |
} |
293 |
|
294 |
static int |
295 |
sysctl_vfs_zfs_arc_cache_target(SYSCTL_HANDLER_ARGS) |
296 |
{ |
297 |
u_int val; |
298 |
int err; |
299 |
|
300 |
val = zfs_arc_cache_target; |
301 |
err = sysctl_handle_int(oidp, &val, 0, req); |
302 |
if (err != 0 || req->newptr == NULL) |
303 |
return (err); |
304 |
|
305 |
if (val < minfree) |
306 |
return (EINVAL); |
307 |
if (val > cnt.v_page_count) |
308 |
return (EINVAL); |
309 |
|
310 |
zfs_arc_cache_target = val; |
311 |
|
312 |
return (0); |
313 |
} |
314 |
#endif |
315 |
|
221 |
/* |
316 |
/* |
222 |
* Note that buffers can be in one of 6 states: |
317 |
* Note that buffers can be in one of 6 states: |
223 |
* ARC_anon - anonymous (discussed below) |
318 |
* ARC_anon - anonymous (discussed below) |
Lines 592-597
static void arc_evict_ghost(arc_state_t *state, ui
Link Here
|
592 |
static void arc_buf_watch(arc_buf_t *buf); |
687 |
static void arc_buf_watch(arc_buf_t *buf); |
593 |
#endif /* illumos */ |
688 |
#endif /* illumos */ |
594 |
|
689 |
|
|
|
690 |
static uint64_t arc_cache_free(void); |
691 |
static boolean_t arc_cache_reclaim_needed(uint64_t size); |
692 |
static boolean_t arc_cache_reclaim(uint64_t size, |
693 |
arc_cache_reclaim_strategy_t strat); |
694 |
static boolean_t arc_cache_reclaim_strat(kmem_cache_t *cache, uint64_t size, |
695 |
arc_cache_reclaim_strategy_t strat); |
696 |
|
595 |
static boolean_t l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab); |
697 |
static boolean_t l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab); |
596 |
|
698 |
|
597 |
#define GHOST_STATE(state) \ |
699 |
#define GHOST_STATE(state) \ |
Lines 2421-2426
arc_flush(spa_t *spa)
Link Here
|
2421 |
void |
2523 |
void |
2422 |
arc_shrink(void) |
2524 |
arc_shrink(void) |
2423 |
{ |
2525 |
{ |
|
|
2526 |
|
2424 |
if (arc_c > arc_c_min) { |
2527 |
if (arc_c > arc_c_min) { |
2425 |
uint64_t to_free; |
2528 |
uint64_t to_free; |
2426 |
|
2529 |
|
Lines 2429-2434
arc_shrink(void)
Link Here
|
2429 |
#else |
2532 |
#else |
2430 |
to_free = arc_c >> arc_shrink_shift; |
2533 |
to_free = arc_c >> arc_shrink_shift; |
2431 |
#endif |
2534 |
#endif |
|
|
2535 |
DTRACE_PROBE4(arc__shrink, uint64_t, arc_c, uint64_t, |
2536 |
arc_c_min, uint64_t, arc_p, uint64_t, to_free); |
2537 |
|
2432 |
if (arc_c > arc_c_min + to_free) |
2538 |
if (arc_c > arc_c_min + to_free) |
2433 |
atomic_add_64(&arc_c, -to_free); |
2539 |
atomic_add_64(&arc_c, -to_free); |
2434 |
else |
2540 |
else |
Lines 2439-2450
arc_shrink(void)
Link Here
|
2439 |
arc_c = MAX(arc_size, arc_c_min); |
2545 |
arc_c = MAX(arc_size, arc_c_min); |
2440 |
if (arc_p > arc_c) |
2546 |
if (arc_p > arc_c) |
2441 |
arc_p = (arc_c >> 1); |
2547 |
arc_p = (arc_c >> 1); |
|
|
2548 |
|
2549 |
DTRACE_PROBE2(arc__shrunk, uint64_t, arc_c, uint64_t, |
2550 |
arc_p); |
2551 |
|
2442 |
ASSERT(arc_c >= arc_c_min); |
2552 |
ASSERT(arc_c >= arc_c_min); |
2443 |
ASSERT((int64_t)arc_p >= 0); |
2553 |
ASSERT((int64_t)arc_p >= 0); |
2444 |
} |
2554 |
} |
2445 |
|
2555 |
|
2446 |
if (arc_size > arc_c) |
2556 |
if (arc_size > arc_c) { |
|
|
2557 |
DTRACE_PROBE2(arc__shrink_adjust, uint64_t, arc_size, |
2558 |
uint64_t, arc_c); |
2447 |
arc_adjust(); |
2559 |
arc_adjust(); |
|
|
2560 |
} |
2448 |
} |
2561 |
} |
2449 |
|
2562 |
|
2450 |
static int needfree = 0; |
2563 |
static int needfree = 0; |
Lines 2454-2469
arc_reclaim_needed(void)
Link Here
|
2454 |
{ |
2567 |
{ |
2455 |
|
2568 |
|
2456 |
#ifdef _KERNEL |
2569 |
#ifdef _KERNEL |
|
|
2570 |
if (arc_size <= arc_c_min) { |
2571 |
DTRACE_PROBE2(arc__reclaim_min, uint64_t, arc_size, |
2572 |
uint64_t, arc_c_min); |
2573 |
return (0); |
2574 |
} |
2457 |
|
2575 |
|
2458 |
if (needfree) |
2576 |
if (needfree) { |
|
|
2577 |
DTRACE_PROBE(arc__reclaim_needfree); |
2459 |
return (1); |
2578 |
return (1); |
|
|
2579 |
} |
2460 |
|
2580 |
|
2461 |
/* |
2581 |
/* |
2462 |
* Cooperate with pagedaemon when it's time for it to scan |
2582 |
* Cooperate with pagedaemon when it's time for it to scan |
2463 |
* and reclaim some pages. |
2583 |
* and reclaim some pages. |
2464 |
*/ |
2584 |
*/ |
2465 |
if (vm_paging_needed()) |
2585 |
if (freemem < zfs_arc_free_target) { |
|
|
2586 |
DTRACE_PROBE2(arc__reclaim_freemem, uint64_t, |
2587 |
freemem, uint64_t, zfs_arc_free_target); |
2466 |
return (1); |
2588 |
return (1); |
|
|
2589 |
} |
2467 |
|
2590 |
|
2468 |
#ifdef sun |
2591 |
#ifdef sun |
2469 |
/* |
2592 |
/* |
Lines 2491-2498
arc_reclaim_needed(void)
Link Here
|
2491 |
if (availrmem < swapfs_minfree + swapfs_reserve + extra) |
2614 |
if (availrmem < swapfs_minfree + swapfs_reserve + extra) |
2492 |
return (1); |
2615 |
return (1); |
2493 |
|
2616 |
|
2494 |
#if defined(__i386) |
|
|
2495 |
/* |
2617 |
/* |
|
|
2618 |
* Check that we have enough availrmem that memory locking (e.g., via |
2619 |
* mlock(3C) or memcntl(2)) can still succeed. (pages_pp_maximum |
2620 |
* stores the number of pages that cannot be locked; when availrmem |
2621 |
* drops below pages_pp_maximum, page locking mechanisms such as |
2622 |
* page_pp_lock() will fail.) |
2623 |
*/ |
2624 |
if (availrmem <= pages_pp_maximum) |
2625 |
return (1); |
2626 |
|
2627 |
#endif /* sun */ |
2628 |
#if defined(__i386) || !defined(UMA_MD_SMALL_ALLOC) |
2629 |
/* |
2496 |
* If we're on an i386 platform, it's possible that we'll exhaust the |
2630 |
* If we're on an i386 platform, it's possible that we'll exhaust the |
2497 |
* kernel heap space before we ever run out of available physical |
2631 |
* kernel heap space before we ever run out of available physical |
2498 |
* memory. Most checks of the size of the heap_area compare against |
2632 |
* memory. Most checks of the size of the heap_area compare against |
Lines 2503-2528
arc_reclaim_needed(void)
Link Here
|
2503 |
* heap is allocated. (Or, in the calculation, if less than 1/4th is |
2637 |
* heap is allocated. (Or, in the calculation, if less than 1/4th is |
2504 |
* free) |
2638 |
* free) |
2505 |
*/ |
2639 |
*/ |
2506 |
if (btop(vmem_size(heap_arena, VMEM_FREE)) < |
2640 |
if (vmem_size(heap_arena, VMEM_FREE) < |
2507 |
(btop(vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC)) >> 2)) |
2641 |
(vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2)) { |
|
|
2642 |
DTRACE_PROBE2(arc__reclaim_used, uint64_t, |
2643 |
vmem_size(heap_arena, VMEM_FREE), uint64_t, |
2644 |
(vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC)) >> 2); |
2508 |
return (1); |
2645 |
return (1); |
|
|
2646 |
} |
2509 |
#endif |
2647 |
#endif |
2510 |
#else /* !sun */ |
2648 |
#ifdef sun |
2511 |
if (kmem_used() > (kmem_size() * 3) / 4) |
2649 |
/* |
|
|
2650 |
* If zio data pages are being allocated out of a separate heap segment, |
2651 |
* then enforce that the size of available vmem for this arena remains |
2652 |
* above about 1/16th free. |
2653 |
* |
2654 |
* Note: The 1/16th arena free requirement was put in place |
2655 |
* to aggressively evict memory from the arc in order to avoid |
2656 |
* memory fragmentation issues. |
2657 |
*/ |
2658 |
if (zio_arena != NULL && |
2659 |
vmem_size(zio_arena, VMEM_FREE) < |
2660 |
(vmem_size(zio_arena, VMEM_ALLOC) >> 4)) |
2512 |
return (1); |
2661 |
return (1); |
2513 |
#endif /* sun */ |
2662 |
#endif /* sun */ |
2514 |
|
2663 |
#else /* _KERNEL */ |
2515 |
#else |
|
|
2516 |
if (spa_get_random(100) == 0) |
2664 |
if (spa_get_random(100) == 0) |
2517 |
return (1); |
2665 |
return (1); |
2518 |
#endif |
2666 |
#endif /* _KERNEL */ |
|
|
2667 |
DTRACE_PROBE(arc__reclaim_no); |
2668 |
|
2519 |
return (0); |
2669 |
return (0); |
2520 |
} |
2670 |
} |
2521 |
|
2671 |
|
2522 |
extern kmem_cache_t *zio_buf_cache[]; |
2672 |
extern kmem_cache_t *zio_buf_cache[]; |
2523 |
extern kmem_cache_t *zio_data_buf_cache[]; |
2673 |
extern kmem_cache_t *zio_data_buf_cache[]; |
|
|
2674 |
extern kmem_cache_t *range_seg_cache; |
2524 |
|
2675 |
|
2525 |
static void |
2676 |
static void __noinline |
2526 |
arc_kmem_reap_now(arc_reclaim_strategy_t strat) |
2677 |
arc_kmem_reap_now(arc_reclaim_strategy_t strat) |
2527 |
{ |
2678 |
{ |
2528 |
size_t i; |
2679 |
size_t i; |
Lines 2529-2534
arc_kmem_reap_now(arc_reclaim_strategy_t strat)
Link Here
|
2529 |
kmem_cache_t *prev_cache = NULL; |
2680 |
kmem_cache_t *prev_cache = NULL; |
2530 |
kmem_cache_t *prev_data_cache = NULL; |
2681 |
kmem_cache_t *prev_data_cache = NULL; |
2531 |
|
2682 |
|
|
|
2683 |
DTRACE_PROBE(arc__kmem_reap_start); |
2532 |
#ifdef _KERNEL |
2684 |
#ifdef _KERNEL |
2533 |
if (arc_meta_used >= arc_meta_limit) { |
2685 |
if (arc_meta_used >= arc_meta_limit) { |
2534 |
/* |
2686 |
/* |
Lines 2537-2543
arc_kmem_reap_now(arc_reclaim_strategy_t strat)
Link Here
|
2537 |
*/ |
2689 |
*/ |
2538 |
dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent); |
2690 |
dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent); |
2539 |
} |
2691 |
} |
2540 |
#if defined(__i386) |
2692 |
#if defined(__i386) || !defined(UMA_MD_SMALL_ALLOC) |
2541 |
/* |
2693 |
/* |
2542 |
* Reclaim unused memory from all kmem caches. |
2694 |
* Reclaim unused memory from all kmem caches. |
2543 |
*/ |
2695 |
*/ |
Lines 2552-2571
arc_kmem_reap_now(arc_reclaim_strategy_t strat)
Link Here
|
2552 |
if (strat == ARC_RECLAIM_AGGR) |
2704 |
if (strat == ARC_RECLAIM_AGGR) |
2553 |
arc_shrink(); |
2705 |
arc_shrink(); |
2554 |
|
2706 |
|
|
|
2707 |
(void) arc_cache_reclaim(0, ARC_CACHE_RECLAIM_FORCE); |
2708 |
|
2709 |
#ifdef sun |
2710 |
/* |
2711 |
* Ask the vmem areana to reclaim unused memory from its |
2712 |
* quantum caches. |
2713 |
*/ |
2714 |
if (zio_arena != NULL && strat == ARC_RECLAIM_AGGR) |
2715 |
vmem_qcache_reap(zio_arena); |
2716 |
#endif |
2717 |
out: |
2718 |
DTRACE_PROBE(arc__kmem_reap_end); |
2719 |
} |
2720 |
|
2721 |
|
2722 |
static boolean_t |
2723 |
arc_cache_reclaim_needed(uint64_t size) |
2724 |
{ |
2725 |
|
2726 |
if (zfs_arc_cache_target && freemem < zfs_arc_cache_target + btop(size)) { |
2727 |
DTRACE_PROBE1(arc_cache_reclaim_needed, int, B_TRUE); |
2728 |
return (B_TRUE); |
2729 |
} |
2730 |
|
2731 |
DTRACE_PROBE1(arc_cache_reclaim_needed, int, B_FALSE); |
2732 |
return (B_FALSE); |
2733 |
} |
2734 |
|
2735 |
static boolean_t |
2736 |
arc_cache_reclaim_strat(kmem_cache_t *cache, uint64_t size, |
2737 |
arc_cache_reclaim_strategy_t strat) |
2738 |
{ |
2739 |
|
2740 |
switch(strat) { |
2741 |
case ARC_CACHE_RECLAIM_NOW: |
2742 |
case ARC_CACHE_RECLAIM_FORCE: |
2743 |
kmem_cache_reap_now(cache); |
2744 |
if (zfs_arc_cache_partial && strat != ARC_CACHE_RECLAIM_FORCE && |
2745 |
!arc_cache_reclaim_needed(size)) { |
2746 |
return (B_TRUE); |
2747 |
} |
2748 |
break; |
2749 |
default: |
2750 |
kmem_cache_reap(cache, zfs_arc_cache_free_max); |
2751 |
} |
2752 |
|
2753 |
return (B_FALSE); |
2754 |
} |
2755 |
|
2756 |
static boolean_t |
2757 |
arc_cache_reclaim(uint64_t size, arc_cache_reclaim_strategy_t strat) |
2758 |
{ |
2759 |
int i; |
2760 |
clock_t now; |
2761 |
kmem_cache_t *prev_cache, *prev_data_cache; |
2762 |
|
2763 |
if (strat != ARC_CACHE_RECLAIM_FORCE) { |
2764 |
now = ddi_get_lbolt(); |
2765 |
DTRACE_PROBE3(arc_cache_reclaim_test, int, strat, int64_t, now, |
2766 |
int64_t, arc_cache_reclaim_last); |
2767 |
if (now - arc_cache_reclaim_last < (zfs_arc_cache_period * hz)) |
2768 |
return (B_FALSE); |
2769 |
} |
2770 |
|
2771 |
DTRACE_PROBE1(arc_cache_reclaim, int, strat); |
2772 |
|
2555 |
for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { |
2773 |
for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { |
2556 |
if (zio_buf_cache[i] != prev_cache) { |
2774 |
if (zio_buf_cache[i] != prev_cache) { |
2557 |
prev_cache = zio_buf_cache[i]; |
2775 |
prev_cache = zio_buf_cache[i]; |
2558 |
kmem_cache_reap_now(zio_buf_cache[i]); |
2776 |
if (arc_cache_reclaim_strat(zio_buf_cache[i], size, |
|
|
2777 |
strat)) { |
2778 |
return (B_TRUE); |
2779 |
} |
2780 |
|
2559 |
} |
2781 |
} |
2560 |
if (zio_data_buf_cache[i] != prev_data_cache) { |
2782 |
if (zio_data_buf_cache[i] != prev_data_cache) { |
2561 |
prev_data_cache = zio_data_buf_cache[i]; |
2783 |
prev_data_cache = zio_data_buf_cache[i]; |
2562 |
kmem_cache_reap_now(zio_data_buf_cache[i]); |
2784 |
if (arc_cache_reclaim_strat(zio_data_buf_cache[i], |
|
|
2785 |
size, strat)) |
2786 |
return (B_TRUE); |
2563 |
} |
2787 |
} |
2564 |
} |
2788 |
} |
2565 |
kmem_cache_reap_now(buf_cache); |
2789 |
if (arc_cache_reclaim_strat(range_seg_cache, size, strat)) |
2566 |
kmem_cache_reap_now(hdr_cache); |
2790 |
return (B_TRUE); |
|
|
2791 |
|
2792 |
if (arc_cache_reclaim_strat(buf_cache, size, strat)) |
2793 |
return (B_TRUE); |
2794 |
|
2795 |
arc_cache_reclaim_strat(hdr_cache, size, strat); |
2796 |
|
2797 |
arc_cache_reclaim_last = ddi_get_lbolt(); |
2798 |
|
2799 |
if (arc_cache_reclaim_needed(size)) |
2800 |
return (B_FALSE); |
2801 |
|
2802 |
return (B_TRUE); |
2567 |
} |
2803 |
} |
2568 |
|
2804 |
|
|
|
2805 |
static uint64_t |
2806 |
arc_cache_free(void) |
2807 |
{ |
2808 |
int i; |
2809 |
uint64_t cachefree; |
2810 |
kmem_cache_t *prev_cache, *prev_data_cache; |
2811 |
|
2812 |
cachefree = kmem_cache_free_size(buf_cache) + |
2813 |
kmem_cache_free_size(hdr_cache); |
2814 |
|
2815 |
prev_cache = NULL; |
2816 |
prev_data_cache = NULL; |
2817 |
|
2818 |
for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { |
2819 |
if (zio_buf_cache[i] != prev_cache) { |
2820 |
prev_cache = zio_buf_cache[i]; |
2821 |
cachefree += kmem_cache_free_size(zio_buf_cache[i]); |
2822 |
} |
2823 |
if (zio_data_buf_cache[i] != prev_data_cache) { |
2824 |
prev_data_cache = zio_data_buf_cache[i]; |
2825 |
cachefree += kmem_cache_free_size(zio_data_buf_cache[i]); |
2826 |
} |
2827 |
} |
2828 |
|
2829 |
return (cachefree); |
2830 |
} |
2831 |
|
2569 |
static void |
2832 |
static void |
2570 |
arc_reclaim_thread(void *dummy __unused) |
2833 |
arc_reclaim_thread(void *dummy __unused) |
2571 |
{ |
2834 |
{ |
Lines 2577-2586
arc_reclaim_thread(void *dummy __unused)
Link Here
|
2577 |
|
2840 |
|
2578 |
mutex_enter(&arc_reclaim_thr_lock); |
2841 |
mutex_enter(&arc_reclaim_thr_lock); |
2579 |
while (arc_thread_exit == 0) { |
2842 |
while (arc_thread_exit == 0) { |
|
|
2843 |
DTRACE_PROBE(arc__reclaim_thread); |
2844 |
/* |
2845 |
* As free items in our caches can take up significant RAM |
2846 |
* possibly in infrequently used zones we trigger a cache |
2847 |
* reclaim if: |
2848 |
* 1. We're below zfs_arc_cache_target threshold |
2849 |
* 2. Its been zfs_arc_cache_free_period seconds since the |
2850 |
* last cache reclaim. |
2851 |
*/ |
2852 |
if (arc_cache_reclaim_needed(0)) { |
2853 |
(void) arc_cache_reclaim(0, ARC_CACHE_RECLAIM_NOW); |
2854 |
} else { |
2855 |
clock_t now; |
2856 |
|
2857 |
now = ddi_get_lbolt(); |
2858 |
DTRACE_PROBE2(arc__reclaim_check, int64_t, now, |
2859 |
int64_t, arc_cache_reclaim_last); |
2860 |
if (now - arc_cache_reclaim_last > |
2861 |
(zfs_arc_cache_free_period * hz)) { |
2862 |
(void) arc_cache_reclaim(0, |
2863 |
ARC_CACHE_RECLAIM_SIZE); |
2864 |
} |
2865 |
} |
2866 |
|
2580 |
if (arc_reclaim_needed()) { |
2867 |
if (arc_reclaim_needed()) { |
2581 |
|
2868 |
DTRACE_PROBE1(arc__caches_free, uint64_t, |
|
|
2869 |
arc_cache_free()); |
2582 |
if (arc_no_grow) { |
2870 |
if (arc_no_grow) { |
2583 |
if (last_reclaim == ARC_RECLAIM_CONS) { |
2871 |
if (last_reclaim == ARC_RECLAIM_CONS) { |
|
|
2872 |
DTRACE_PROBE(arc__reclaim_aggr_no_grow); |
2584 |
last_reclaim = ARC_RECLAIM_AGGR; |
2873 |
last_reclaim = ARC_RECLAIM_AGGR; |
2585 |
} else { |
2874 |
} else { |
2586 |
last_reclaim = ARC_RECLAIM_CONS; |
2875 |
last_reclaim = ARC_RECLAIM_CONS; |
Lines 2588-2593
arc_reclaim_thread(void *dummy __unused)
Link Here
|
2588 |
} else { |
2877 |
} else { |
2589 |
arc_no_grow = TRUE; |
2878 |
arc_no_grow = TRUE; |
2590 |
last_reclaim = ARC_RECLAIM_AGGR; |
2879 |
last_reclaim = ARC_RECLAIM_AGGR; |
|
|
2880 |
DTRACE_PROBE(arc__reclaim_aggr); |
2591 |
membar_producer(); |
2881 |
membar_producer(); |
2592 |
} |
2882 |
} |
2593 |
|
2883 |
|
Lines 2602-2607
arc_reclaim_thread(void *dummy __unused)
Link Here
|
2602 |
*/ |
2892 |
*/ |
2603 |
arc_no_grow = TRUE; |
2893 |
arc_no_grow = TRUE; |
2604 |
last_reclaim = ARC_RECLAIM_AGGR; |
2894 |
last_reclaim = ARC_RECLAIM_AGGR; |
|
|
2895 |
DTRACE_PROBE(arc__reclaim_aggr_needfree); |
2605 |
} |
2896 |
} |
2606 |
arc_kmem_reap_now(last_reclaim); |
2897 |
arc_kmem_reap_now(last_reclaim); |
2607 |
arc_warm = B_TRUE; |
2898 |
arc_warm = B_TRUE; |
Lines 2618-2623
arc_reclaim_thread(void *dummy __unused)
Link Here
|
2618 |
#ifdef _KERNEL |
2909 |
#ifdef _KERNEL |
2619 |
if (needfree) { |
2910 |
if (needfree) { |
2620 |
needfree = 0; |
2911 |
needfree = 0; |
|
|
2912 |
DTRACE_PROBE(arc__clear_needfree); |
2621 |
wakeup(&needfree); |
2913 |
wakeup(&needfree); |
2622 |
} |
2914 |
} |
2623 |
#endif |
2915 |
#endif |
Lines 2692-2697
arc_adapt(int bytes, arc_state_t *state)
Link Here
|
2692 |
* cache size, increment the target cache size |
2984 |
* cache size, increment the target cache size |
2693 |
*/ |
2985 |
*/ |
2694 |
if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) { |
2986 |
if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) { |
|
|
2987 |
DTRACE_PROBE1(arc__inc_adapt, int, bytes); |
2695 |
atomic_add_64(&arc_c, (int64_t)bytes); |
2988 |
atomic_add_64(&arc_c, (int64_t)bytes); |
2696 |
if (arc_c > arc_c_max) |
2989 |
if (arc_c > arc_c_max) |
2697 |
arc_c = arc_c_max; |
2990 |
arc_c = arc_c_max; |
Lines 2713-2732
arc_evict_needed(arc_buf_contents_t type)
Link Here
|
2713 |
if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit) |
3006 |
if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit) |
2714 |
return (1); |
3007 |
return (1); |
2715 |
|
3008 |
|
2716 |
#ifdef sun |
|
|
2717 |
#ifdef _KERNEL |
2718 |
/* |
2719 |
* If zio data pages are being allocated out of a separate heap segment, |
2720 |
* then enforce that the size of available vmem for this area remains |
2721 |
* above about 1/32nd free. |
2722 |
*/ |
2723 |
if (type == ARC_BUFC_DATA && zio_arena != NULL && |
2724 |
vmem_size(zio_arena, VMEM_FREE) < |
2725 |
(vmem_size(zio_arena, VMEM_ALLOC) >> 5)) |
2726 |
return (1); |
2727 |
#endif |
2728 |
#endif /* sun */ |
2729 |
|
2730 |
if (arc_reclaim_needed()) |
3009 |
if (arc_reclaim_needed()) |
2731 |
return (1); |
3010 |
return (1); |
2732 |
|
3011 |
|
Lines 2763-2768
arc_get_data_buf(arc_buf_t *buf)
Link Here
|
2763 |
uint64_t size = buf->b_hdr->b_size; |
3042 |
uint64_t size = buf->b_hdr->b_size; |
2764 |
arc_buf_contents_t type = buf->b_hdr->b_type; |
3043 |
arc_buf_contents_t type = buf->b_hdr->b_type; |
2765 |
|
3044 |
|
|
|
3045 |
if (arc_cache_reclaim_needed(size)) |
3046 |
(void) arc_cache_reclaim(size, ARC_CACHE_RECLAIM_NOW); |
3047 |
|
2766 |
arc_adapt(size, state); |
3048 |
arc_adapt(size, state); |
2767 |
|
3049 |
|
2768 |
/* |
3050 |
/* |
Lines 3885-3904
static int
Link Here
|
3885 |
arc_memory_throttle(uint64_t reserve, uint64_t txg) |
4167 |
arc_memory_throttle(uint64_t reserve, uint64_t txg) |
3886 |
{ |
4168 |
{ |
3887 |
#ifdef _KERNEL |
4169 |
#ifdef _KERNEL |
3888 |
uint64_t available_memory = |
4170 |
uint64_t available_memory = ptob(freemem); |
3889 |
ptoa((uintmax_t)cnt.v_free_count + cnt.v_cache_count); |
|
|
3890 |
static uint64_t page_load = 0; |
4171 |
static uint64_t page_load = 0; |
3891 |
static uint64_t last_txg = 0; |
4172 |
static uint64_t last_txg = 0; |
3892 |
|
4173 |
|
3893 |
#ifdef sun |
4174 |
#if defined(__i386) || !defined(UMA_MD_SMALL_ALLOC) |
3894 |
#if defined(__i386) |
|
|
3895 |
available_memory = |
4175 |
available_memory = |
3896 |
MIN(available_memory, vmem_size(heap_arena, VMEM_FREE)); |
4176 |
MIN(available_memory, ptob(vmem_size(heap_arena, VMEM_FREE))); |
3897 |
#endif |
4177 |
#endif |
3898 |
#endif /* sun */ |
|
|
3899 |
|
4178 |
|
3900 |
if (cnt.v_free_count + cnt.v_cache_count > |
4179 |
if (freemem > (uint64_t)physmem * arc_lotsfree_percent / 100) |
3901 |
(uint64_t)physmem * arc_lotsfree_percent / 100) |
|
|
3902 |
return (0); |
4180 |
return (0); |
3903 |
|
4181 |
|
3904 |
if (txg > last_txg) { |
4182 |
if (txg > last_txg) { |
Lines 3911-3917
arc_memory_throttle(uint64_t reserve, uint64_t txg
Link Here
|
3911 |
* continue to let page writes occur as quickly as possible. |
4189 |
* continue to let page writes occur as quickly as possible. |
3912 |
*/ |
4190 |
*/ |
3913 |
if (curproc == pageproc) { |
4191 |
if (curproc == pageproc) { |
3914 |
if (page_load > available_memory / 4) |
4192 |
if (page_load > MAX(ptob(minfree), available_memory) / 4) |
3915 |
return (SET_ERROR(ERESTART)); |
4193 |
return (SET_ERROR(ERESTART)); |
3916 |
/* Note: reserve is inflated, so we deflate */ |
4194 |
/* Note: reserve is inflated, so we deflate */ |
3917 |
page_load += reserve / 8; |
4195 |
page_load += reserve / 8; |
Lines 3939-3946
arc_tempreserve_space(uint64_t reserve, uint64_t t
Link Here
|
3939 |
int error; |
4217 |
int error; |
3940 |
uint64_t anon_size; |
4218 |
uint64_t anon_size; |
3941 |
|
4219 |
|
3942 |
if (reserve > arc_c/4 && !arc_no_grow) |
4220 |
if (reserve > arc_c/4 && !arc_no_grow) { |
3943 |
arc_c = MIN(arc_c_max, reserve * 4); |
4221 |
arc_c = MIN(arc_c_max, reserve * 4); |
|
|
4222 |
DTRACE_PROBE1(arc__set_reserve, uint64_t, arc_c); |
4223 |
} |
3944 |
if (reserve > arc_c) |
4224 |
if (reserve > arc_c) |
3945 |
return (SET_ERROR(ENOMEM)); |
4225 |
return (SET_ERROR(ENOMEM)); |
3946 |
|
4226 |
|
Lines 3994-3999
arc_lowmem(void *arg __unused, int howto __unused)
Link Here
|
3994 |
mutex_enter(&arc_lowmem_lock); |
4274 |
mutex_enter(&arc_lowmem_lock); |
3995 |
mutex_enter(&arc_reclaim_thr_lock); |
4275 |
mutex_enter(&arc_reclaim_thr_lock); |
3996 |
needfree = 1; |
4276 |
needfree = 1; |
|
|
4277 |
DTRACE_PROBE(arc__needfree); |
3997 |
cv_signal(&arc_reclaim_thr_cv); |
4278 |
cv_signal(&arc_reclaim_thr_cv); |
3998 |
|
4279 |
|
3999 |
/* |
4280 |
/* |