Attachment #146852 for bug #187594

View | Details | Raw Unified | Return to bug 187594 | Differences between

and this patch




}
SYSINIT(kmem_size_init, SI_SUB_KMEM, SI_ORDER_ANY, kmem_size_init, NULL);

/*
 * The return values from kmem_free_* are only valid once the pagedaemon
 * has been initialised, before then they return 0.
 * 
 * To ensure the returns are valid the caller can use a SYSINIT with
 * subsystem set to SI_SUB_KTHREAD_PAGE and an order of at least
 * SI_ORDER_SECOND.
 */
u_int
kmem_free_target(void)
{

	return (vm_cnt.v_free_target);
}

u_int
kmem_free_min(void)
{

	return (vm_cnt.v_free_min);
}

u_int
kmem_free_count(void)
{

	return (vm_cnt.v_free_count + vm_cnt.v_cache_count);
}

u_int
kmem_page_count(void)
{

	return (vm_cnt.v_page_count);
}

uint64_t
kmem_size(void)
{

	return (kmem_size_val);
}

uint64_t
kmem_used(void)
{

	return (vmem_size(kmem_arena, VMEM_ALLOC));
}

static int
kmem_std_constructor(void *mem, int size __unused, void *private, int flags)
{




void *zfs_kmem_alloc(size_t size, int kmflags);
void zfs_kmem_free(void *buf, size_t size);
uint64_t kmem_size(void);
uint64_t kmem_used(void);
u_int kmem_page_count(void);

/*
 * The return values from kmem_free_* are only valid once the pagedaemon
 * has been initialised, before then they return 0.
 */
u_int kmem_free_count(void);
u_int kmem_free_target(void);
u_int kmem_free_min(void);

kmem_cache_t *kmem_cache_create(char *name, size_t bufsize, size_t align,
    int (*constructor)(void *, void *, int), void (*destructor)(void *, void *),
    void (*reclaim)(void *) __unused, void *private, vmem_t *vmp, int cflags);

int kmem_debugging(void);
void *calloc(size_t n, size_t s);

#define	freemem				(vm_cnt.v_free_count + vm_cnt.v_cache_count)
#define	minfree				vm_cnt.v_free_min
#define	heap_arena			kmem_arena
#define	kmem_alloc(size, kmflags)	zfs_kmem_alloc((size), (kmflags))
#define	kmem_zalloc(size, kmflags)	zfs_kmem_alloc((size), (kmflags) | M_ZERO)
#define	kmem_free(buf, size)		zfs_kmem_free((buf), (size))




int zfs_arc_p_min_shift = 0;
int zfs_disable_dup_eviction = 0;
uint64_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */
u_int zfs_arc_free_target = (1 << 16); /* default before pagedaemon init only */

static int sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS);


arc_free_target_init(void *unused __unused)
{

	zfs_arc_free_target = vm_pageout_wakeup_thresh;
}
SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY,
    arc_free_target_init, NULL);
#endif

TUNABLE_QUAD("vfs.zfs.arc_meta_limit", &zfs_arc_meta_limit);
SYSCTL_DECL(_vfs_zfs);

	if (err != 0 || req->newptr == NULL)
		return (err);

	if (val < minfree)
		return (EINVAL);
	if (val > vm_cnt.v_page_count)
		return (EINVAL);

	zfs_arc_free_target = val;


	return (0);
}
#endif

/*
 * Note that buffers can be in one of 6 states:

		return (1);
	}

	if (kmem_free_count() < zfs_arc_free_target) {
		DTRACE_PROBE2(arc__reclaim_freetarget, uint64_t,
		    kmem_free_count(), uint64_t, zfs_arc_free_target);
		return (1);
	}

	/*
	 * Cooperate with pagedaemon when it's time for it to scan
	 * and reclaim some pages.
	 */
	if (freemem < zfs_arc_free_target) {
		DTRACE_PROBE2(arc__reclaim_freemem, uint64_t,
		    freemem, uint64_t, zfs_arc_free_target);
		return (1);
	}


	if (availrmem < swapfs_minfree + swapfs_reserve + extra)
		return (1);

#if defined(__i386)
	/*
	 * Check that we have enough availrmem that memory locking (e.g., via
	 * mlock(3C) or memcntl(2)) can still succeed.  (pages_pp_maximum
	 * stores the number of pages that cannot be locked; when availrmem
	 * drops below pages_pp_maximum, page locking mechanisms such as
	 * page_pp_lock() will fail.)
	 */
	if (availrmem <= pages_pp_maximum)
		return (1);

#endif	/* sun */
#if defined(__i386) || !defined(UMA_MD_SMALL_ALLOC)
	/*
	 * If we're on an i386 platform, it's possible that we'll exhaust the
	 * kernel heap space before we ever run out of available physical
	 * memory.  Most checks of the size of the heap_area compare against

	 * heap is allocated.  (Or, in the calculation, if less than 1/4th is
	 * free)
	 */
	if (vmem_size(heap_arena, VMEM_FREE) <
	    (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2)) {
		return (1);
#endif
#else	/* sun */
#ifdef __i386__
	/* i386 has KVA limits that the raw page counts above don't consider */
	if (kmem_used() > (kmem_size() * 3) / 4) {
		DTRACE_PROBE2(arc__reclaim_used, uint64_t,
		    vmem_size(heap_arena, VMEM_FREE), uint64_t,
		    (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC)) >> 2);
		return (1);
	}
#endif
#ifdef sun
	/*
	 * If zio data pages are being allocated out of a separate heap segment,
	 * then enforce that the size of available vmem for this arena remains
	 * above about 1/16th free.
	 *
	 * Note: The 1/16th arena free requirement was put in place
	 * to aggressively evict memory from the arc in order to avoid
	 * memory fragmentation issues.
	 */
	if (zio_arena != NULL &&
	    vmem_size(zio_arena, VMEM_FREE) <
	    (vmem_size(zio_arena, VMEM_ALLOC) >> 4))
		return (1);
#endif	/* sun */
#else	/* _KERNEL */
#else
	if (spa_get_random(100) == 0)
		return (1);
#endif	/* _KERNEL */
	DTRACE_PROBE(arc__reclaim_no);

	return (0);

	}
	kmem_cache_reap_now(buf_cache);
	kmem_cache_reap_now(hdr_cache);

#ifdef sun
	/*
	 * Ask the vmem areana to reclaim unused memory from its
	 * quantum caches.
	 */
	if (zio_arena != NULL && strat == ARC_RECLAIM_AGGR)
		vmem_qcache_reap(zio_arena);
#endif
}

static void

	if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit)
		return (1);

#ifdef sun
#ifdef _KERNEL
	/*
	 * If zio data pages are being allocated out of a separate heap segment,
	 * then enforce that the size of available vmem for this area remains
	 * above about 1/32nd free.
	 */
	if (type == ARC_BUFC_DATA && zio_arena != NULL &&
	    vmem_size(zio_arena, VMEM_FREE) <
	    (vmem_size(zio_arena, VMEM_ALLOC) >> 5))
		return (1);
#endif
#endif	/* sun */

	if (arc_reclaim_needed())
		return (1);


arc_memory_throttle(uint64_t reserve, uint64_t txg)
{
#ifdef _KERNEL
	uint64_t available_memory = ptob(freemem);
	    ptoa((uintmax_t)vm_cnt.v_free_count + vm_cnt.v_cache_count);
	static uint64_t page_load = 0;
	static uint64_t last_txg = 0;

#if defined(__i386) || !defined(UMA_MD_SMALL_ALLOC)
#if defined(__i386)
	available_memory =
	    MIN(available_memory, ptob(vmem_size(heap_arena, VMEM_FREE)));
#endif
#endif	/* sun */

	if (freemem > (uint64_t)physmem * arc_lotsfree_percent / 100)
	    (uint64_t)physmem * arc_lotsfree_percent / 100)
		return (0);

	if (txg > last_txg) {

	 * continue to let page writes occur as quickly as possible.
	 */
	if (curproc == pageproc) {
		if (page_load > MAX(ptob(minfree), available_memory) / 4)
			return (SET_ERROR(ERESTART));
		/* Note: reserve is inflated, so we deflate */
		page_load += reserve / 8;

Return to bug 187594

Lines 126-167 kmem_size_init(void *unused __unused) Link Here

(-)sys/cddl/compat/opensolaris/kern/opensolaris_kmem.c (-43 lines)
126	}	126	}
127	SYSINIT(kmem_size_init, SI_SUB_KMEM, SI_ORDER_ANY, kmem_size_init, NULL);	127	SYSINIT(kmem_size_init, SI_SUB_KMEM, SI_ORDER_ANY, kmem_size_init, NULL);
128		128
129	/*
130	* The return values from kmem_free_* are only valid once the pagedaemon
131	* has been initialised, before then they return 0.
132	*
133	* To ensure the returns are valid the caller can use a SYSINIT with
134	* subsystem set to SI_SUB_KTHREAD_PAGE and an order of at least
135	* SI_ORDER_SECOND.
136	*/
137	u_int
138	kmem_free_target(void)
139	{
140
141	return (vm_cnt.v_free_target);
142	}
143
144	u_int
145	kmem_free_min(void)
146	{
147
148	return (vm_cnt.v_free_min);
149	}
150
151	u_int
152	kmem_free_count(void)
153	{
154
155	return (vm_cnt.v_free_count + vm_cnt.v_cache_count);
156	}
157
158	u_int
159	kmem_page_count(void)
160	{
161
162	return (vm_cnt.v_page_count);
163	}
164
165	uint64_t	129	uint64_t
166	kmem_size(void)	130	kmem_size(void)
167	{	131	{
Lines 169-181 kmem_size(void) Link Here
169	return (kmem_size_val);	133	return (kmem_size_val);
170	}	134	}
171		135
172	uint64_t
173	kmem_used(void)
174	{
175
176	return (vmem_size(kmem_arena, VMEM_ALLOC));
177	}
178
179	static int	136	static int
180	kmem_std_constructor(void mem, int size __unused, void private, int flags)	137	kmem_std_constructor(void mem, int size __unused, void private, int flags)
181	{	138	{

Lines 201-207 int zfs_arc_shrink_shift = 0; Link Here

(-)sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c (-49 / +54 lines)
201	int zfs_arc_p_min_shift = 0;	201	int zfs_arc_p_min_shift = 0;
202	int zfs_disable_dup_eviction = 0;	202	int zfs_disable_dup_eviction = 0;
203	uint64_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */	203	uint64_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */
204	u_int zfs_arc_free_target = (1 << 19); /* default before pagedaemon init only */	204	u_int zfs_arc_free_target = (1 << 16); /* default before pagedaemon init only */
205		205
206	static int sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS);	206	static int sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS);
207		207
Lines 210-220 static void Link Here
210	arc_free_target_init(void *unused __unused)	210	arc_free_target_init(void *unused __unused)
211	{	211	{
212		212
213	zfs_arc_free_target = kmem_free_target();	213	zfs_arc_free_target = vm_pageout_wakeup_thresh;
214	}	214	}
215	SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY,	215	SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY,
216	arc_free_target_init, NULL);	216	arc_free_target_init, NULL);
217	#endif
218		217
219	TUNABLE_QUAD("vfs.zfs.arc_meta_limit", &zfs_arc_meta_limit);	218	TUNABLE_QUAD("vfs.zfs.arc_meta_limit", &zfs_arc_meta_limit);
220	SYSCTL_DECL(_vfs_zfs);	219	SYSCTL_DECL(_vfs_zfs);
Lines 245-253 sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS Link Here
245	if (err != 0 \|\| req->newptr == NULL)	244	if (err != 0 \|\| req->newptr == NULL)
246	return (err);	245	return (err);
247		246
248	if (val < kmem_free_min())	247	if (val < minfree)
249	return (EINVAL);	248	return (EINVAL);
250	if (val > kmem_page_count())	249	if (val > vm_cnt.v_page_count)
251	return (EINVAL);	250	return (EINVAL);
252		251
253	zfs_arc_free_target = val;	252	zfs_arc_free_target = val;
Lines 254-259 sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS Link Here
254		253
255	return (0);	254	return (0);
256	}	255	}
		256	#endif
257		257
258	/*	258	/*
259	* Note that buffers can be in one of 6 states:	259	* Note that buffers can be in one of 6 states:
Lines 2503-2520 arc_reclaim_needed(void) Link Here
2503	return (1);	2503	return (1);
2504	}	2504	}
2505		2505
2506	if (kmem_free_count() < zfs_arc_free_target) {
2507	DTRACE_PROBE2(arc__reclaim_freetarget, uint64_t,
2508	kmem_free_count(), uint64_t, zfs_arc_free_target);
2509	return (1);
2510	}
2511
2512	/*	2506	/*
2513	* Cooperate with pagedaemon when it's time for it to scan	2507	* Cooperate with pagedaemon when it's time for it to scan
2514	* and reclaim some pages.	2508	* and reclaim some pages.
2515	*/	2509	*/
2516	if (vm_paging_needed()) {	2510	if (freemem < zfs_arc_free_target) {
2517	DTRACE_PROBE(arc__reclaim_paging);	2511	DTRACE_PROBE2(arc__reclaim_freemem, uint64_t,
		2512	freemem, uint64_t, zfs_arc_free_target);
2518	return (1);	2513	return (1);
2519	}	2514	}
2520		2515
Lines 2544-2551 arc_reclaim_needed(void) Link Here
2544	if (availrmem < swapfs_minfree + swapfs_reserve + extra)	2539	if (availrmem < swapfs_minfree + swapfs_reserve + extra)
2545	return (1);	2540	return (1);
2546		2541
2547	#if defined(__i386)
2548	/*	2542	/*
		2543	* Check that we have enough availrmem that memory locking (e.g., via
		2544	* mlock(3C) or memcntl(2)) can still succeed. (pages_pp_maximum
		2545	* stores the number of pages that cannot be locked; when availrmem
		2546	* drops below pages_pp_maximum, page locking mechanisms such as
		2547	* page_pp_lock() will fail.)
		2548	*/
		2549	if (availrmem <= pages_pp_maximum)
		2550	return (1);
		2551
		2552	#endif /* sun */
		2553	#if defined(__i386) \|\| !defined(UMA_MD_SMALL_ALLOC)
		2554	/*
2549	* If we're on an i386 platform, it's possible that we'll exhaust the	2555	* If we're on an i386 platform, it's possible that we'll exhaust the
2550	* kernel heap space before we ever run out of available physical	2556	* kernel heap space before we ever run out of available physical
2551	* memory. Most checks of the size of the heap_area compare against	2557	* memory. Most checks of the size of the heap_area compare against
Lines 2556-2580 arc_reclaim_needed(void) Link Here
2556	* heap is allocated. (Or, in the calculation, if less than 1/4th is	2562	* heap is allocated. (Or, in the calculation, if less than 1/4th is
2557	* free)	2563	* free)
2558	*/	2564	*/
2559	if (btop(vmem_size(heap_arena, VMEM_FREE)) <	2565	if (vmem_size(heap_arena, VMEM_FREE) <
2560	(btop(vmem_size(heap_arena, VMEM_FREE \| VMEM_ALLOC)) >> 2))	2566	(vmem_size(heap_arena, VMEM_FREE \| VMEM_ALLOC) >> 2)) {
2561	return (1);
2562	#endif
2563	#else /* sun */
2564	#ifdef __i386__
2565	/* i386 has KVA limits that the raw page counts above don't consider */
2566	if (kmem_used() > (kmem_size() * 3) / 4) {
2567	DTRACE_PROBE2(arc__reclaim_used, uint64_t,	2567	DTRACE_PROBE2(arc__reclaim_used, uint64_t,
2568	kmem_used(), uint64_t, (kmem_size() * 3) / 4);	2568	vmem_size(heap_arena, VMEM_FREE), uint64_t,
		2569	(vmem_size(heap_arena, VMEM_FREE \| VMEM_ALLOC)) >> 2);
2569	return (1);	2570	return (1);
2570	}	2571	}
2571	#endif	2572	#endif
		2573	#ifdef sun
		2574	/*
		2575	* If zio data pages are being allocated out of a separate heap segment,
		2576	* then enforce that the size of available vmem for this arena remains
		2577	* above about 1/16th free.
		2578	*
		2579	* Note: The 1/16th arena free requirement was put in place
		2580	* to aggressively evict memory from the arc in order to avoid
		2581	* memory fragmentation issues.
		2582	*/
		2583	if (zio_arena != NULL &&
		2584	vmem_size(zio_arena, VMEM_FREE) <
		2585	(vmem_size(zio_arena, VMEM_ALLOC) >> 4))
		2586	return (1);
2572	#endif /* sun */	2587	#endif /* sun */
2573		2588	#else /* _KERNEL */
2574	#else
2575	if (spa_get_random(100) == 0)	2589	if (spa_get_random(100) == 0)
2576	return (1);	2590	return (1);
2577	#endif	2591	#endif /* _KERNEL */
2578	DTRACE_PROBE(arc__reclaim_no);	2592	DTRACE_PROBE(arc__reclaim_no);
2579		2593
2580	return (0);	2594	return (0);
Lines 2625-2630 arc_kmem_reap_now(arc_reclaim_strategy_t strat) Link Here
2625	}	2639	}
2626	kmem_cache_reap_now(buf_cache);	2640	kmem_cache_reap_now(buf_cache);
2627	kmem_cache_reap_now(hdr_cache);	2641	kmem_cache_reap_now(hdr_cache);
		2642
		2643	#ifdef sun
		2644	/*
		2645	* Ask the vmem areana to reclaim unused memory from its
		2646	* quantum caches.
		2647	*/
		2648	if (zio_arena != NULL && strat == ARC_RECLAIM_AGGR)
		2649	vmem_qcache_reap(zio_arena);
		2650	#endif
2628	}	2651	}
2629		2652
2630	static void	2653	static void
Lines 2774-2793 arc_evict_needed(arc_buf_contents_t type) Link Here
2774	if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit)	2797	if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit)
2775	return (1);	2798	return (1);
2776		2799
2777	#ifdef sun
2778	#ifdef _KERNEL
2779	/*
2780	* If zio data pages are being allocated out of a separate heap segment,
2781	* then enforce that the size of available vmem for this area remains
2782	* above about 1/32nd free.
2783	*/
2784	if (type == ARC_BUFC_DATA && zio_arena != NULL &&
2785	vmem_size(zio_arena, VMEM_FREE) <
2786	(vmem_size(zio_arena, VMEM_ALLOC) >> 5))
2787	return (1);
2788	#endif
2789	#endif /* sun */
2790
2791	if (arc_reclaim_needed())	2800	if (arc_reclaim_needed())
2792	return (1);	2801	return (1);
2793		2802
Lines 3946-3965 static int Link Here
3946	arc_memory_throttle(uint64_t reserve, uint64_t txg)	3955	arc_memory_throttle(uint64_t reserve, uint64_t txg)
3947	{	3956	{
3948	#ifdef _KERNEL	3957	#ifdef _KERNEL
3949	uint64_t available_memory =	3958	uint64_t available_memory = ptob(freemem);
3950	ptoa((uintmax_t)vm_cnt.v_free_count + vm_cnt.v_cache_count);
3951	static uint64_t page_load = 0;	3959	static uint64_t page_load = 0;
3952	static uint64_t last_txg = 0;	3960	static uint64_t last_txg = 0;
3953		3961
3954	#ifdef sun	3962	#if defined(__i386) \|\| !defined(UMA_MD_SMALL_ALLOC)
3955	#if defined(__i386)
3956	available_memory =	3963	available_memory =
3957	MIN(available_memory, vmem_size(heap_arena, VMEM_FREE));	3964	MIN(available_memory, ptob(vmem_size(heap_arena, VMEM_FREE)));
3958	#endif	3965	#endif
3959	#endif /* sun */
3960		3966
3961	if (vm_cnt.v_free_count + vm_cnt.v_cache_count >	3967	if (freemem > (uint64_t)physmem * arc_lotsfree_percent / 100)
3962	(uint64_t)physmem * arc_lotsfree_percent / 100)
3963	return (0);	3968	return (0);
3964		3969
3965	if (txg > last_txg) {	3970	if (txg > last_txg) {
Lines 3972-3978 arc_memory_throttle(uint64_t reserve, uint64_t txg Link Here
3972	* continue to let page writes occur as quickly as possible.	3977	* continue to let page writes occur as quickly as possible.
3973	*/	3978	*/
3974	if (curproc == pageproc) {	3979	if (curproc == pageproc) {
3975	if (page_load > available_memory / 4)	3980	if (page_load > MAX(ptob(minfree), available_memory) / 4)
3976	return (SET_ERROR(ERESTART));	3981	return (SET_ERROR(ERESTART));
3977	/* Note: reserve is inflated, so we deflate */	3982	/* Note: reserve is inflated, so we deflate */
3978	page_load += reserve / 8;	3983	page_load += reserve / 8;

Lines 66-82 typedef struct kmem_cache { Link Here

(-)sys/cddl/compat/opensolaris/sys/kmem.h (-11 / +3 lines)
66	void *zfs_kmem_alloc(size_t size, int kmflags);	66	void *zfs_kmem_alloc(size_t size, int kmflags);
67	void zfs_kmem_free(void *buf, size_t size);	67	void zfs_kmem_free(void *buf, size_t size);
68	uint64_t kmem_size(void);	68	uint64_t kmem_size(void);
69	uint64_t kmem_used(void);
70	u_int kmem_page_count(void);
71
72	/*
73	* The return values from kmem_free_* are only valid once the pagedaemon
74	* has been initialised, before then they return 0.
75	*/
76	u_int kmem_free_count(void);
77	u_int kmem_free_target(void);
78	u_int kmem_free_min(void);
79
80	kmem_cache_t kmem_cache_create(char name, size_t bufsize, size_t align,	69	kmem_cache_t kmem_cache_create(char name, size_t bufsize, size_t align,
81	int (constructor)(void , void , int), void (destructor)(void , void ),	70	int (constructor)(void , void , int), void (destructor)(void , void ),
82	void (reclaim)(void ) __unused, void private, vmem_t vmp, int cflags);	71	void (reclaim)(void ) __unused, void private, vmem_t vmp, int cflags);
Lines 88-93 void kmem_reap(void); Link Here
88	int kmem_debugging(void);	77	int kmem_debugging(void);
89	void *calloc(size_t n, size_t s);	78	void *calloc(size_t n, size_t s);
90		79
		80	#define freemem (vm_cnt.v_free_count + vm_cnt.v_cache_count)
		81	#define minfree vm_cnt.v_free_min
		82	#define heap_arena kmem_arena
91	#define kmem_alloc(size, kmflags) zfs_kmem_alloc((size), (kmflags))	83	#define kmem_alloc(size, kmflags) zfs_kmem_alloc((size), (kmflags))
92	#define kmem_zalloc(size, kmflags) zfs_kmem_alloc((size), (kmflags) \| M_ZERO)	84	#define kmem_zalloc(size, kmflags) zfs_kmem_alloc((size), (kmflags) \| M_ZERO)
93	#define kmem_free(buf, size) zfs_kmem_free((buf), (size))	85	#define kmem_free(buf, size) zfs_kmem_free((buf), (size))