Attachment #146856 for bug #191510




}

uint64_t
kmem_map_used(void)
{

	return (kmem_map->size);
}

uint64_t
kmem_map_free(void)
{
	uint64_t size;

	vm_map_lock_read(kmem_map);
	size = kmem_map->root != NULL ? kmem_map->root->max_free :
	    kmem_map->max_offset - kmem_map->min_offset;
	vm_map_unlock_read(kmem_map);

	return (size);
}

static int
kmem_std_constructor(void *mem, int size __unused, void *private, int flags)
{

Lines 66-72 typedef struct kmem_cache { Link Here

(-)sys/cddl/compat/opensolaris/sys/kmem.h (-1 / +4 lines)
66	void *zfs_kmem_alloc(size_t size, int kmflags);	66	void *zfs_kmem_alloc(size_t size, int kmflags);
67	void zfs_kmem_free(void *buf, size_t size);	67	void zfs_kmem_free(void *buf, size_t size);
68	uint64_t kmem_size(void);	68	uint64_t kmem_size(void);
69	uint64_t kmem_used(void);	69	uint64_t kmem_map_used(void);
		70	uint64_t kmem_map_free(void);
70	kmem_cache_t kmem_cache_create(char name, size_t bufsize, size_t align,	71	kmem_cache_t kmem_cache_create(char name, size_t bufsize, size_t align,
71	int (constructor)(void , void , int), void (destructor)(void , void ),	72	int (constructor)(void , void , int), void (destructor)(void , void ),
72	void (reclaim)(void ) __unused, void private, vmem_t vmp, int cflags);	73	void (reclaim)(void ) __unused, void private, vmem_t vmp, int cflags);
Lines 78-83 void kmem_reap(void); Link Here
78	int kmem_debugging(void);	79	int kmem_debugging(void);
79	void *calloc(size_t n, size_t s);	80	void *calloc(size_t n, size_t s);
80		81
		82	#define freemem (cnt.v_free_count + cnt.v_cache_count)
		83	#define minfree cnt.v_free_min
81	#define kmem_alloc(size, kmflags) zfs_kmem_alloc((size), (kmflags))	84	#define kmem_alloc(size, kmflags) zfs_kmem_alloc((size), (kmflags))
82	#define kmem_zalloc(size, kmflags) zfs_kmem_alloc((size), (kmflags) \| M_ZERO)	85	#define kmem_zalloc(size, kmflags) zfs_kmem_alloc((size), (kmflags) \| M_ZERO)
83	#define kmem_free(buf, size) zfs_kmem_free((buf), (size))	86	#define kmem_free(buf, size) zfs_kmem_free((buf), (size))




 */
static boolean_t arc_warm;

/*
 * These tunables are for performance analysis.
 */
uint64_t zfs_arc_max;
uint64_t zfs_arc_min;
uint64_t zfs_arc_meta_limit = 0;

int zfs_arc_shrink_shift = 0;
int zfs_arc_p_min_shift = 0;
int zfs_disable_dup_eviction = 0;
u_int zfs_arc_free_target = (1 << 19); /* default before pagedaemon init only */

static int sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS);

#ifdef _KERNEL
static void
arc_free_target_init(void *unused __unused)
{

	zfs_arc_free_target = cnt.v_free_reserved + cnt.v_cache_min;
}
SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY,
    arc_free_target_init, NULL);

TUNABLE_QUAD("vfs.zfs.arc_max", &zfs_arc_max);
TUNABLE_QUAD("vfs.zfs.arc_min", &zfs_arc_min);
TUNABLE_QUAD("vfs.zfs.arc_meta_limit", &zfs_arc_meta_limit);

    "Minimum ARC size");

/*
 * We don't have a tunable for arc_free_target due to the dependency on
 * pagedaemon initialisation.
 */
SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_free_target,
    CTLTYPE_UINT | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, sizeof(u_int),
    sysctl_vfs_zfs_arc_free_target, "IU",
    "Desired number of free pages below which ARC triggers reclaim");

static int
sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS)
{
	u_int val;
	int err;

	val = zfs_arc_free_target;
	err = sysctl_handle_int(oidp, &val, 0, req);
	if (err != 0 || req->newptr == NULL)
		 return (err);

	if (val < minfree)
		 return (EINVAL);
	if (val > cnt.v_page_count)
		 return (EINVAL);

	zfs_arc_free_target = val;

	return (0);
}
#endif

/*
 * Note that buffers can be in one of 6 states:
 *	ARC_anon	- anonymous (discussed below)
 *	ARC_mru		- recently used, currently cached

void
arc_shrink(void)
{

	if (arc_c > arc_c_min) {
		uint64_t to_free;

		DTRACE_PROBE2(arc__shrink, uint64_t, arc_c, uint64_t,
			arc_c_min);
#ifdef _KERNEL
		to_free = arc_c >> arc_shrink_shift;
#else

		ASSERT((int64_t)arc_p >= 0);
	}

	if (arc_size > arc_c) {
		DTRACE_PROBE2(arc__shrink_adjust, uint64_t, arc_size,
			uint64_t, arc_c);
		arc_adjust();
	}
}

static int needfree = 0;


#ifdef _KERNEL

	if (needfree) {
		DTRACE_PROBE(arc__reclaim_needfree);
		return (1);
	}

	/*
	 * Cooperate with pagedaemon when it's time for it to scan
	 * and reclaim some pages.
	 */
	if (freemem < zfs_arc_free_target) {
		DTRACE_PROBE2(arc__reclaim_freetarget, uint64_t,
		    freemem, uint64_t, zfs_arc_free_target);
		return (1);
	}

#ifdef sun
	/*

	 * heap is allocated.  (Or, in the calculation, if less than 1/4th is
	 * free)
	 */
	if (vmem_size(heap_arena, VMEM_FREE) <
	    (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2))
		return (1);
#endif

	/*
	 * If zio data pages are being allocated out of a separate heap segment,
	 * then enforce that the size of available vmem for this arena remains
	 * above about 1/16th free.
	 *
	 * Note: The 1/16th arena free requirement was put in place
	 * to aggressively evict memory from the arc in order to avoid
	 * memory fragmentation issues.
	 */
	if (zio_arena != NULL &&
	    vmem_size(zio_arena, VMEM_FREE) <
	    (vmem_size(zio_arena, VMEM_ALLOC) >> 4))
		return (1);
#else	/* !sun */
#ifndef UMA_MD_SMALL_ALLOC
	/* i386 has KVA limits that the raw page counts above don't consider */
	if (kmem_map_used() > (kmem_size() * 3) / 4) {
		DTRACE_PROBE2(arc__reclaim_used, uint64_t,
		kmem_map_used(), uint64_t, (kmem_size() * 3) / 4);
		return (1);
	}
#endif
#endif	/* sun */

#else	/* !_KERNEL */
	if (spa_get_random(100) == 0)
		return (1);
#endif	/* _KERNEL */
	DTRACE_PROBE(arc__reclaim_no);

	return (0);
}


	if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit)
		return (1);

#ifdef sun
#ifdef _KERNEL
	/*
	 * If zio data pages are being allocated out of a separate heap segment,
	 * then enforce that the size of available vmem for this area remains
	 * above about 1/32nd free.
	 */
	if (type == ARC_BUFC_DATA && zio_arena != NULL &&
	    vmem_size(zio_arena, VMEM_FREE) <
	    (vmem_size(zio_arena, VMEM_ALLOC) >> 5))
		return (1);
#endif
#endif	/* sun */

	if (arc_reclaim_needed())
		return (1);


arc_memory_throttle(uint64_t reserve, uint64_t txg)
{
#ifdef _KERNEL
	uint64_t available_memory = ptob(freemem);
	    ptoa((uintmax_t)cnt.v_free_count + cnt.v_cache_count);
	static uint64_t page_load = 0;
	static uint64_t last_txg = 0;


	available_memory =
	    MIN(available_memory, vmem_size(heap_arena, VMEM_FREE));
#endif
#else	/* sun */
#ifndef UMA_MD_SMALL_ALLOC
	available_memory = MIN(available_memory, kmem_map_free());
#endif
#endif	/* sun */

	if (freemem > (uint64_t)physmem * arc_lotsfree_percent / 100)
	    (uint64_t)physmem * arc_lotsfree_percent / 100)
		return (0);

	if (txg > last_txg) {

	 * continue to let page writes occur as quickly as possible.
	 */
	if (curproc == pageproc) {
		if (page_load > MAX(ptob(minfree), available_memory) / 4)
			return (SET_ERROR(ERESTART));
		/* Note: reserve is inflated, so we deflate */
		page_load += reserve / 8;

Lines 112-120 __FBSDID("$FreeBSD$"); Link Here

(-)sys/vm/vm_pageout.c (-5 / +16 lines)
112		112
113	/* the kernel process "vm_pageout"*/	113	/* the kernel process "vm_pageout"*/
114	static void vm_pageout(void);	114	static void vm_pageout(void);
		115	static void vm_pageout_init(void);
115	static int vm_pageout_clean(vm_page_t);	116	static int vm_pageout_clean(vm_page_t);
116	static void vm_pageout_scan(int pass);	117	static void vm_pageout_scan(int pass);
117		118
		119	SYSINIT(pagedaemon_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_FIRST, vm_pageout_init,
		120	NULL);
		121
118	struct proc *pageproc;	122	struct proc *pageproc;
119		123
120	static struct kproc_desc page_kp = {	124	static struct kproc_desc page_kp = {
Lines 122-128 static struct kproc_desc page_kp = { Link Here
122	vm_pageout,	126	vm_pageout,
123	&pageproc	127	&pageproc
124	};	128	};
125	SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_FIRST, kproc_start,	129	SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_SECOND, kproc_start,
126	&page_kp);	130	&page_kp);
127		131
128	#if !defined(NO_SWAPPING)	132	#if !defined(NO_SWAPPING)
Lines 1506-1518 vm_pageout_page_stats() Link Here
1506	}	1510	}
1507		1511
1508	/*	1512	/*
1509	* vm_pageout is the high level pageout daemon.	1513	* vm_pageout_init initialises basic pageout daemon settings.
1510	*/	1514	*/
1511	static void	1515	static void
1512	vm_pageout()	1516	vm_pageout_init()
1513	{	1517	{
1514	int error, pass;
1515
1516	/*	1518	/*
1517	* Initialize some paging parameters.	1519	* Initialize some paging parameters.
1518	*/	1520	*/
Lines 1579-1585 static void Link Here
1579	vm_pageout_stats_interval = 5;	1581	vm_pageout_stats_interval = 5;
1580	if (vm_pageout_full_stats_interval == 0)	1582	if (vm_pageout_full_stats_interval == 0)
1581	vm_pageout_full_stats_interval = vm_pageout_stats_interval * 4;	1583	vm_pageout_full_stats_interval = vm_pageout_stats_interval * 4;
		1584	}
1582		1585
		1586	/*
		1587	* vm_pageout is the high level pageout daemon.
		1588	*/
		1589	static void
		1590	vm_pageout()
		1591	{
		1592	int error, pass;
		1593
1583	swap_pager_swap_init();	1594	swap_pager_swap_init();
1584	pass = 0;	1595	pass = 0;
1585	/*	1596	/*

Return to bug 191510

Lines 132-143 kmem_size(void) Link Here

(-)sys/cddl/compat/opensolaris/kern/opensolaris_kmem.c (-1 / +14 lines)
132	}	132	}
133		133
134	uint64_t	134	uint64_t
135	kmem_used(void)	135	kmem_map_used(void)
136	{	136	{
137		137
138	return (kmem_map->size);	138	return (kmem_map->size);
139	}	139	}
140		140
		141	uint64_t
		142	kmem_map_free(void)
		143	{
		144	uint64_t size;
		145
		146	vm_map_lock_read(kmem_map);
		147	size = kmem_map->root != NULL ? kmem_map->root->max_free :
		148	kmem_map->max_offset - kmem_map->min_offset;
		149	vm_map_unlock_read(kmem_map);
		150
		151	return (size);
		152	}
		153
141	static int	154	static int
142	kmem_std_constructor(void mem, int size __unused, void private, int flags)	155	kmem_std_constructor(void mem, int size __unused, void private, int flags)
143	{	156	{

Lines 193-201 extern int zfs_prefetch_disable; Link Here

(-)sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c (-30 / +92 lines)
193	*/	193	*/
194	static boolean_t arc_warm;	194	static boolean_t arc_warm;
195		195
196	/*
197	* These tunables are for performance analysis.
198	*/
199	uint64_t zfs_arc_max;	196	uint64_t zfs_arc_max;
200	uint64_t zfs_arc_min;	197	uint64_t zfs_arc_min;
201	uint64_t zfs_arc_meta_limit = 0;	198	uint64_t zfs_arc_meta_limit = 0;
Lines 203-209 int zfs_arc_grow_retry = 0; Link Here
203	int zfs_arc_shrink_shift = 0;	200	int zfs_arc_shrink_shift = 0;
204	int zfs_arc_p_min_shift = 0;	201	int zfs_arc_p_min_shift = 0;
205	int zfs_disable_dup_eviction = 0;	202	int zfs_disable_dup_eviction = 0;
		203	u_int zfs_arc_free_target = (1 << 19); /* default before pagedaemon init only */
206		204
		205	static int sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS);
		206
		207	#ifdef _KERNEL
		208	static void
		209	arc_free_target_init(void *unused __unused)
		210	{
		211
		212	zfs_arc_free_target = cnt.v_free_reserved + cnt.v_cache_min;
		213	}
		214	SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY,
		215	arc_free_target_init, NULL);
		216
207	TUNABLE_QUAD("vfs.zfs.arc_max", &zfs_arc_max);	217	TUNABLE_QUAD("vfs.zfs.arc_max", &zfs_arc_max);
208	TUNABLE_QUAD("vfs.zfs.arc_min", &zfs_arc_min);	218	TUNABLE_QUAD("vfs.zfs.arc_min", &zfs_arc_min);
209	TUNABLE_QUAD("vfs.zfs.arc_meta_limit", &zfs_arc_meta_limit);	219	TUNABLE_QUAD("vfs.zfs.arc_meta_limit", &zfs_arc_meta_limit);
Lines 214-219 SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_min, CTLFLAG_ Link Here
214	"Minimum ARC size");	224	"Minimum ARC size");
215		225
216	/*	226	/*
		227	* We don't have a tunable for arc_free_target due to the dependency on
		228	* pagedaemon initialisation.
		229	*/
		230	SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_free_target,
		231	CTLTYPE_UINT \| CTLFLAG_MPSAFE \| CTLFLAG_RW, 0, sizeof(u_int),
		232	sysctl_vfs_zfs_arc_free_target, "IU",
		233	"Desired number of free pages below which ARC triggers reclaim");
		234
		235	static int
		236	sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS)
		237	{
		238	u_int val;
		239	int err;
		240
		241	val = zfs_arc_free_target;
		242	err = sysctl_handle_int(oidp, &val, 0, req);
		243	if (err != 0 \|\| req->newptr == NULL)
		244	return (err);
		245
		246	if (val < minfree)
		247	return (EINVAL);
		248	if (val > cnt.v_page_count)
		249	return (EINVAL);
		250
		251	zfs_arc_free_target = val;
		252
		253	return (0);
		254	}
		255	#endif
		256
		257	/*
217	* Note that buffers can be in one of 6 states:	258	* Note that buffers can be in one of 6 states:
218	* ARC_anon - anonymous (discussed below)	259	* ARC_anon - anonymous (discussed below)
219	* ARC_mru - recently used, currently cached	260	* ARC_mru - recently used, currently cached
Lines 2405-2413 arc_flush(spa_t *spa) Link Here
2405	void	2446	void
2406	arc_shrink(void)	2447	arc_shrink(void)
2407	{	2448	{
		2449
2408	if (arc_c > arc_c_min) {	2450	if (arc_c > arc_c_min) {
2409	uint64_t to_free;	2451	uint64_t to_free;
2410		2452
		2453	DTRACE_PROBE2(arc__shrink, uint64_t, arc_c, uint64_t,
		2454	arc_c_min);
2411	#ifdef _KERNEL	2455	#ifdef _KERNEL
2412	to_free = arc_c >> arc_shrink_shift;	2456	to_free = arc_c >> arc_shrink_shift;
2413	#else	2457	#else
Lines 2427-2434 arc_shrink(void) Link Here
2427	ASSERT((int64_t)arc_p >= 0);	2471	ASSERT((int64_t)arc_p >= 0);
2428	}	2472	}
2429		2473
2430	if (arc_size > arc_c)	2474	if (arc_size > arc_c) {
		2475	DTRACE_PROBE2(arc__shrink_adjust, uint64_t, arc_size,
		2476	uint64_t, arc_c);
2431	arc_adjust();	2477	arc_adjust();
		2478	}
2432	}	2479	}
2433		2480
2434	static int needfree = 0;	2481	static int needfree = 0;
Lines 2439-2453 arc_reclaim_needed(void) Link Here
2439		2486
2440	#ifdef _KERNEL	2487	#ifdef _KERNEL
2441		2488
2442	if (needfree)	2489	if (needfree) {
		2490	DTRACE_PROBE(arc__reclaim_needfree);
2443	return (1);	2491	return (1);
		2492	}
2444		2493
2445	/*	2494	/*
2446	* Cooperate with pagedaemon when it's time for it to scan	2495	* Cooperate with pagedaemon when it's time for it to scan
2447	* and reclaim some pages.	2496	* and reclaim some pages.
2448	*/	2497	*/
2449	if (vm_paging_needed())	2498	if (freemem < zfs_arc_free_target) {
		2499	DTRACE_PROBE2(arc__reclaim_freetarget, uint64_t,
		2500	freemem, uint64_t, zfs_arc_free_target);
2450	return (1);	2501	return (1);
		2502	}
2451		2503
2452	#ifdef sun	2504	#ifdef sun
2453	/*	2505	/*
Lines 2487-2505 arc_reclaim_needed(void) Link Here
2487	* heap is allocated. (Or, in the calculation, if less than 1/4th is	2539	* heap is allocated. (Or, in the calculation, if less than 1/4th is
2488	* free)	2540	* free)
2489	*/	2541	*/
2490	if (btop(vmem_size(heap_arena, VMEM_FREE)) <	2542	if (vmem_size(heap_arena, VMEM_FREE) <
2491	(btop(vmem_size(heap_arena, VMEM_FREE \| VMEM_ALLOC)) >> 2))	2543	(vmem_size(heap_arena, VMEM_FREE \| VMEM_ALLOC) >> 2))
2492	return (1);	2544	return (1);
2493	#endif	2545	#endif
		2546
		2547	/*
		2548	* If zio data pages are being allocated out of a separate heap segment,
		2549	* then enforce that the size of available vmem for this arena remains
		2550	* above about 1/16th free.
		2551	*
		2552	* Note: The 1/16th arena free requirement was put in place
		2553	* to aggressively evict memory from the arc in order to avoid
		2554	* memory fragmentation issues.
		2555	*/
		2556	if (zio_arena != NULL &&
		2557	vmem_size(zio_arena, VMEM_FREE) <
		2558	(vmem_size(zio_arena, VMEM_ALLOC) >> 4))
		2559	return (1);
2494	#else /* !sun */	2560	#else /* !sun */
2495	if (kmem_used() > (kmem_size() * 3) / 4)	2561	#ifndef UMA_MD_SMALL_ALLOC
		2562	/* i386 has KVA limits that the raw page counts above don't consider */
		2563	if (kmem_map_used() > (kmem_size() * 3) / 4) {
		2564	DTRACE_PROBE2(arc__reclaim_used, uint64_t,
		2565	kmem_map_used(), uint64_t, (kmem_size() * 3) / 4);
2496	return (1);	2566	return (1);
		2567	}
		2568	#endif
2497	#endif /* sun */	2569	#endif /* sun */
2498		2570
2499	#else	2571	#else /* !_KERNEL */
2500	if (spa_get_random(100) == 0)	2572	if (spa_get_random(100) == 0)
2501	return (1);	2573	return (1);
2502	#endif	2574	#endif /* _KERNEL */
		2575	DTRACE_PROBE(arc__reclaim_no);
		2576
2503	return (0);	2577	return (0);
2504	}	2578	}
2505		2579
Lines 2697-2716 arc_evict_needed(arc_buf_contents_t type) Link Here
2697	if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit)	2771	if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit)
2698	return (1);	2772	return (1);
2699		2773
2700	#ifdef sun
2701	#ifdef _KERNEL
2702	/*
2703	* If zio data pages are being allocated out of a separate heap segment,
2704	* then enforce that the size of available vmem for this area remains
2705	* above about 1/32nd free.
2706	*/
2707	if (type == ARC_BUFC_DATA && zio_arena != NULL &&
2708	vmem_size(zio_arena, VMEM_FREE) <
2709	(vmem_size(zio_arena, VMEM_ALLOC) >> 5))
2710	return (1);
2711	#endif
2712	#endif /* sun */
2713
2714	if (arc_reclaim_needed())	2774	if (arc_reclaim_needed())
2715	return (1);	2775	return (1);
2716		2776
Lines 3876-3883 static int Link Here
3876	arc_memory_throttle(uint64_t reserve, uint64_t txg)	3936	arc_memory_throttle(uint64_t reserve, uint64_t txg)
3877	{	3937	{
3878	#ifdef _KERNEL	3938	#ifdef _KERNEL
3879	uint64_t available_memory =	3939	uint64_t available_memory = ptob(freemem);
3880	ptoa((uintmax_t)cnt.v_free_count + cnt.v_cache_count);
3881	static uint64_t page_load = 0;	3940	static uint64_t page_load = 0;
3882	static uint64_t last_txg = 0;	3941	static uint64_t last_txg = 0;
3883		3942
Lines 3886-3895 arc_memory_throttle(uint64_t reserve, uint64_t txg Link Here
3886	available_memory =	3945	available_memory =
3887	MIN(available_memory, vmem_size(heap_arena, VMEM_FREE));	3946	MIN(available_memory, vmem_size(heap_arena, VMEM_FREE));
3888	#endif	3947	#endif
		3948	#else /* sun */
		3949	#ifndef UMA_MD_SMALL_ALLOC
		3950	available_memory = MIN(available_memory, kmem_map_free());
		3951	#endif
3889	#endif /* sun */	3952	#endif /* sun */
3890		3953
3891	if (cnt.v_free_count + cnt.v_cache_count >	3954	if (freemem > (uint64_t)physmem * arc_lotsfree_percent / 100)
3892	(uint64_t)physmem * arc_lotsfree_percent / 100)
3893	return (0);	3955	return (0);
3894		3956
3895	if (txg > last_txg) {	3957	if (txg > last_txg) {
Lines 3902-3908 arc_memory_throttle(uint64_t reserve, uint64_t txg Link Here
3902	* continue to let page writes occur as quickly as possible.	3964	* continue to let page writes occur as quickly as possible.
3903	*/	3965	*/
3904	if (curproc == pageproc) {	3966	if (curproc == pageproc) {
3905	if (page_load > available_memory / 4)	3967	if (page_load > MAX(ptob(minfree), available_memory) / 4)
3906	return (SET_ERROR(ERESTART));	3968	return (SET_ERROR(ERESTART));
3907	/* Note: reserve is inflated, so we deflate */	3969	/* Note: reserve is inflated, so we deflate */
3908	page_load += reserve / 8;	3970	page_load += reserve / 8;