Attachment #147607 for bug #187594

View | Details | Raw Unified | Return to bug 187594 | Differences between

and this patch




	return (kmem_size_val);
}

uint64_t
kmem_used(void)
{

	return (vmem_size(kmem_arena, VMEM_ALLOC));
}

static int
kmem_std_constructor(void *mem, int size __unused, void *private, int flags)
{




void *zfs_kmem_alloc(size_t size, int kmflags);
void zfs_kmem_free(void *buf, size_t size);
uint64_t kmem_size(void);
uint64_t kmem_used(void);
kmem_cache_t *kmem_cache_create(char *name, size_t bufsize, size_t align,
    int (*constructor)(void *, void *, int), void (*destructor)(void *, void *),
    void (*reclaim)(void *) __unused, void *private, vmem_t *vmp, int cflags);

int kmem_debugging(void);
void *calloc(size_t n, size_t s);

#define	freemem				(cnt.v_free_count + cnt.v_cache_count)
#define	minfree				cnt.v_free_min
#define	heap_arena			kmem_arena
#define	kmem_alloc(size, kmflags)	zfs_kmem_alloc((size), (kmflags))
#define	kmem_zalloc(size, kmflags)	zfs_kmem_alloc((size), (kmflags) | M_ZERO)
#define	kmem_free(buf, size)		zfs_kmem_free((buf), (size))




#include <sys/sdt.h>

#include <vm/vm_pageout.h>
#include <machine/vmparam.h>

#ifdef illumos
#ifndef _KERNEL

static int arc_dead;
extern int zfs_prefetch_disable;

/* 
 * KD 2014-09-22
 * We have to be able to test for UIO use inside the arc allocator.
 * NOTE: DO NOT MODIFY HERE!
 */
extern int zio_use_uma;

/*
 * The arc has filled available memory and has now warmed up.
 */
static boolean_t arc_warm;

/*
 * These tunables are for performance analysis.
 */
uint64_t zfs_arc_max;
uint64_t zfs_arc_min;
uint64_t zfs_arc_meta_limit = 0;

int zfs_arc_p_min_shift = 0;
int zfs_disable_dup_eviction = 0;
uint64_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */
u_int zfs_arc_free_target = (1 << 16); /* default before pagedaemon init only */
int zfs_arc_reclaim_cache_free = 1;

static int sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS);

#ifdef _KERNEL
static void
arc_free_target_init(void *unused __unused)
{
	zfs_arc_free_target = vm_pageout_wakeup_thresh + ((cnt.v_free_target - vm_pageout_wakeup_thresh) / 2);
}
SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY,
    arc_free_target_init, NULL);

TUNABLE_QUAD("vfs.zfs.arc_max", &zfs_arc_max);
TUNABLE_QUAD("vfs.zfs.arc_min", &zfs_arc_min);
TUNABLE_QUAD("vfs.zfs.arc_meta_limit", &zfs_arc_meta_limit);

SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, arc_average_blocksize, CTLFLAG_RDTUN,
    &zfs_arc_average_blocksize, 0,
    "ARC average blocksize");
SYSCTL_INT(_vfs_zfs, OID_AUTO, arc_reclaim_cache_free, CTLFLAG_RWTUN,
    &zfs_arc_reclaim_cache_free, 0,
    "ARC treats cached pages as free blocksize");
/*
 * We don't have a tunable for arc_free_target due to the dependency on
 * pagedaemon initialisation.
 */
SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_free_target,
    CTLTYPE_UINT | CTLFLAG_MPSAFE | CTLFLAG_RW, 0, sizeof(u_int),
    sysctl_vfs_zfs_arc_free_target, "IU",
    "Desired number of free pages below which ARC triggers reclaim");

static int
sysctl_vfs_zfs_arc_free_target(SYSCTL_HANDLER_ARGS)
{
	u_int val;
	int err;

	val = zfs_arc_free_target;
	err = sysctl_handle_int(oidp, &val, 0, req);
	if (err != 0 || req->newptr == NULL)
		return (err);

	if (val < minfree)
		return (EINVAL);
	if (val > cnt.v_page_count)
		return (EINVAL);

	zfs_arc_free_target = val;

	return (0);
}
#endif

/*
 * Note that buffers can be in one of 6 states:
 *	ARC_anon	- anonymous (discussed below)

void
arc_shrink(void)
{

	if (arc_c > arc_c_min) {
		uint64_t to_free;


#else
		to_free = arc_c >> arc_shrink_shift;
#endif
		DTRACE_PROBE4(arc__shrink, uint64_t, arc_c, uint64_t,
			arc_c_min, uint64_t, arc_p, uint64_t, to_free);

		if (arc_c > arc_c_min + to_free)
			atomic_add_64(&arc_c, -to_free);
		else

			arc_c = MAX(arc_size, arc_c_min);
		if (arc_p > arc_c)
			arc_p = (arc_c >> 1);

		DTRACE_PROBE2(arc__shrunk, uint64_t, arc_c, uint64_t,
			arc_p);

		ASSERT(arc_c >= arc_c_min);
		ASSERT((int64_t)arc_p >= 0);
	}

	if (arc_size > arc_c) {
		DTRACE_PROBE2(arc__shrink_adjust, uint64_t, arc_size,
			uint64_t, arc_c);
		arc_adjust();
	}
}

static int needfree = 0;

static int
arc_reclaim_needed(void)
{
	u_int fm;

#ifdef _KERNEL
	if (arc_size <= arc_c_min) {
		DTRACE_PROBE2(arc__reclaim_min, uint64_t, arc_size,
		    uint64_t, arc_c_min);
		return (0);
	}

	if (needfree) {
		DTRACE_PROBE(arc__reclaim_needfree);
		return (1);
	}

	/*
	 * Cooperate with pagedaemon when it's time for it to scan
	 * and reclaim some pages.
	 */
	if (zfs_arc_reclaim_cache_free == 0)
		fm = cnt.v_free_count;
	else
		fm = freemem;

	if (fm < zfs_arc_free_target) {
		DTRACE_PROBE3(arc__reclaim_freemem, uint64_t,
		    fm, uint64_t, zfs_arc_free_target,
		    int, zfs_arc_reclaim_cache_free);
		return (1);
	}

#ifdef sun
	/*

	if (availrmem < swapfs_minfree + swapfs_reserve + extra)
		return (1);

#if defined(__i386)
	/*
	 * Check that we have enough availrmem that memory locking (e.g., via
	 * mlock(3C) or memcntl(2)) can still succeed.  (pages_pp_maximum
	 * stores the number of pages that cannot be locked; when availrmem
	 * drops below pages_pp_maximum, page locking mechanisms such as
	 * page_pp_lock() will fail.)
	 */
	if (availrmem <= pages_pp_maximum)
		return (1);

#endif	/* sun */
#if defined(__i386) || !defined(UMA_MD_SMALL_ALLOC)
	/*
	 * If we're on an i386 platform, it's possible that we'll exhaust the
	 * kernel heap space before we ever run out of available physical
	 * memory.  Most checks of the size of the heap_area compare against

	 * heap is allocated.  (Or, in the calculation, if less than 1/4th is
	 * free)
	 */
	if (vmem_size(heap_arena, VMEM_FREE) <
	    (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2)) {
		DTRACE_PROBE2(arc__reclaim_used, uint64_t,
		    vmem_size(heap_arena, VMEM_FREE), uint64_t,
		    (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC)) >> 2);
		return (1);
	}
#endif
#ifdef sun
	/*
	 * If zio data pages are being allocated out of a separate heap segment,
	 * then enforce that the size of available vmem for this arena remains
	 * above about 1/16th free.
	 *
	 * Note: The 1/16th arena free requirement was put in place
	 * to aggressively evict memory from the arc in order to avoid
	 * memory fragmentation issues.
	 */
	if (zio_arena != NULL &&
	    vmem_size(zio_arena, VMEM_FREE) <
	    (vmem_size(zio_arena, VMEM_ALLOC) >> 4))
		return (1);
#endif	/* sun */
#else	/* _KERNEL */
#else
	if (spa_get_random(100) == 0)
		return (1);
#endif	/* _KERNEL */
	DTRACE_PROBE(arc__reclaim_no);

	return (0);
}

extern kmem_cache_t	*zio_buf_cache[];
extern kmem_cache_t	*zio_data_buf_cache[];

static void __used
reap_arc_caches()
{
	size_t		i;
	kmem_cache_t		*prev_cache = NULL;
	kmem_cache_t		*prev_data_cache = NULL;

	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
		if (zio_buf_cache[i] != prev_cache) {
			prev_cache = zio_buf_cache[i];
			kmem_cache_reap_now(zio_buf_cache[i]);
		}
		if (zio_data_buf_cache[i] != prev_data_cache) {
			prev_data_cache = zio_data_buf_cache[i];
			kmem_cache_reap_now(zio_data_buf_cache[i]);
		}
	}
	kmem_cache_reap_now(buf_cache);
	kmem_cache_reap_now(hdr_cache);
}

static void __used
arc_kmem_reap_now(arc_reclaim_strategy_t strat)
{

	DTRACE_PROBE(arc__kmem_reap_start);
#ifdef _KERNEL
	if (arc_meta_used >= arc_meta_limit) {
		/*

	if (strat == ARC_RECLAIM_AGGR)
		arc_shrink();

	reap_arc_caches();

#ifdef sun
	/*
	 * Ask the vmem areana to reclaim unused memory from its
	 * quantum caches.
	 */
	if (zio_arena != NULL && strat == ARC_RECLAIM_AGGR)
		vmem_qcache_reap(zio_arena);
#endif
	DTRACE_PROBE(arc__kmem_reap_end);
	kmem_cache_reap_now(hdr_cache);
}

static void

	clock_t			growtime = 0;
	arc_reclaim_strategy_t	last_reclaim = ARC_RECLAIM_CONS;
	callb_cpr_t		cpr;
	int			autoreap = 0;

	CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG);

	mutex_enter(&arc_reclaim_thr_lock);

	while (arc_thread_exit == 0) {

#ifdef	_KERNEL
/* KD 2014-09-22
 * Protect against UMA free memory bloat.  We already do this on a low-memory
 * basis in the allocator; it has to happen there rather than here due to 
 * response time considerations.  Make the call here once every 10 passes as 
 * well; this reclaims unused UMA buffers every 10 seconds on an idle system 
 * and more frequently if the reclaim thread gets woken up by low RAM 
 * conditions.
 */
		if ((zio_use_uma) && (autoreap++ == 10)) {
			autoreap = 0;
			DTRACE_PROBE(arc__reclaim_timed_reap);
			reap_arc_caches();
		}
#endif	/* _KERNEL */

		if (arc_reclaim_needed()) {

			if (arc_no_grow) {
				if (last_reclaim == ARC_RECLAIM_CONS) {
					DTRACE_PROBE(arc__reclaim_aggr_no_grow);
					last_reclaim = ARC_RECLAIM_AGGR;
				} else {
					last_reclaim = ARC_RECLAIM_CONS;

			} else {
				arc_no_grow = TRUE;
				last_reclaim = ARC_RECLAIM_AGGR;
				DTRACE_PROBE(arc__reclaim_aggr);
				membar_producer();
			}


				 */
				arc_no_grow = TRUE;
				last_reclaim = ARC_RECLAIM_AGGR;
				DTRACE_PROBE(arc__reclaim_aggr_needfree);
			}
			arc_kmem_reap_now(last_reclaim);
			arc_warm = B_TRUE;

#ifdef _KERNEL
		if (needfree) {
			needfree = 0;
			DTRACE_PROBE(arc__clear_needfree);
			wakeup(&needfree);
		}
#endif

	 * cache size, increment the target cache size
	 */
	if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) {
		DTRACE_PROBE1(arc__inc_adapt, int, bytes);
		atomic_add_64(&arc_c, (int64_t)bytes);
		if (arc_c > arc_c_max)
			arc_c = arc_c_max;

	if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit)
		return (1);

#ifdef sun
#ifdef _KERNEL
	/*
	 * If zio data pages are being allocated out of a separate heap segment,
	 * then enforce that the size of available vmem for this area remains
	 * above about 1/32nd free.
	 */
	if (type == ARC_BUFC_DATA && zio_arena != NULL &&
	    vmem_size(zio_arena, VMEM_FREE) <
	    (vmem_size(zio_arena, VMEM_ALLOC) >> 5))
		return (1);
#endif
#endif	/* sun */

	if (arc_reclaim_needed())
		return (1);


			arc_space_consume(size, ARC_SPACE_DATA);
		} else {
			ASSERT(type == ARC_BUFC_DATA);
#ifdef	_KERNEL
/* KD 2014-09-22
 * It would be nice if we could leave this to the arc_reclaim thread.
 * Unfortunately we cannot; the test has to be done here as well, because
 * under heavy I/O demand we can grab enough RAM fast enough to induce
 * nasty oscillation problems.  Fortunately we only need to call this when
 * the system is under reasonably-severe memory stress.
 */
			if (zio_use_uma && (ptob(cnt.v_free_count) + size < ptob(cnt.v_free_target))) {
				DTRACE_PROBE3(arc__alloc_lowmem_reap, int, cnt.v_free_count, int, size, int, cnt.v_free_target); 
				reap_arc_caches();
			}
#endif	/* _KERNEL */
			buf->b_data = zio_data_buf_alloc(size);
			ARCSTAT_INCR(arcstat_data_size, size);
			atomic_add_64(&arc_size, size);

arc_memory_throttle(uint64_t reserve, uint64_t txg)
{
#ifdef _KERNEL
	uint64_t available_memory = ptob(freemem);
	    ptoa((uintmax_t)cnt.v_free_count + cnt.v_cache_count);
	static uint64_t page_load = 0;
	static uint64_t last_txg = 0;

#if defined(__i386) || !defined(UMA_MD_SMALL_ALLOC)
#if defined(__i386)
	available_memory =
	    MIN(available_memory, ptob(vmem_size(heap_arena, VMEM_FREE)));
#endif
#endif	/* sun */

	if (freemem > (uint64_t)physmem * arc_lotsfree_percent / 100)
	    (uint64_t)physmem * arc_lotsfree_percent / 100)
		return (0);

	if (txg > last_txg) {

	 * continue to let page writes occur as quickly as possible.
	 */
	if (curproc == pageproc) {
		if (page_load > MAX(ptob(minfree), available_memory) / 4)
			return (SET_ERROR(ERESTART));
		/* Note: reserve is inflated, so we deflate */
		page_load += reserve / 8;

	int error;
	uint64_t anon_size;

	if (reserve > arc_c/4 && !arc_no_grow) {
		arc_c = MIN(arc_c_max, reserve * 4);
		DTRACE_PROBE1(arc__set_reserve, uint64_t, arc_c);
	}
	if (reserve > arc_c)
		return (SET_ERROR(ENOMEM));


	mutex_enter(&arc_lowmem_lock);
	mutex_enter(&arc_reclaim_thr_lock);
	needfree = 1;
	DTRACE_PROBE(arc__needfree);
	cv_signal(&arc_reclaim_thr_cv);

	/*




typedef void (*dmu_tx_hold_func_t)(dmu_tx_t *tx, struct dnode *dn,
    uint64_t arg1, uint64_t arg2);

extern	int	zio_use_uma;	/* Needs to be visible; DO NOT MODIFY! */

dmu_tx_t *
dmu_tx_create_dd(dsl_dir_t *dd)

	if (txg_how == TXG_WAITED)
		tx->tx_waited = B_TRUE;

#ifdef	_KERNEL
	/*
 	 * KD 2014-09-22
 	 * If UMA is enabled it can only return a previously-used block
	 * of identical size to what it had out before.  If it's not the
	 * same size it will allocate a new one.  This is a problem because
	 * dirty_data_max is the total dirty write data allowed out at any 
	 * given time, but with UMA on that can multiply by the number of 
	 * different block sizes (!!) requested in terms of free RAM that 
	 * is left allocated but unused.  For this reason never allow 
	 * dirty_data_max to exceed the difference between the paging 
	 * threshold and the current free memory, with a minimum of 256MB.
	 * This throttles "burst" allocations and prevents the system from 
	 * choking during times of high write I/O demand.
 	 */
	if (zio_use_uma) {
		zfs_dirty_data_max = 1 << 28;
		zfs_dirty_data_max = MAX(zfs_dirty_data_max, ptob(cnt.v_free_count - cnt.v_free_target));
		zfs_dirty_data_max = MIN(zfs_dirty_data_max, ptob(physmem) * zfs_dirty_data_max_percent / 100);
		zfs_dirty_data_max = MIN(zfs_dirty_data_max, zfs_dirty_data_max_max);
		DTRACE_PROBE1(dmu__tx_dirty, uint64_t, zfs_dirty_data_max / (1024 * 1024));
	}
#endif	/* _KERNEL */

	while ((err = dmu_tx_try_assign(tx, txg_how)) != 0) {
		dmu_tx_unassign(tx);





__FBSDID("$FreeBSD$");

#include "opt_vm.h"
#include "opt_kdtrace.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>

#include <sys/racct.h>
#include <sys/resourcevar.h>
#include <sys/sched.h>
#include <sys/sdt.h>
#include <sys/signalvar.h>
#include <sys/smp.h>
#include <sys/vnode.h>


/* the kernel process "vm_pageout"*/
static void vm_pageout(void);
static void vm_pageout_init(void);
static int vm_pageout_clean(vm_page_t);
static void vm_pageout_scan(struct vm_domain *vmd, int pass);
static void vm_pageout_mightbe_oom(struct vm_domain *vmd, int pass);

SYSINIT(pagedaemon_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_FIRST, vm_pageout_init,
    NULL);

struct proc *pageproc;

static struct kproc_desc page_kp = {

	vm_pageout,
	&pageproc
};
SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_SECOND, kproc_start,
    &page_kp);

SDT_PROVIDER_DEFINE(vm);
SDT_PROBE_DEFINE(vm, , , vm__lowmem_cache);
SDT_PROBE_DEFINE(vm, , , vm__lowmem_scan);

#if !defined(NO_SWAPPING)
/* the kernel process "vm_daemon"*/
static void vm_daemon(void);

		 * may acquire locks and/or sleep, so they can only be invoked
		 * when "tries" is greater than zero.
		 */
		SDT_PROBE0(vm, , , vm__lowmem_cache);
		EVENTHANDLER_INVOKE(vm_lowmem, 0);

		/*

 *	pass 1 - Move inactive to cache or free
 *	pass 2 - Launder dirty pages
 */
static void __used
vm_pageout_scan(struct vm_domain *vmd, int pass)
{
	vm_page_t m, next;

		/*
		 * Decrease registered cache sizes.
		 */
		SDT_PROBE0(vm, , , vm__lowmem_scan);
		EVENTHANDLER_INVOKE(vm_lowmem, 0);
		/*
		 * We do this explicitly after the caches have been

}

/*
 *	vm_pageout_init initialises basic pageout daemon settings.
 */
static void
vm_pageout_init(void)
{
#if MAXMEMDOM > 1
	int error, i;
#endif

	/*
	 * Initialize some paging parameters.
	 */

	/* XXX does not really belong here */
	if (vm_page_max_wired == 0)
		vm_page_max_wired = cnt.v_free_count / 3;
}

/*
 *     vm_pageout is the high level pageout daemon.
 */
static void
vm_pageout(void)
{
#if MAXMEMDOM > 1
	int error, i;
#endif

	swap_pager_swap_init();
#if MAXMEMDOM > 1
	for (i = 1; i < vm_ndomains; i++) {

Return to bug 187594

Lines 133-145 kmem_size(void) Link Here

(-)sys/cddl/compat/opensolaris/kern/opensolaris_kmem.c (-7 lines)
133	return (kmem_size_val);	133	return (kmem_size_val);
134	}	134	}
135		135
136	uint64_t
137	kmem_used(void)
138	{
139
140	return (vmem_size(kmem_arena, VMEM_ALLOC));
141	}
142
143	static int	136	static int
144	kmem_std_constructor(void mem, int size __unused, void private, int flags)	137	kmem_std_constructor(void mem, int size __unused, void private, int flags)
145	{	138	{

Lines 66-72 typedef struct kmem_cache { Link Here

(-)sys/cddl/compat/opensolaris/sys/kmem.h (-1 / +3 lines)
66	void *zfs_kmem_alloc(size_t size, int kmflags);	66	void *zfs_kmem_alloc(size_t size, int kmflags);
67	void zfs_kmem_free(void *buf, size_t size);	67	void zfs_kmem_free(void *buf, size_t size);
68	uint64_t kmem_size(void);	68	uint64_t kmem_size(void);
69	uint64_t kmem_used(void);
70	kmem_cache_t kmem_cache_create(char name, size_t bufsize, size_t align,	69	kmem_cache_t kmem_cache_create(char name, size_t bufsize, size_t align,
71	int (constructor)(void , void , int), void (destructor)(void , void ),	70	int (constructor)(void , void , int), void (destructor)(void , void ),
72	void (reclaim)(void ) __unused, void private, vmem_t vmp, int cflags);	71	void (reclaim)(void ) __unused, void private, vmem_t vmp, int cflags);
Lines 78-83 void kmem_reap(void); Link Here
78	int kmem_debugging(void);	77	int kmem_debugging(void);
79	void *calloc(size_t n, size_t s);	78	void *calloc(size_t n, size_t s);
80		79
		80	#define freemem (cnt.v_free_count + cnt.v_cache_count)
		81	#define minfree cnt.v_free_min
		82	#define heap_arena kmem_arena
81	#define kmem_alloc(size, kmflags) zfs_kmem_alloc((size), (kmflags))	83	#define kmem_alloc(size, kmflags) zfs_kmem_alloc((size), (kmflags))
82	#define kmem_zalloc(size, kmflags) zfs_kmem_alloc((size), (kmflags) \| M_ZERO)	84	#define kmem_zalloc(size, kmflags) zfs_kmem_alloc((size), (kmflags) \| M_ZERO)
83	#define kmem_free(buf, size) zfs_kmem_free((buf), (size))	85	#define kmem_free(buf, size) zfs_kmem_free((buf), (size))

Lines 42-47 Link Here

(-)sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c (+25 lines)
42	typedef void (dmu_tx_hold_func_t)(dmu_tx_t tx, struct dnode *dn,	42	typedef void (dmu_tx_hold_func_t)(dmu_tx_t tx, struct dnode *dn,
43	uint64_t arg1, uint64_t arg2);	43	uint64_t arg1, uint64_t arg2);
44		44
		45	extern int zio_use_uma; /* Needs to be visible; DO NOT MODIFY! */
45		46
46	dmu_tx_t *	47	dmu_tx_t *
47	dmu_tx_create_dd(dsl_dir_t *dd)	48	dmu_tx_create_dd(dsl_dir_t *dd)
Lines 1281-1286 dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) Link Here
1281	if (txg_how == TXG_WAITED)	1282	if (txg_how == TXG_WAITED)
1282	tx->tx_waited = B_TRUE;	1283	tx->tx_waited = B_TRUE;
1283		1284
		1285	#ifdef _KERNEL
		1286	/*
		1287	* KD 2014-09-22
		1288	* If UMA is enabled it can only return a previously-used block
		1289	* of identical size to what it had out before. If it's not the
		1290	* same size it will allocate a new one. This is a problem because
		1291	* dirty_data_max is the total dirty write data allowed out at any
		1292	* given time, but with UMA on that can multiply by the number of
		1293	* different block sizes (!!) requested in terms of free RAM that
		1294	* is left allocated but unused. For this reason never allow
		1295	* dirty_data_max to exceed the difference between the paging
		1296	* threshold and the current free memory, with a minimum of 256MB.
		1297	* This throttles "burst" allocations and prevents the system from
		1298	* choking during times of high write I/O demand.
		1299	*/
		1300	if (zio_use_uma) {
		1301	zfs_dirty_data_max = 1 << 28;
		1302	zfs_dirty_data_max = MAX(zfs_dirty_data_max, ptob(cnt.v_free_count - cnt.v_free_target));
		1303	zfs_dirty_data_max = MIN(zfs_dirty_data_max, ptob(physmem) * zfs_dirty_data_max_percent / 100);
		1304	zfs_dirty_data_max = MIN(zfs_dirty_data_max, zfs_dirty_data_max_max);
		1305	DTRACE_PROBE1(dmu__tx_dirty, uint64_t, zfs_dirty_data_max / (1024 * 1024));
		1306	}
		1307	#endif /* _KERNEL */
		1308
1284	while ((err = dmu_tx_try_assign(tx, txg_how)) != 0) {	1309	while ((err = dmu_tx_try_assign(tx, txg_how)) != 0) {
1285	dmu_tx_unassign(tx);	1310	dmu_tx_unassign(tx);
1286		1311

Lines 76-81 Link Here

(-)sys/vm/vm_pageout.c (-8 / +27 lines)
76	__FBSDID("$FreeBSD$");	76	__FBSDID("$FreeBSD$");
77		77
78	#include "opt_vm.h"	78	#include "opt_vm.h"
		79	#include "opt_kdtrace.h"
79	#include <sys/param.h>	80	#include <sys/param.h>
80	#include <sys/systm.h>	81	#include <sys/systm.h>
81	#include <sys/kernel.h>	82	#include <sys/kernel.h>
Lines 89-94 __FBSDID("$FreeBSD$"); Link Here
89	#include <sys/racct.h>	90	#include <sys/racct.h>
90	#include <sys/resourcevar.h>	91	#include <sys/resourcevar.h>
91	#include <sys/sched.h>	92	#include <sys/sched.h>
		93	#include <sys/sdt.h>
92	#include <sys/signalvar.h>	94	#include <sys/signalvar.h>
93	#include <sys/smp.h>	95	#include <sys/smp.h>
94	#include <sys/vnode.h>	96	#include <sys/vnode.h>
Lines 115-124 __FBSDID("$FreeBSD$"); Link Here
115		117
116	/* the kernel process "vm_pageout"*/	118	/* the kernel process "vm_pageout"*/
117	static void vm_pageout(void);	119	static void vm_pageout(void);
		120	static void vm_pageout_init(void);
118	static int vm_pageout_clean(vm_page_t);	121	static int vm_pageout_clean(vm_page_t);
119	static void vm_pageout_scan(struct vm_domain *vmd, int pass);	122	static void vm_pageout_scan(struct vm_domain *vmd, int pass);
120	static void vm_pageout_mightbe_oom(struct vm_domain *vmd, int pass);	123	static void vm_pageout_mightbe_oom(struct vm_domain *vmd, int pass);
121		124
		125	SYSINIT(pagedaemon_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_FIRST, vm_pageout_init,
		126	NULL);
		127
122	struct proc *pageproc;	128	struct proc *pageproc;
123		129
124	static struct kproc_desc page_kp = {	130	static struct kproc_desc page_kp = {
Lines 126-134 static struct kproc_desc page_kp = { Link Here
126	vm_pageout,	132	vm_pageout,
127	&pageproc	133	&pageproc
128	};	134	};
129	SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_FIRST, kproc_start,	135	SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_SECOND, kproc_start,
130	&page_kp);	136	&page_kp);
131		137
		138	SDT_PROVIDER_DEFINE(vm);
		139	SDT_PROBE_DEFINE(vm, , , vm__lowmem_cache);
		140	SDT_PROBE_DEFINE(vm, , , vm__lowmem_scan);
		141
132	#if !defined(NO_SWAPPING)	142	#if !defined(NO_SWAPPING)
133	/* the kernel process "vm_daemon"*/	143	/* the kernel process "vm_daemon"*/
134	static void vm_daemon(void);	144	static void vm_daemon(void);
Lines 663-668 vm_pageout_grow_cache(int tries, vm_paddr_t low, v Link Here
663	* may acquire locks and/or sleep, so they can only be invoked	673	* may acquire locks and/or sleep, so they can only be invoked
664	* when "tries" is greater than zero.	674	* when "tries" is greater than zero.
665	*/	675	*/
		676	SDT_PROBE0(vm, , , vm__lowmem_cache);
666	EVENTHANDLER_INVOKE(vm_lowmem, 0);	677	EVENTHANDLER_INVOKE(vm_lowmem, 0);
667		678
668	/*	679	/*
Lines 904-910 vm_pageout_map_deactivate_pages(map, desired) Link Here
904	* pass 1 - Move inactive to cache or free	915	* pass 1 - Move inactive to cache or free
905	* pass 2 - Launder dirty pages	916	* pass 2 - Launder dirty pages
906	*/	917	*/
907	static void	918	static void __used
908	vm_pageout_scan(struct vm_domain *vmd, int pass)	919	vm_pageout_scan(struct vm_domain *vmd, int pass)
909	{	920	{
910	vm_page_t m, next;	921	vm_page_t m, next;
Lines 925-930 vm_pageout_scan(struct vm_domain *vmd, int pass) Link Here
925	/*	936	/*
926	* Decrease registered cache sizes.	937	* Decrease registered cache sizes.
927	*/	938	*/
		939	SDT_PROBE0(vm, , , vm__lowmem_scan);
928	EVENTHANDLER_INVOKE(vm_lowmem, 0);	940	EVENTHANDLER_INVOKE(vm_lowmem, 0);
929	/*	941	/*
930	* We do this explicitly after the caches have been	942	* We do this explicitly after the caches have been
Lines 1650-1664 vm_pageout_worker(void *arg) Link Here
1650	}	1662	}
1651		1663
1652	/*	1664	/*
1653	* vm_pageout is the high level pageout daemon.	1665	* vm_pageout_init initialises basic pageout daemon settings.
1654	*/	1666	*/
1655	static void	1667	static void
1656	vm_pageout(void)	1668	vm_pageout_init(void)
1657	{	1669	{
1658	#if MAXMEMDOM > 1
1659	int error, i;
1660	#endif
1661
1662	/*	1670	/*
1663	* Initialize some paging parameters.	1671	* Initialize some paging parameters.
1664	*/	1672	*/
Lines 1704-1710 static void Link Here
1704	/* XXX does not really belong here */	1712	/* XXX does not really belong here */
1705	if (vm_page_max_wired == 0)	1713	if (vm_page_max_wired == 0)
1706	vm_page_max_wired = cnt.v_free_count / 3;	1714	vm_page_max_wired = cnt.v_free_count / 3;
		1715	}
1707		1716
		1717	/*
		1718	* vm_pageout is the high level pageout daemon.
		1719	*/
		1720	static void
		1721	vm_pageout(void)
		1722	{
		1723	#if MAXMEMDOM > 1
		1724	int error, i;
		1725	#endif
		1726
1708	swap_pager_swap_init();	1727	swap_pager_swap_init();
1709	#if MAXMEMDOM > 1	1728	#if MAXMEMDOM > 1
1710	for (i = 1; i < vm_ndomains; i++) {	1729	for (i = 1; i < vm_ndomains; i++) {