Attachment #182268 for bug #209759




SYSCTL_UINT(_kern_geom_eli, OID_AUTO, batch, CTLFLAG_RWTUN, &g_eli_batch, 0,
    "Use crypto operations batching");

uma_zone_t g_eli_zone;
static u_int g_eli_uma_reserve = 1;
SYSCTL_UINT(_kern_geom_eli, OID_AUTO, uma_reserve, CTLFLAG_RDTUN,
    &g_eli_uma_reserve, 0, "Items to pre-allocate in dedicated uma zone "
    "and reserve for unauthenticated writes to onetime disks");

u_int g_eli_all_writes_use_uma = 0;
SYSCTL_UINT(_kern_geom_eli, OID_AUTO, use_uma_for_all_writes, CTLFLAG_RDTUN,
    &g_eli_all_writes_use_uma, 0, "Use the dedicated uma zone for all "
    "unauthenticated writes that fit, not just for writes to onetime "
    "providers. May reduce write latency but also increases memory usage.");

/*
 * Items in the dedicated uma zone have a fixed size and need
 * to be big enough for all bio lengths for which it is being used.
 *
 * g_eli_max_uma_bio_length is the largest amount of data geli can receive
 * in a row and fit into an uma zone item, it is used to calculate
 * g_eli_zone_item_size which additionally accounts for the encryption
 * overhead, which depends on the number of sectors.
 */
u_int g_eli_max_uma_bio_length = MAXPHYS;
SYSCTL_UINT(_kern_geom_eli, OID_AUTO, max_uma_bio_length, CTLFLAG_RDTUN,
    &g_eli_max_uma_bio_length, 0,
    "Maximum bio length for which the uma zone is being used.");
u_int g_eli_zone_item_size;

/*
 * Passphrase cached during boot, in order to be more user-friendly if
 * there are multiple providers using the same passphrase.

	pbp->bio_inbed++;
	if (pbp->bio_inbed < pbp->bio_children)
		return;
	sc = pbp->bio_to->geom->softc;
	g_eli_free_driver2(pbp, sc->sc_flags);
	if (pbp->bio_error != 0) {
		G_ELI_LOGREQ(0, pbp, "%s() failed (error=%d)", __func__,
		    pbp->bio_error);

	/*
	 * Write is finished, send it up.
	 */
	sc = pbp->bio_to->geom->softc;
	g_io_deliver(pbp, pbp->bio_error);
	if (sc != NULL)
		atomic_subtract_int(&sc->sc_inflight, 1);

g_eli_init(struct g_class *mp)
{

	/*
	 * Calculate uma zone item size based on the largest bio length
	 * we need to use it for.
	 *
	 * 512 bytes is the smallest sector size supported and results in the
	 * largest overhead. If larger sectors are being used, we'll just waste
	 * a bit more memory.
	 *
	 * We currently only use the zone for unauthenticated writes,
	 * otherwise the item size would have to be even bigger.
	 */
	g_eli_zone_item_size = (g_eli_max_uma_bio_length + \
	    (g_eli_max_uma_bio_length / 512) * \
	    (sizeof(struct cryptop) + sizeof(struct cryptodesc)));

	G_ELI_DEBUG(3, "Using uma zone item size %d for max bio length %d",
	    g_eli_zone_item_size, g_eli_max_uma_bio_length);
	g_eli_zone = uma_zcreate("g_eli", g_eli_zone_item_size, NULL, NULL,
	    NULL, NULL, 0, UMA_ZONE_NOFREE);
	if (0 < g_eli_uma_reserve) {
		/* Increase the chances that items are available when needed. */
		uma_prealloc(g_eli_zone, g_eli_uma_reserve);
		uma_zone_reserve(g_eli_zone, g_eli_uma_reserve);
	}

	g_eli_pre_sync = EVENTHANDLER_REGISTER(shutdown_pre_sync,
	    g_eli_shutdown_pre_sync, mp, SHUTDOWN_PRI_FIRST);
	if (g_eli_pre_sync == NULL)

g_eli_fini(struct g_class *mp)
{

	uma_zdestroy(g_eli_zone);
	if (g_eli_pre_sync != NULL)
		EVENTHANDLER_DEREGISTER(shutdown_pre_sync, g_eli_pre_sync);
}

Lines 139-144 Link Here

(-)b/sys/geom/eli/g_eli.h (+4 lines)
139	#define G_ELI_CRYPTO_SW 2	139	#define G_ELI_CRYPTO_SW 2
140		140
141	#ifdef _KERNEL	141	#ifdef _KERNEL
		142	extern u_int g_eli_max_uma_bio_length;
		143	extern u_int g_eli_zone_item_size;
		144
142	extern int g_eli_debug;	145	extern int g_eli_debug;
143	extern u_int g_eli_overwrites;	146	extern u_int g_eli_overwrites;
144	extern u_int g_eli_batch;	147	extern u_int g_eli_batch;
Lines 705-709 void g_eli_key_init(struct g_eli_softc *sc); Link Here
705	void g_eli_key_destroy(struct g_eli_softc *sc);	708	void g_eli_key_destroy(struct g_eli_softc *sc);
706	uint8_t g_eli_key_hold(struct g_eli_softc sc, off_t offset, size_t blocksize);	709	uint8_t g_eli_key_hold(struct g_eli_softc sc, off_t offset, size_t blocksize);
707	void g_eli_key_drop(struct g_eli_softc sc, uint8_t rawkey);	710	void g_eli_key_drop(struct g_eli_softc sc, uint8_t rawkey);
		711	void g_eli_free_driver2(struct bio *bp, int sc_flags);
708	#endif	712	#endif
709	#endif /* !_G_ELI_H_ */	713	#endif /* !_G_ELI_H_ */

Lines 49-54 __FBSDID("$FreeBSD$"); Link Here

(-)b/sys/geom/eli/g_eli_privacy.c (-7 / +76 lines)
49	#include <geom/eli/g_eli.h>	49	#include <geom/eli/g_eli.h>
50	#include <geom/eli/pkcs5v2.h>	50	#include <geom/eli/pkcs5v2.h>
51		51
		52	extern u_int g_eli_all_writes_use_uma;
		53	extern uma_zone_t g_eli_zone;
		54
52	/*	55	/*
53	* Code paths:	56	* Code paths:
54	* BIO_READ:	57	* BIO_READ:
Lines 59-64 __FBSDID("$FreeBSD$"); Link Here
59		62
60	MALLOC_DECLARE(M_ELI);	63	MALLOC_DECLARE(M_ELI);
61		64
		65	static int
		66	g_eli_use_uma_zone(const struct bio *bp, int sc_flags)
		67	{
		68
		69	return (bp->bio_cmd == BIO_WRITE &&
		70	bp->bio_length <= g_eli_max_uma_bio_length &&
		71	((sc_flags & G_ELI_FLAG_ONETIME) != 0 \|\| g_eli_all_writes_use_uma) &&
		72	(sc_flags & G_ELI_FLAG_AUTH) == 0);
		73	}
		74
		75	static void *
		76	g_eli_alloc_driver2(const struct bio *bp, size_t size, int sc_flags)
		77	{
		78	void *p;
		79
		80	/*
		81	* While requests from the authentication code path should
		82	* be handled correctly (by never using the uma zone) they
		83	* currently aren't expected to occur at all.
		84	*/
		85	KASSERT((sc_flags & G_ELI_FLAG_AUTH) == 0,
		86	("g_eli_alloc_driver2() used for authenticated write"));
		87
		88	if (g_eli_use_uma_zone(bp, sc_flags)) {
		89	int uma_flags;
		90
		91	KASSERT(size <= g_eli_zone_item_size,
		92	("Insufficient g_eli_zone_item_size %u < %u",
		93	(unsigned)g_eli_zone_item_size, (unsigned)size));
		94	/*
		95	* Writes to onetime providers are likely to originate
		96	* from the page daemon, therefore we try to get the
		97	* memory a bit harder for them to prevent vm deadlocks.
		98	*/
		99	if ((sc_flags & G_ELI_FLAG_ONETIME) != 0)
		100	uma_flags = M_NOWAIT\|M_USE_RESERVE;
		101	else
		102	uma_flags = M_WAITOK;
		103
		104	G_ELI_DEBUG(3, "Using uma zone for size: %jd, bio_length: %jd",
		105	size, bp->bio_length);
		106	while (NULL == (p = uma_zalloc(g_eli_zone, uma_flags))) {
		107	/* Only reachable for onetime providers */
		108	pause("g_eli:uma", min(hz/1000, 1));
		109	}
		110	} else {
		111	G_ELI_DEBUG(3, "Using malloc for size: %jd, bio_length: %jd",
		112	size, bp->bio_length);
		113	p = malloc(size, M_ELI, M_WAITOK);
		114	}
		115
		116	return (p);
		117	}
		118
		119	void
		120	g_eli_free_driver2(struct bio *bp, int sc_flags)
		121	{
		122
		123	if (g_eli_use_uma_zone(bp, sc_flags)) {
		124	G_ELI_DEBUG(3, "Using uma_free for bio_length: %jd",
		125	bp->bio_length);
		126	uma_zfree(g_eli_zone, bp->bio_driver2);
		127	} else {
		128	G_ELI_DEBUG(3, "Using free for bio_length: %jd",
129	bp->bio_length);
130	free(bp->bio_driver2, M_ELI);
131	}
132	bp->bio_driver2 = NULL;
133	}
134
62	/*	135	/*
63	* The function is called after we read and decrypt data.	136	* The function is called after we read and decrypt data.
64	*	137	*
Lines 94-101 g_eli_crypto_read_done(struct cryptop *crp) Link Here
94	*/	167	*/
95	if (bp->bio_inbed < bp->bio_children)	168	if (bp->bio_inbed < bp->bio_children)
96	return (0);	169	return (0);
97	free(bp->bio_driver2, M_ELI);	170	g_eli_free_driver2(bp, sc->sc_flags);
98	bp->bio_driver2 = NULL;
99	if (bp->bio_error != 0) {	171	if (bp->bio_error != 0) {
100	G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).",	172	G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).",
101	bp->bio_error);	173	bp->bio_error);
Lines 153-160 g_eli_crypto_write_done(struct cryptop *crp) Link Here
153	if (bp->bio_error != 0) {	225	if (bp->bio_error != 0) {
154	G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).",	226	G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).",
155	bp->bio_error);	227	bp->bio_error);
156	free(bp->bio_driver2, M_ELI);	228	g_eli_free_driver2(bp, sc->sc_flags);
157	bp->bio_driver2 = NULL;
158	g_destroy_bio(cbp);	229	g_destroy_bio(cbp);
159	g_io_deliver(bp, bp->bio_error);	230	g_io_deliver(bp, bp->bio_error);
160	atomic_subtract_int(&sc->sc_inflight, 1);	231	atomic_subtract_int(&sc->sc_inflight, 1);
Lines 259-265 g_eli_crypto_run(struct g_eli_worker wr, struct bio bp) Link Here
259	*/	330	*/
260	if (bp->bio_cmd == BIO_WRITE)	331	if (bp->bio_cmd == BIO_WRITE)
261	size += bp->bio_length;	332	size += bp->bio_length;
262	p = malloc(size, M_ELI, M_WAITOK);	333	p = g_eli_alloc_driver2(bp, size, sc->sc_flags);
263		334
264	bp->bio_inbed = 0;	335	bp->bio_inbed = 0;
265	bp->bio_children = nsec;	336	bp->bio_children = nsec;
266	-
267	UMA_ZONE_OFFPAGE\|UMA_ZONE_HASH	337	UMA_ZONE_OFFPAGE\|UMA_ZONE_HASH
268	--
269	sys/geom/eli/g_eli.c \| 9 ++++++++-	338	sys/geom/eli/g_eli.c \| 9 ++++++++-
270	1 file changed, 8 insertions(+), 1 deletion(-)	339	1 file changed, 8 insertions(+), 1 deletion(-)





	G_ELI_DEBUG(3, "Using uma zone item size %d for max bio length %d",
	    g_eli_zone_item_size, g_eli_max_uma_bio_length);

	/*
	 * XXX: Setting UMA_ZONE_OFFPAGE|UMA_ZONE_HASH instead of
	 * letting uma decide itself works around:
	 * https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=218911
	 * Once this is fixed, no flags are needed anymore.
	 */
	g_eli_zone = uma_zcreate("g_eli", g_eli_zone_item_size, NULL, NULL,
	    NULL, NULL, UMA_ZONE_OFFPAGE|UMA_ZONE_HASH, UMA_ZONE_NOFREE);
	if (0 < g_eli_uma_reserve) {
		/* Increase the chances that items are available when needed. */
		uma_prealloc(g_eli_zone, g_eli_uma_reserve);
- 

Return to bug 209759

Lines 1297-1304 g_eli_init(struct g_class *mp) Link Here

(-)b/sys/geom/eli/g_eli.c (-2 / +8 lines)
1297		1297
1298	G_ELI_DEBUG(3, "Using uma zone item size %d for max bio length %d",	1298	G_ELI_DEBUG(3, "Using uma zone item size %d for max bio length %d",
1299	g_eli_zone_item_size, g_eli_max_uma_bio_length);	1299	g_eli_zone_item_size, g_eli_max_uma_bio_length);
		1300
		1301	/*
		1302	* XXX: Setting UMA_ZONE_OFFPAGE\|UMA_ZONE_HASH instead of
		1303	* letting uma decide itself works around:
		1304	* https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=218911
		1305	* Once this is fixed, no flags are needed anymore.
		1306	*/
1300	g_eli_zone = uma_zcreate("g_eli", g_eli_zone_item_size, NULL, NULL,	1307	g_eli_zone = uma_zcreate("g_eli", g_eli_zone_item_size, NULL, NULL,
1301	NULL, NULL, 0, UMA_ZONE_NOFREE);	1308	NULL, NULL, UMA_ZONE_OFFPAGE\|UMA_ZONE_HASH, UMA_ZONE_NOFREE);
1302	if (0 < g_eli_uma_reserve) {	1309	if (0 < g_eli_uma_reserve) {
1303	/* Increase the chances that items are available when needed. */	1310	/* Increase the chances that items are available when needed. */
1304	uma_prealloc(g_eli_zone, g_eli_uma_reserve);	1311	uma_prealloc(g_eli_zone, g_eli_uma_reserve);
1305	-

Lines 82-87 u_int g_eli_batch = 0; Link Here

(-)b/sys/geom/eli/g_eli.c (-3 / +55 lines)
82	SYSCTL_UINT(_kern_geom_eli, OID_AUTO, batch, CTLFLAG_RWTUN, &g_eli_batch, 0,	82	SYSCTL_UINT(_kern_geom_eli, OID_AUTO, batch, CTLFLAG_RWTUN, &g_eli_batch, 0,
83	"Use crypto operations batching");	83	"Use crypto operations batching");
84		84
		85	uma_zone_t g_eli_zone;
		86	static u_int g_eli_uma_reserve = 1;
		87	SYSCTL_UINT(_kern_geom_eli, OID_AUTO, uma_reserve, CTLFLAG_RDTUN,
		88	&g_eli_uma_reserve, 0, "Items to pre-allocate in dedicated uma zone "
		89	"and reserve for unauthenticated writes to onetime disks");
		90
		91	u_int g_eli_all_writes_use_uma = 0;
		92	SYSCTL_UINT(_kern_geom_eli, OID_AUTO, use_uma_for_all_writes, CTLFLAG_RDTUN,
		93	&g_eli_all_writes_use_uma, 0, "Use the dedicated uma zone for all "
		94	"unauthenticated writes that fit, not just for writes to onetime "
		95	"providers. May reduce write latency but also increases memory usage.");
		96
		97	/*
		98	* Items in the dedicated uma zone have a fixed size and need
		99	* to be big enough for all bio lengths for which it is being used.
		100	*
		101	* g_eli_max_uma_bio_length is the largest amount of data geli can receive
		102	* in a row and fit into an uma zone item, it is used to calculate
		103	* g_eli_zone_item_size which additionally accounts for the encryption
		104	* overhead, which depends on the number of sectors.
		105	*/
		106	u_int g_eli_max_uma_bio_length = MAXPHYS;
		107	SYSCTL_UINT(_kern_geom_eli, OID_AUTO, max_uma_bio_length, CTLFLAG_RDTUN,
		108	&g_eli_max_uma_bio_length, 0,
		109	"Maximum bio length for which the uma zone is being used.");
		110	u_int g_eli_zone_item_size;
		111
85	/*	112	/*
86	* Passphrase cached during boot, in order to be more user-friendly if	113	* Passphrase cached during boot, in order to be more user-friendly if
87	* there are multiple providers using the same passphrase.	114	* there are multiple providers using the same passphrase.
Lines 246-253 g_eli_write_done(struct bio *bp) Link Here
246	pbp->bio_inbed++;	273	pbp->bio_inbed++;
247	if (pbp->bio_inbed < pbp->bio_children)	274	if (pbp->bio_inbed < pbp->bio_children)
248	return;	275	return;
249	free(pbp->bio_driver2, M_ELI);	276	sc = pbp->bio_to->geom->softc;
250	pbp->bio_driver2 = NULL;	277	g_eli_free_driver2(pbp, sc->sc_flags);
251	if (pbp->bio_error != 0) {	278	if (pbp->bio_error != 0) {
252	G_ELI_LOGREQ(0, pbp, "%s() failed (error=%d)", __func__,	279	G_ELI_LOGREQ(0, pbp, "%s() failed (error=%d)", __func__,
253	pbp->bio_error);	280	pbp->bio_error);
Lines 258-264 g_eli_write_done(struct bio *bp) Link Here
258	/*	285	/*
259	* Write is finished, send it up.	286	* Write is finished, send it up.
260	*/	287	*/
261	sc = pbp->bio_to->geom->softc;
262	g_io_deliver(pbp, pbp->bio_error);	288	g_io_deliver(pbp, pbp->bio_error);
263	if (sc != NULL)	289	if (sc != NULL)
264	atomic_subtract_int(&sc->sc_inflight, 1);	290	atomic_subtract_int(&sc->sc_inflight, 1);
Lines 1254-1259 static void Link Here
1254	g_eli_init(struct g_class *mp)	1280	g_eli_init(struct g_class *mp)
1255	{	1281	{
1256		1282
		1283	/*
		1284	* Calculate uma zone item size based on the largest bio length
		1285	* we need to use it for.
		1286	*
		1287	* 512 bytes is the smallest sector size supported and results in the
		1288	* largest overhead. If larger sectors are being used, we'll just waste
		1289	* a bit more memory.
		1290	*
		1291	* We currently only use the zone for unauthenticated writes,
		1292	* otherwise the item size would have to be even bigger.
		1293	*/
		1294	g_eli_zone_item_size = (g_eli_max_uma_bio_length + \
		1295	(g_eli_max_uma_bio_length / 512) * \
		1296	(sizeof(struct cryptop) + sizeof(struct cryptodesc)));
		1297
		1298	G_ELI_DEBUG(3, "Using uma zone item size %d for max bio length %d",
		1299	g_eli_zone_item_size, g_eli_max_uma_bio_length);
		1300	g_eli_zone = uma_zcreate("g_eli", g_eli_zone_item_size, NULL, NULL,
		1301	NULL, NULL, 0, UMA_ZONE_NOFREE);
		1302	if (0 < g_eli_uma_reserve) {
		1303	/* Increase the chances that items are available when needed. */
		1304	uma_prealloc(g_eli_zone, g_eli_uma_reserve);
		1305	uma_zone_reserve(g_eli_zone, g_eli_uma_reserve);
		1306	}
		1307
1257	g_eli_pre_sync = EVENTHANDLER_REGISTER(shutdown_pre_sync,	1308	g_eli_pre_sync = EVENTHANDLER_REGISTER(shutdown_pre_sync,
1258	g_eli_shutdown_pre_sync, mp, SHUTDOWN_PRI_FIRST);	1309	g_eli_shutdown_pre_sync, mp, SHUTDOWN_PRI_FIRST);
1259	if (g_eli_pre_sync == NULL)	1310	if (g_eli_pre_sync == NULL)
Lines 1264-1269 static void Link Here
1264	g_eli_fini(struct g_class *mp)	1315	g_eli_fini(struct g_class *mp)
1265	{	1316	{
1266		1317
		1318	uma_zdestroy(g_eli_zone);
1267	if (g_eli_pre_sync != NULL)	1319	if (g_eli_pre_sync != NULL)
1268	EVENTHANDLER_DEREGISTER(shutdown_pre_sync, g_eli_pre_sync);	1320	EVENTHANDLER_DEREGISTER(shutdown_pre_sync, g_eli_pre_sync);
1269	}	1321	}