Attachment #213153 for bug #242272

View | Details | Raw Unified | Return to bug 242272
Collapse All | Expand All





#define	LINUX_KFREE_RCU_OFFSET_MAX	4096	/* exclusive */

/* BSD specific defines */
#define	RCU_TYPE_REGULAR 0
#define	RCU_TYPE_SLEEPABLE 1
#define	RCU_TYPE_MAX 2

#define	RCU_INITIALIZER(v)			\
	((__typeof(*(v)) *)(v))


} while (0)

#define	call_rcu(ptr, func) do {		\
	linux_call_rcu(RCU_TYPE_REGULAR, ptr, func);	\
} while (0)

#define	rcu_barrier(void) do {			\
	linux_rcu_barrier(RCU_TYPE_REGULAR);	\
} while (0)

#define	rcu_read_lock(void) do {		\
	linux_rcu_read_lock(RCU_TYPE_REGULAR);	\
} while (0)

#define	rcu_read_unlock(void) do {		\
	linux_rcu_read_unlock(RCU_TYPE_REGULAR);\
} while (0)

#define	synchronize_rcu(void) do {	\
	linux_synchronize_rcu(RCU_TYPE_REGULAR);	\
} while (0)

#define	synchronize_rcu_expedited(void) do {	\
	linux_synchronize_rcu(RCU_TYPE_REGULAR);	\
} while (0)

#define	kfree_rcu(ptr, rcu_head) do {				\


/* prototypes */

extern void linux_call_rcu(unsigned type, struct rcu_head *ptr, rcu_callback_t func);
extern void linux_rcu_barrier(unsigned type);
extern void linux_rcu_read_lock(unsigned type);
extern void linux_rcu_read_unlock(unsigned type);
extern void linux_synchronize_rcu(unsigned type);

/* Empty implementation for !DEBUG */
#define	init_rcu_head(...)

Lines 1-5 Link Here

(-)sys/compat/linuxkpi/common/include/linux/srcu.h (-3 / +6 lines)
1	/*-	1	/*-
2	* Copyright (c) 2015-2017 Mellanox Technologies, Ltd.	2	* Copyright (c) 2015-2020 Mellanox Technologies, Ltd.
3	* All rights reserved.	3	* All rights reserved.
4	*	4	*
5	* Redistribution and use in source and binary forms, with or without	5	* Redistribution and use in source and binary forms, with or without
Lines 29-41 Link Here
29	#ifndef _LINUX_SRCU_H_	29	#ifndef _LINUX_SRCU_H_
30	#define _LINUX_SRCU_H_	30	#define _LINUX_SRCU_H_
31		31
		32	#include <linux/compiler.h>
		33
32	struct srcu_struct {	34	struct srcu_struct {
33	};	35	};
34		36
35	#define srcu_dereference(ptr,srcu) ((__typeof((ptr)) )(ptr))	37	#define srcu_dereference(p, srcu) \
		38	((__typeof((p)) )READ_ONCE(p))
36		39
37	#define DEFINE_STATIC_SRCU(name) \	40	#define DEFINE_STATIC_SRCU(name) \
38	static struct srcu_struct name = {}	41	static struct srcu_struct name
39		42
40	/* prototypes */	43	/* prototypes */
41		44




/*-
 * Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io)
 * Copyright (c) 2017-2020 Hans Petter Selasky (hselasky@freebsd.org)
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without

 */
CTASSERT(offsetof(struct linux_epoch_record, epoch_record) == 0);

static ck_epoch_t linux_epoch[RCU_TYPE_MAX];
static struct linux_epoch_head linux_epoch_head[RCU_TYPE_MAX];
DPCPU_DEFINE_STATIC(struct linux_epoch_record, linux_epoch_record[RCU_TYPE_MAX]);

static void linux_rcu_cleaner_func(void *, int);


{
	struct linux_epoch_head *head;
	int i;
	int j;

	for (j = 0; j != RCU_TYPE_MAX; j++) {
		ck_epoch_init(&linux_epoch[j]);

		head = &linux_epoch_head[j];

		mtx_init(&head->lock, "LRCU-HEAD", NULL, MTX_DEF);
		TASK_INIT(&head->task, 0, linux_rcu_cleaner_func, head);
		STAILQ_INIT(&head->cb_head);

		CPU_FOREACH(i) {
			struct linux_epoch_record *record;

			record = &DPCPU_ID_GET(i, linux_epoch_record[j]);

			record->cpuid = i;
			ck_epoch_register(&linux_epoch[j],
			    &record->epoch_record, NULL);
			TAILQ_INIT(&record->ts_head);
		}
	}
}
SYSINIT(linux_rcu_runtime, SI_SUB_CPU, SI_ORDER_ANY, linux_rcu_runtime_init, NULL);

linux_rcu_runtime_uninit(void *arg __unused)
{
	struct linux_epoch_head *head;
	int j;

	for (j = 0; j != RCU_TYPE_MAX; j++) {
		head = &linux_epoch_head[j];

		/* destroy head lock */
		mtx_destroy(&head->lock);
	}
}
SYSUNINIT(linux_rcu_runtime, SI_SUB_LOCK, SI_ORDER_SECOND, linux_rcu_runtime_uninit, NULL);

static void
linux_rcu_cleaner_func(void *context, int pending __unused)
{
	struct linux_epoch_head *head;
	struct callback_head *rcu;


	linux_set_current(curthread);

	head = context;

	/* move current callbacks into own queue */
	mtx_lock(&head->lock);

	mtx_unlock(&head->lock);

	/* synchronize */
	linux_synchronize_rcu(head - linux_epoch_head);

	/* dispatch all callbacks, if any */
	while ((rcu = STAILQ_FIRST(&tmp_head)) != NULL) {

}

void
linux_rcu_read_lock(unsigned type)
{
	struct linux_epoch_record *record;
	struct task_struct *ts;

	 */
	sched_pin();

	record = &DPCPU_GET(linux_epoch_record[type]);
	ts = current;

	/*

}

void
linux_rcu_read_unlock(unsigned type)
{
	struct linux_epoch_record *record;
	struct task_struct *ts;

	if (RCU_SKIP())
		return;

	record = &DPCPU_GET(linux_epoch_record[type]);
	ts = current;

	/*

}

void
linux_synchronize_rcu(unsigned type)
{
	struct thread *td;
	int was_bound;

	td->td_pinned = 0;
	sched_bind(td, old_cpu);

	ck_epoch_synchronize_wait(&linux_epoch[type],
	    &linux_synchronize_rcu_cb, NULL);

	/* restore CPU binding, if any */

}

void
linux_rcu_barrier(unsigned type)
{
	struct linux_epoch_head *head;

	linux_synchronize_rcu(type);

	head = &linux_epoch_head[type];

	/* wait for callbacks to complete */
	taskqueue_drain(taskqueue_fast, &head->task);

}

void
linux_call_rcu(unsigned type, struct rcu_head *context, rcu_callback_t func)
{
	struct callback_head *rcu = (struct callback_head *)context;
	struct linux_epoch_head *head = &linux_epoch_head[type];

	mtx_lock(&head->lock);
	rcu->func = func;

int
srcu_read_lock(struct srcu_struct *srcu)
{
	linux_rcu_read_lock(RCU_TYPE_SLEEPABLE);
	return (0);
}


void
srcu_read_unlock(struct srcu_struct *srcu, int key __unused)
{
	linux_rcu_read_unlock(RCU_TYPE_SLEEPABLE);
}

void
synchronize_srcu(struct srcu_struct *srcu)
{
	linux_synchronize_rcu(RCU_TYPE_SLEEPABLE);
}

void
srcu_barrier(struct srcu_struct *srcu)
{
	linux_rcu_barrier(RCU_TYPE_SLEEPABLE);
}

Lines 60-66 Link Here

(-)sys/sys/param.h (-1 / +1 lines)
60	* in the range 5 to 9.	60	* in the range 5 to 9.
61	*/	61	*/
62	#undef __FreeBSD_version	62	#undef __FreeBSD_version
63	#define __FreeBSD_version 1300088 /* Master, propagated to newvers */	63	#define __FreeBSD_version 1300089 /* Master, propagated to newvers */
64		64
65	/*	65	/*
66	* __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,	66	* __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,

Return to bug 242272

Lines 1-6 Link Here

(-)sys/compat/linuxkpi/common/src/linux_rcu.c (-36 / +43 lines)
1	/*-	1	/*-
2	* Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io)	2	* Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io)
3	* Copyright (c) 2017 Hans Petter Selasky (hselasky@freebsd.org)	3	* Copyright (c) 2017-2020 Hans Petter Selasky (hselasky@freebsd.org)
4	* All rights reserved.	4	* All rights reserved.
5	*	5	*
6	* Redistribution and use in source and binary forms, with or without	6	* Redistribution and use in source and binary forms, with or without
Lines 90-98 Link Here
90	*/	90	*/
91	CTASSERT(offsetof(struct linux_epoch_record, epoch_record) == 0);	91	CTASSERT(offsetof(struct linux_epoch_record, epoch_record) == 0);
92		92
93	static ck_epoch_t linux_epoch;	93	static ck_epoch_t linux_epoch[RCU_TYPE_MAX];
94	static struct linux_epoch_head linux_epoch_head;	94	static struct linux_epoch_head linux_epoch_head[RCU_TYPE_MAX];
95	DPCPU_DEFINE_STATIC(struct linux_epoch_record, linux_epoch_record);	95	DPCPU_DEFINE_STATIC(struct linux_epoch_record, linux_epoch_record[RCU_TYPE_MAX]);
96		96
97	static void linux_rcu_cleaner_func(void *, int);	97	static void linux_rcu_cleaner_func(void *, int);
98		98
Lines 101-123 Link Here
101	{	101	{
102	struct linux_epoch_head *head;	102	struct linux_epoch_head *head;
103	int i;	103	int i;
		104	int j;
104		105
105	ck_epoch_init(&linux_epoch);	106	for (j = 0; j != RCU_TYPE_MAX; j++) {
		107	ck_epoch_init(&linux_epoch[j]);
106		108
107	head = &linux_epoch_head;	109	head = &linux_epoch_head[j];
108		110
109	mtx_init(&head->lock, "LRCU-HEAD", NULL, MTX_DEF);	111	mtx_init(&head->lock, "LRCU-HEAD", NULL, MTX_DEF);
110	TASK_INIT(&head->task, 0, linux_rcu_cleaner_func, NULL);	112	TASK_INIT(&head->task, 0, linux_rcu_cleaner_func, head);
111	STAILQ_INIT(&head->cb_head);	113	STAILQ_INIT(&head->cb_head);
112		114
113	CPU_FOREACH(i) {	115	CPU_FOREACH(i) {
114	struct linux_epoch_record *record;	116	struct linux_epoch_record *record;
115		117
116	record = &DPCPU_ID_GET(i, linux_epoch_record);	118	record = &DPCPU_ID_GET(i, linux_epoch_record[j]);
117		119
118	record->cpuid = i;	120	record->cpuid = i;
119	ck_epoch_register(&linux_epoch, &record->epoch_record, NULL);	121	ck_epoch_register(&linux_epoch[j],
120	TAILQ_INIT(&record->ts_head);	122	&record->epoch_record, NULL);
		123	TAILQ_INIT(&record->ts_head);
		124	}
121	}	125	}
122	}	126	}
123	SYSINIT(linux_rcu_runtime, SI_SUB_CPU, SI_ORDER_ANY, linux_rcu_runtime_init, NULL);	127	SYSINIT(linux_rcu_runtime, SI_SUB_CPU, SI_ORDER_ANY, linux_rcu_runtime_init, NULL);
Lines 126-141 Link Here
126	linux_rcu_runtime_uninit(void *arg __unused)	130	linux_rcu_runtime_uninit(void *arg __unused)
127	{	131	{
128	struct linux_epoch_head *head;	132	struct linux_epoch_head *head;
		133	int j;
129		134
130	head = &linux_epoch_head;	135	for (j = 0; j != RCU_TYPE_MAX; j++) {
		136	head = &linux_epoch_head[j];
131		137
132	/* destroy head lock */	138	/* destroy head lock */
133	mtx_destroy(&head->lock);	139	mtx_destroy(&head->lock);
		140	}
134	}	141	}
135	SYSUNINIT(linux_rcu_runtime, SI_SUB_LOCK, SI_ORDER_SECOND, linux_rcu_runtime_uninit, NULL);	142	SYSUNINIT(linux_rcu_runtime, SI_SUB_LOCK, SI_ORDER_SECOND, linux_rcu_runtime_uninit, NULL);
136		143
137	static void	144	static void
138	linux_rcu_cleaner_func(void *context __unused, int pending __unused)	145	linux_rcu_cleaner_func(void *context, int pending __unused)
139	{	146	{
140	struct linux_epoch_head *head;	147	struct linux_epoch_head *head;
141	struct callback_head *rcu;	148	struct callback_head *rcu;
Lines 143-149 Link Here
143		150
144	linux_set_current(curthread);	151	linux_set_current(curthread);
145		152
146	head = &linux_epoch_head;	153	head = context;
147		154
148	/* move current callbacks into own queue */	155	/* move current callbacks into own queue */
149	mtx_lock(&head->lock);	156	mtx_lock(&head->lock);
Lines 152-158 Link Here
152	mtx_unlock(&head->lock);	159	mtx_unlock(&head->lock);
153		160
154	/* synchronize */	161	/* synchronize */
155	linux_synchronize_rcu();	162	linux_synchronize_rcu(head - linux_epoch_head);
156		163
157	/* dispatch all callbacks, if any */	164	/* dispatch all callbacks, if any */
158	while ((rcu = STAILQ_FIRST(&tmp_head)) != NULL) {	165	while ((rcu = STAILQ_FIRST(&tmp_head)) != NULL) {
Lines 170-176 Link Here
170	}	177	}
171		178
172	void	179	void
173	linux_rcu_read_lock(void)	180	linux_rcu_read_lock(unsigned type)
174	{	181	{
175	struct linux_epoch_record *record;	182	struct linux_epoch_record *record;
176	struct task_struct *ts;	183	struct task_struct *ts;
Lines 184-190 Link Here
184	*/	191	*/
185	sched_pin();	192	sched_pin();
186		193
187	record = &DPCPU_GET(linux_epoch_record);	194	record = &DPCPU_GET(linux_epoch_record[type]);
188	ts = current;	195	ts = current;
189		196
190	/*	197	/*
Lines 200-206 Link Here
200	}	207	}
201		208
202	void	209	void
203	linux_rcu_read_unlock(void)	210	linux_rcu_read_unlock(unsigned type)
204	{	211	{
205	struct linux_epoch_record *record;	212	struct linux_epoch_record *record;
206	struct task_struct *ts;	213	struct task_struct *ts;
Lines 208-214 Link Here
208	if (RCU_SKIP())	215	if (RCU_SKIP())
209	return;	216	return;
210		217
211	record = &DPCPU_GET(linux_epoch_record);	218	record = &DPCPU_GET(linux_epoch_record[type]);
212	ts = current;	219	ts = current;
213		220
214	/*	221	/*
Lines 283-289 Link Here
283	}	290	}
284		291
285	void	292	void
286	linux_synchronize_rcu(void)	293	linux_synchronize_rcu(unsigned type)
287	{	294	{
288	struct thread *td;	295	struct thread *td;
289	int was_bound;	296	int was_bound;
Lines 314-320 Link Here
314	td->td_pinned = 0;	321	td->td_pinned = 0;
315	sched_bind(td, old_cpu);	322	sched_bind(td, old_cpu);
316		323
317	ck_epoch_synchronize_wait(&linux_epoch,	324	ck_epoch_synchronize_wait(&linux_epoch[type],
318	&linux_synchronize_rcu_cb, NULL);	325	&linux_synchronize_rcu_cb, NULL);
319		326
320	/* restore CPU binding, if any */	327	/* restore CPU binding, if any */
Lines 337-349 Link Here
337	}	344	}
338		345
339	void	346	void
340	linux_rcu_barrier(void)	347	linux_rcu_barrier(unsigned type)
341	{	348	{
342	struct linux_epoch_head *head;	349	struct linux_epoch_head *head;
343		350
344	linux_synchronize_rcu();	351	linux_synchronize_rcu(type);
345		352
346	head = &linux_epoch_head;	353	head = &linux_epoch_head[type];
347		354
348	/* wait for callbacks to complete */	355	/* wait for callbacks to complete */
349	taskqueue_drain(taskqueue_fast, &head->task);	356	taskqueue_drain(taskqueue_fast, &head->task);
Lines 350-359 Link Here
350	}	357	}
351		358
352	void	359	void
353	linux_call_rcu(struct rcu_head *context, rcu_callback_t func)	360	linux_call_rcu(unsigned type, struct rcu_head *context, rcu_callback_t func)
354	{	361	{
355	struct callback_head rcu = (struct callback_head )context;	362	struct callback_head rcu = (struct callback_head )context;
356	struct linux_epoch_head *head = &linux_epoch_head;	363	struct linux_epoch_head *head = &linux_epoch_head[type];
357		364
358	mtx_lock(&head->lock);	365	mtx_lock(&head->lock);
359	rcu->func = func;	366	rcu->func = func;
Lines 376-382 Link Here
376	int	383	int
377	srcu_read_lock(struct srcu_struct *srcu)	384	srcu_read_lock(struct srcu_struct *srcu)
378	{	385	{
379	linux_rcu_read_lock();	386	linux_rcu_read_lock(RCU_TYPE_SLEEPABLE);
380	return (0);	387	return (0);
381	}	388	}
382		389
Lines 383-399 Link Here
383	void	390	void
384	srcu_read_unlock(struct srcu_struct *srcu, int key __unused)	391	srcu_read_unlock(struct srcu_struct *srcu, int key __unused)
385	{	392	{
386	linux_rcu_read_unlock();	393	linux_rcu_read_unlock(RCU_TYPE_SLEEPABLE);
387	}	394	}
388		395
389	void	396	void
390	synchronize_srcu(struct srcu_struct *srcu)	397	synchronize_srcu(struct srcu_struct *srcu)
391	{	398	{
392	linux_synchronize_rcu();	399	linux_synchronize_rcu(RCU_TYPE_SLEEPABLE);
393	}	400	}
394		401
395	void	402	void
396	srcu_barrier(struct srcu_struct *srcu)	403	srcu_barrier(struct srcu_struct *srcu)
397	{	404	{
398	linux_rcu_barrier();	405	linux_rcu_barrier(RCU_TYPE_SLEEPABLE);
399	}	406	}

Lines 35-40 Link Here

(-)sys/compat/linuxkpi/common/include/linux/rcupdate.h (-11 / +16 lines)
35		35
36	#define LINUX_KFREE_RCU_OFFSET_MAX 4096 /* exclusive */	36	#define LINUX_KFREE_RCU_OFFSET_MAX 4096 /* exclusive */
37		37
		38	/* BSD specific defines */
		39	#define RCU_TYPE_REGULAR 0
		40	#define RCU_TYPE_SLEEPABLE 1
		41	#define RCU_TYPE_MAX 2
		42
38	#define RCU_INITIALIZER(v) \	43	#define RCU_INITIALIZER(v) \
39	((__typeof((v)) )(v))	44	((__typeof((v)) )(v))
40		45
Lines 43-69 Link Here
43	} while (0)	48	} while (0)
44		49
45	#define call_rcu(ptr, func) do { \	50	#define call_rcu(ptr, func) do { \
46	linux_call_rcu(ptr, func); \	51	linux_call_rcu(RCU_TYPE_REGULAR, ptr, func); \
47	} while (0)	52	} while (0)
48		53
49	#define rcu_barrier(void) do { \	54	#define rcu_barrier(void) do { \
50	linux_rcu_barrier(); \	55	linux_rcu_barrier(RCU_TYPE_REGULAR); \
51	} while (0)	56	} while (0)
52		57
53	#define rcu_read_lock(void) do { \	58	#define rcu_read_lock(void) do { \
54	linux_rcu_read_lock(); \	59	linux_rcu_read_lock(RCU_TYPE_REGULAR); \
55	} while (0)	60	} while (0)
56		61
57	#define rcu_read_unlock(void) do { \	62	#define rcu_read_unlock(void) do { \
58	linux_rcu_read_unlock(); \	63	linux_rcu_read_unlock(RCU_TYPE_REGULAR);\
59	} while (0)	64	} while (0)
60		65
61	#define synchronize_rcu(void) do { \	66	#define synchronize_rcu(void) do { \
62	linux_synchronize_rcu(); \	67	linux_synchronize_rcu(RCU_TYPE_REGULAR); \
63	} while (0)	68	} while (0)
64		69
65	#define synchronize_rcu_expedited(void) do { \	70	#define synchronize_rcu_expedited(void) do { \
66	linux_synchronize_rcu(); \	71	linux_synchronize_rcu(RCU_TYPE_REGULAR); \
67	} while (0)	72	} while (0)
68		73
69	#define kfree_rcu(ptr, rcu_head) do { \	74	#define kfree_rcu(ptr, rcu_head) do { \
Lines 94-104 Link Here
94		99
95	/* prototypes */	100	/* prototypes */
96		101
97	extern void linux_call_rcu(struct rcu_head *ptr, rcu_callback_t func);	102	extern void linux_call_rcu(unsigned type, struct rcu_head *ptr, rcu_callback_t func);
98	extern void linux_rcu_barrier(void);	103	extern void linux_rcu_barrier(unsigned type);
99	extern void linux_rcu_read_lock(void);	104	extern void linux_rcu_read_lock(unsigned type);
100	extern void linux_rcu_read_unlock(void);	105	extern void linux_rcu_read_unlock(unsigned type);
101	extern void linux_synchronize_rcu(void);	106	extern void linux_synchronize_rcu(unsigned type);
102		107
103	/* Empty implementation for !DEBUG */	108	/* Empty implementation for !DEBUG */
104	#define init_rcu_head(...)	109	#define init_rcu_head(...)