Index: sys/compat/linuxkpi/common/include/linux/rcupdate.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/rcupdate.h (revision 359686) +++ sys/compat/linuxkpi/common/include/linux/rcupdate.h (working copy) @@ -35,6 +35,11 @@ #define LINUX_KFREE_RCU_OFFSET_MAX 4096 /* exclusive */ +/* BSD specific defines */ +#define RCU_TYPE_REGULAR 0 +#define RCU_TYPE_SLEEPABLE 1 +#define RCU_TYPE_MAX 2 + #define RCU_INITIALIZER(v) \ ((__typeof(*(v)) *)(v)) @@ -43,27 +48,27 @@ } while (0) #define call_rcu(ptr, func) do { \ - linux_call_rcu(ptr, func); \ + linux_call_rcu(RCU_TYPE_REGULAR, ptr, func); \ } while (0) #define rcu_barrier(void) do { \ - linux_rcu_barrier(); \ + linux_rcu_barrier(RCU_TYPE_REGULAR); \ } while (0) #define rcu_read_lock(void) do { \ - linux_rcu_read_lock(); \ + linux_rcu_read_lock(RCU_TYPE_REGULAR); \ } while (0) #define rcu_read_unlock(void) do { \ - linux_rcu_read_unlock(); \ + linux_rcu_read_unlock(RCU_TYPE_REGULAR);\ } while (0) #define synchronize_rcu(void) do { \ - linux_synchronize_rcu(); \ + linux_synchronize_rcu(RCU_TYPE_REGULAR); \ } while (0) #define synchronize_rcu_expedited(void) do { \ - linux_synchronize_rcu(); \ + linux_synchronize_rcu(RCU_TYPE_REGULAR); \ } while (0) #define kfree_rcu(ptr, rcu_head) do { \ @@ -94,11 +99,11 @@ /* prototypes */ -extern void linux_call_rcu(struct rcu_head *ptr, rcu_callback_t func); -extern void linux_rcu_barrier(void); -extern void linux_rcu_read_lock(void); -extern void linux_rcu_read_unlock(void); -extern void linux_synchronize_rcu(void); +extern void linux_call_rcu(unsigned type, struct rcu_head *ptr, rcu_callback_t func); +extern void linux_rcu_barrier(unsigned type); +extern void linux_rcu_read_lock(unsigned type); +extern void linux_rcu_read_unlock(unsigned type); +extern void linux_synchronize_rcu(unsigned type); /* Empty implementation for !DEBUG */ #define init_rcu_head(...) Index: sys/compat/linuxkpi/common/include/linux/srcu.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/srcu.h (revision 359686) +++ sys/compat/linuxkpi/common/include/linux/srcu.h (working copy) @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015-2017 Mellanox Technologies, Ltd. + * Copyright (c) 2015-2020 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -29,13 +29,16 @@ #ifndef _LINUX_SRCU_H_ #define _LINUX_SRCU_H_ +#include + struct srcu_struct { }; -#define srcu_dereference(ptr,srcu) ((__typeof(*(ptr)) *)(ptr)) +#define srcu_dereference(p, srcu) \ + ((__typeof(*(p)) *)READ_ONCE(p)) #define DEFINE_STATIC_SRCU(name) \ - static struct srcu_struct name = {} + static struct srcu_struct name /* prototypes */ Index: sys/compat/linuxkpi/common/src/linux_rcu.c =================================================================== --- sys/compat/linuxkpi/common/src/linux_rcu.c (revision 359686) +++ sys/compat/linuxkpi/common/src/linux_rcu.c (working copy) @@ -1,6 +1,6 @@ /*- * Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io) - * Copyright (c) 2017 Hans Petter Selasky (hselasky@freebsd.org) + * Copyright (c) 2017-2020 Hans Petter Selasky (hselasky@freebsd.org) * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -90,9 +90,9 @@ */ CTASSERT(offsetof(struct linux_epoch_record, epoch_record) == 0); -static ck_epoch_t linux_epoch; -static struct linux_epoch_head linux_epoch_head; -DPCPU_DEFINE_STATIC(struct linux_epoch_record, linux_epoch_record); +static ck_epoch_t linux_epoch[RCU_TYPE_MAX]; +static struct linux_epoch_head linux_epoch_head[RCU_TYPE_MAX]; +DPCPU_DEFINE_STATIC(struct linux_epoch_record, linux_epoch_record[RCU_TYPE_MAX]); static void linux_rcu_cleaner_func(void *, int); @@ -101,23 +101,27 @@ { struct linux_epoch_head *head; int i; + int j; - ck_epoch_init(&linux_epoch); + for (j = 0; j != RCU_TYPE_MAX; j++) { + ck_epoch_init(&linux_epoch[j]); - head = &linux_epoch_head; + head = &linux_epoch_head[j]; - mtx_init(&head->lock, "LRCU-HEAD", NULL, MTX_DEF); - TASK_INIT(&head->task, 0, linux_rcu_cleaner_func, NULL); - STAILQ_INIT(&head->cb_head); + mtx_init(&head->lock, "LRCU-HEAD", NULL, MTX_DEF); + TASK_INIT(&head->task, 0, linux_rcu_cleaner_func, head); + STAILQ_INIT(&head->cb_head); - CPU_FOREACH(i) { - struct linux_epoch_record *record; + CPU_FOREACH(i) { + struct linux_epoch_record *record; - record = &DPCPU_ID_GET(i, linux_epoch_record); + record = &DPCPU_ID_GET(i, linux_epoch_record[j]); - record->cpuid = i; - ck_epoch_register(&linux_epoch, &record->epoch_record, NULL); - TAILQ_INIT(&record->ts_head); + record->cpuid = i; + ck_epoch_register(&linux_epoch[j], + &record->epoch_record, NULL); + TAILQ_INIT(&record->ts_head); + } } } SYSINIT(linux_rcu_runtime, SI_SUB_CPU, SI_ORDER_ANY, linux_rcu_runtime_init, NULL); @@ -126,16 +130,19 @@ linux_rcu_runtime_uninit(void *arg __unused) { struct linux_epoch_head *head; + int j; - head = &linux_epoch_head; + for (j = 0; j != RCU_TYPE_MAX; j++) { + head = &linux_epoch_head[j]; - /* destroy head lock */ - mtx_destroy(&head->lock); + /* destroy head lock */ + mtx_destroy(&head->lock); + } } SYSUNINIT(linux_rcu_runtime, SI_SUB_LOCK, SI_ORDER_SECOND, linux_rcu_runtime_uninit, NULL); static void -linux_rcu_cleaner_func(void *context __unused, int pending __unused) +linux_rcu_cleaner_func(void *context, int pending __unused) { struct linux_epoch_head *head; struct callback_head *rcu; @@ -143,7 +150,7 @@ linux_set_current(curthread); - head = &linux_epoch_head; + head = context; /* move current callbacks into own queue */ mtx_lock(&head->lock); @@ -152,7 +159,7 @@ mtx_unlock(&head->lock); /* synchronize */ - linux_synchronize_rcu(); + linux_synchronize_rcu(head - linux_epoch_head); /* dispatch all callbacks, if any */ while ((rcu = STAILQ_FIRST(&tmp_head)) != NULL) { @@ -170,7 +177,7 @@ } void -linux_rcu_read_lock(void) +linux_rcu_read_lock(unsigned type) { struct linux_epoch_record *record; struct task_struct *ts; @@ -184,7 +191,7 @@ */ sched_pin(); - record = &DPCPU_GET(linux_epoch_record); + record = &DPCPU_GET(linux_epoch_record[type]); ts = current; /* @@ -200,7 +207,7 @@ } void -linux_rcu_read_unlock(void) +linux_rcu_read_unlock(unsigned type) { struct linux_epoch_record *record; struct task_struct *ts; @@ -208,7 +215,7 @@ if (RCU_SKIP()) return; - record = &DPCPU_GET(linux_epoch_record); + record = &DPCPU_GET(linux_epoch_record[type]); ts = current; /* @@ -283,7 +290,7 @@ } void -linux_synchronize_rcu(void) +linux_synchronize_rcu(unsigned type) { struct thread *td; int was_bound; @@ -314,7 +321,7 @@ td->td_pinned = 0; sched_bind(td, old_cpu); - ck_epoch_synchronize_wait(&linux_epoch, + ck_epoch_synchronize_wait(&linux_epoch[type], &linux_synchronize_rcu_cb, NULL); /* restore CPU binding, if any */ @@ -337,13 +344,13 @@ } void -linux_rcu_barrier(void) +linux_rcu_barrier(unsigned type) { struct linux_epoch_head *head; - linux_synchronize_rcu(); + linux_synchronize_rcu(type); - head = &linux_epoch_head; + head = &linux_epoch_head[type]; /* wait for callbacks to complete */ taskqueue_drain(taskqueue_fast, &head->task); @@ -350,10 +357,10 @@ } void -linux_call_rcu(struct rcu_head *context, rcu_callback_t func) +linux_call_rcu(unsigned type, struct rcu_head *context, rcu_callback_t func) { struct callback_head *rcu = (struct callback_head *)context; - struct linux_epoch_head *head = &linux_epoch_head; + struct linux_epoch_head *head = &linux_epoch_head[type]; mtx_lock(&head->lock); rcu->func = func; @@ -376,7 +383,7 @@ int srcu_read_lock(struct srcu_struct *srcu) { - linux_rcu_read_lock(); + linux_rcu_read_lock(RCU_TYPE_SLEEPABLE); return (0); } @@ -383,17 +390,17 @@ void srcu_read_unlock(struct srcu_struct *srcu, int key __unused) { - linux_rcu_read_unlock(); + linux_rcu_read_unlock(RCU_TYPE_SLEEPABLE); } void synchronize_srcu(struct srcu_struct *srcu) { - linux_synchronize_rcu(); + linux_synchronize_rcu(RCU_TYPE_SLEEPABLE); } void srcu_barrier(struct srcu_struct *srcu) { - linux_rcu_barrier(); + linux_rcu_barrier(RCU_TYPE_SLEEPABLE); } Index: sys/sys/param.h =================================================================== --- sys/sys/param.h (revision 359686) +++ sys/sys/param.h (working copy) @@ -60,7 +60,7 @@ * in the range 5 to 9. */ #undef __FreeBSD_version -#define __FreeBSD_version 1300088 /* Master, propagated to newvers */ +#define __FreeBSD_version 1300089 /* Master, propagated to newvers */ /* * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,